diff options
Diffstat (limited to 'drivers/net/ethernet/stmicro')
70 files changed, 10490 insertions, 2155 deletions
diff --git a/drivers/net/ethernet/stmicro/Kconfig b/drivers/net/ethernet/stmicro/Kconfig index 39ef86360417..cc136b4c9afd 100644 --- a/drivers/net/ethernet/stmicro/Kconfig +++ b/drivers/net/ethernet/stmicro/Kconfig @@ -7,7 +7,7 @@ config NET_VENDOR_STMICRO bool "STMicroelectronics devices" default y depends on HAS_IOMEM - ---help--- + help If you have a network (Ethernet) card based on Synopsys Ethernet IP Cores, say Y. diff --git a/drivers/net/ethernet/stmicro/stmmac/Kconfig b/drivers/net/ethernet/stmicro/stmmac/Kconfig index 338e25a6374e..31ff35174034 100644 --- a/drivers/net/ethernet/stmicro/stmmac/Kconfig +++ b/drivers/net/ethernet/stmicro/stmmac/Kconfig @@ -2,13 +2,14 @@ config STMMAC_ETH tristate "STMicroelectronics Multi-Gigabit Ethernet driver" depends on HAS_IOMEM && HAS_DMA + depends on PTP_1588_CLOCK_OPTIONAL select MII + select PCS_XPCS select PAGE_POOL select PHYLINK select CRC32 - imply PTP_1588_CLOCK select RESET_CONTROLLER - ---help--- + help This is the driver for the Ethernet IPs built around a Synopsys IP Core. @@ -19,7 +20,7 @@ config STMMAC_SELFTESTS depends on INET depends on STMMAC_ETH default n - ---help--- + help This adds support for STMMAC Selftests using ethtool. Enable this feature if you are facing problems with your HW and submit the test results to the netdev Mailing List. @@ -29,7 +30,7 @@ config STMMAC_PLATFORM depends on STMMAC_ETH select MFD_SYSCON default y - ---help--- + help This selects the platform specific bus support for the stmmac driver. This is the driver used on several SoCs: STi, Allwinner, Amlogic Meson, Altera SOCFPGA. @@ -51,7 +52,7 @@ config DWMAC_DWC_QOS_ETH config DWMAC_GENERIC tristate "Generic driver for DWMAC" default STMMAC_PLATFORM - ---help--- + help Generic DWMAC driver for platforms that don't require any platform specific code to function or is using platform data for setup. @@ -65,6 +66,18 @@ config DWMAC_ANARION This selects the Anarion SoC glue layer support for the stmmac driver. +config DWMAC_INGENIC + tristate "Ingenic MAC support" + default MACH_INGENIC + depends on OF && HAS_IOMEM && (MACH_INGENIC || COMPILE_TEST) + select MFD_SYSCON + help + Support for ethernet controller on Ingenic SoCs. + + This selects Ingenic SoCs glue layer support for the stmmac + device driver. This driver is used on for the Ingenic SoCs + MAC ethernet controller. + config DWMAC_IPQ806X tristate "QCA IPQ806x DWMAC support" default ARCH_QCOM @@ -78,12 +91,15 @@ config DWMAC_IPQ806X acceleration features available on this SoC. Network devices will behave like standard non-accelerated ethernet interfaces. + Select the QCOM_SOCINFO config flag to enable specific dwmac + fixup based on the ipq806x SoC revision. + config DWMAC_LPC18XX tristate "NXP LPC18xx/43xx DWMAC support" default ARCH_LPC18XX depends on OF && (ARCH_LPC18XX || COMPILE_TEST) select MFD_SYSCON - ---help--- + help Support for NXP LPC18xx/43xx DWMAC Ethernet. config DWMAC_MEDIATEK @@ -139,8 +155,8 @@ config DWMAC_ROCKCHIP config DWMAC_SOCFPGA tristate "SOCFPGA dwmac support" - default (ARCH_SOCFPGA || ARCH_STRATIX10) - depends on OF && (ARCH_SOCFPGA || ARCH_STRATIX10 || COMPILE_TEST) + default ARCH_INTEL_SOCFPGA + depends on OF && (ARCH_INTEL_SOCFPGA || COMPILE_TEST) select MFD_SYSCON help Support for ethernet controller on Altera SOCFPGA @@ -154,7 +170,7 @@ config DWMAC_STI default ARCH_STI depends on OF && (ARCH_STI || COMPILE_TEST) select MFD_SYSCON - ---help--- + help Support for ethernet controller on STi SOCs. This selects STi SoC glue layer support for the stmmac @@ -166,7 +182,7 @@ config DWMAC_STM32 default ARCH_STM32 depends on OF && HAS_IOMEM && (ARCH_STM32 || COMPILE_TEST) select MFD_SYSCON - ---help--- + help Support for ethernet controller on STM32 SOCs. This selects STM32 SoC glue layer support for the stmmac @@ -177,7 +193,7 @@ config DWMAC_SUNXI tristate "Allwinner GMAC support" default ARCH_SUNXI depends on OF && (ARCH_SUNXI || COMPILE_TEST) - ---help--- + help Support for Allwinner A20/A31 GMAC ethernet controllers. This selects Allwinner SoC glue layer support for the @@ -189,19 +205,68 @@ config DWMAC_SUN8I default ARCH_SUNXI depends on OF && (ARCH_SUNXI || COMPILE_TEST) select MDIO_BUS_MUX - ---help--- + help Support for Allwinner H3 A83T A64 EMAC ethernet controllers. This selects Allwinner SoC glue layer support for the stmmac device driver. This driver is used for H3/A83T/A64 EMAC ethernet controller. + +config DWMAC_IMX8 + tristate "NXP IMX8 DWMAC support" + default ARCH_MXC + depends on OF && (ARCH_MXC || COMPILE_TEST) + select MFD_SYSCON + help + Support for ethernet controller on NXP i.MX8 SOCs. + + This selects NXP SoC glue layer support for the stmmac + device driver. This driver is used for i.MX8 series like + iMX8MP/iMX8DXL GMAC ethernet controller. + +config DWMAC_INTEL_PLAT + tristate "Intel dwmac support" + depends on OF && COMMON_CLK + depends on STMMAC_ETH + help + Support for ethernet controllers on Intel SoCs + + This selects the Intel platform specific glue layer support for + the stmmac device driver. This driver is used for the Intel Keem Bay + SoC. + +config DWMAC_VISCONTI + tristate "Toshiba Visconti DWMAC support" + default ARCH_VISCONTI + depends on OF && COMMON_CLK && (ARCH_VISCONTI || COMPILE_TEST) + help + Support for ethernet controller on Visconti SoCs. + endif +config DWMAC_INTEL + tristate "Intel GMAC support" + default X86 + depends on X86 && STMMAC_ETH && PCI + depends on COMMON_CLK + help + This selects the Intel platform specific bus support for the + stmmac driver. This driver is used for Intel Quark/EHL/TGL. + +config DWMAC_LOONGSON + tristate "Loongson PCI DWMAC support" + default MACH_LOONGSON64 + depends on STMMAC_ETH && PCI + depends on COMMON_CLK + help + This selects the LOONGSON PCI bus support for the stmmac driver, + Support for ethernet controller on Loongson-2K1000 SoC and LS7A1000 bridge. + config STMMAC_PCI tristate "STMMAC PCI bus support" depends on STMMAC_ETH && PCI depends on COMMON_CLK - ---help--- + help This selects the platform specific bus support for the stmmac driver. This driver was tested on XLINX XC2V3000 FF1152AMT0221 D1215994A VIRTEX FPGA board and SNPS QoS IPK Prototyping Kit. diff --git a/drivers/net/ethernet/stmicro/stmmac/Makefile b/drivers/net/ethernet/stmicro/stmmac/Makefile index c59926d96bcc..d4e12e9ace4f 100644 --- a/drivers/net/ethernet/stmicro/stmmac/Makefile +++ b/drivers/net/ethernet/stmicro/stmmac/Makefile @@ -6,6 +6,7 @@ stmmac-objs:= stmmac_main.o stmmac_ethtool.o stmmac_mdio.o ring_mode.o \ mmc_core.o stmmac_hwtstamp.o stmmac_ptp.o dwmac4_descs.o \ dwmac4_dma.o dwmac4_lib.o dwmac4_core.o dwmac5.o hwif.o \ stmmac_tc.o dwxgmac2_core.o dwxgmac2_dma.o dwxgmac2_descs.o \ + stmmac_xdp.o \ $(stmmac-y) stmmac-$(CONFIG_STMMAC_SELFTESTS) += stmmac_selftests.o @@ -13,6 +14,7 @@ stmmac-$(CONFIG_STMMAC_SELFTESTS) += stmmac_selftests.o # Ordering matters. Generic driver must be last. obj-$(CONFIG_STMMAC_PLATFORM) += stmmac-platform.o obj-$(CONFIG_DWMAC_ANARION) += dwmac-anarion.o +obj-$(CONFIG_DWMAC_INGENIC) += dwmac-ingenic.o obj-$(CONFIG_DWMAC_IPQ806X) += dwmac-ipq806x.o obj-$(CONFIG_DWMAC_LPC18XX) += dwmac-lpc18xx.o obj-$(CONFIG_DWMAC_MEDIATEK) += dwmac-mediatek.o @@ -26,9 +28,14 @@ obj-$(CONFIG_DWMAC_STM32) += dwmac-stm32.o obj-$(CONFIG_DWMAC_SUNXI) += dwmac-sunxi.o obj-$(CONFIG_DWMAC_SUN8I) += dwmac-sun8i.o obj-$(CONFIG_DWMAC_DWC_QOS_ETH) += dwmac-dwc-qos-eth.o +obj-$(CONFIG_DWMAC_INTEL_PLAT) += dwmac-intel-plat.o obj-$(CONFIG_DWMAC_GENERIC) += dwmac-generic.o +obj-$(CONFIG_DWMAC_IMX8) += dwmac-imx.o +obj-$(CONFIG_DWMAC_VISCONTI) += dwmac-visconti.o stmmac-platform-objs:= stmmac_platform.o dwmac-altr-socfpga-objs := altr_tse_pcs.o dwmac-socfpga.o -obj-$(CONFIG_STMMAC_PCI) += stmmac-pci.o +obj-$(CONFIG_STMMAC_PCI) += stmmac-pci.o +obj-$(CONFIG_DWMAC_INTEL) += dwmac-intel.o +obj-$(CONFIG_DWMAC_LOONGSON) += dwmac-loongson.o stmmac-pci-objs:= stmmac_pci.o diff --git a/drivers/net/ethernet/stmicro/stmmac/altr_tse_pcs.c b/drivers/net/ethernet/stmicro/stmmac/altr_tse_pcs.c index cd478d2cd871..00f6d347eaf7 100644 --- a/drivers/net/ethernet/stmicro/stmmac/altr_tse_pcs.c +++ b/drivers/net/ethernet/stmicro/stmmac/altr_tse_pcs.c @@ -57,10 +57,6 @@ #define TSE_PCS_USE_SGMII_ENA BIT(0) #define TSE_PCS_IF_USE_SGMII 0x03 -#define SGMII_ADAPTER_CTRL_REG 0x00 -#define SGMII_ADAPTER_DISABLE 0x0001 -#define SGMII_ADAPTER_ENABLE 0x0000 - #define AUTONEGO_LINK_TIMER 20 static int tse_pcs_reset(void __iomem *base, struct tse_pcs *pcs) @@ -202,12 +198,8 @@ void tse_pcs_fix_mac_speed(struct tse_pcs *pcs, struct phy_device *phy_dev, unsigned int speed) { void __iomem *tse_pcs_base = pcs->tse_pcs_base; - void __iomem *sgmii_adapter_base = pcs->sgmii_adapter_base; u32 val; - writew(SGMII_ADAPTER_ENABLE, - sgmii_adapter_base + SGMII_ADAPTER_CTRL_REG); - pcs->autoneg = phy_dev->autoneg; if (phy_dev->autoneg == AUTONEG_ENABLE) { diff --git a/drivers/net/ethernet/stmicro/stmmac/altr_tse_pcs.h b/drivers/net/ethernet/stmicro/stmmac/altr_tse_pcs.h index 442812c0a4bd..694ac25ef426 100644 --- a/drivers/net/ethernet/stmicro/stmmac/altr_tse_pcs.h +++ b/drivers/net/ethernet/stmicro/stmmac/altr_tse_pcs.h @@ -10,6 +10,10 @@ #include <linux/phy.h> #include <linux/timer.h> +#define SGMII_ADAPTER_CTRL_REG 0x00 +#define SGMII_ADAPTER_ENABLE 0x0000 +#define SGMII_ADAPTER_DISABLE 0x0001 + struct tse_pcs { struct device *dev; void __iomem *tse_pcs_base; diff --git a/drivers/net/ethernet/stmicro/stmmac/chain_mode.c b/drivers/net/ethernet/stmicro/stmmac/chain_mode.c index 52971f5293aa..2e8744ac6b91 100644 --- a/drivers/net/ethernet/stmicro/stmmac/chain_mode.c +++ b/drivers/net/ethernet/stmicro/stmmac/chain_mode.c @@ -46,7 +46,7 @@ static int jumbo_frm(void *p, struct sk_buff *skb, int csum) while (len != 0) { tx_q->tx_skbuff[entry] = NULL; - entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE); + entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); desc = tx_q->dma_tx + entry; if (len > bmax) { @@ -137,7 +137,7 @@ static void refill_desc3(void *priv_ptr, struct dma_desc *p) */ p->des3 = cpu_to_le32((unsigned int)(rx_q->dma_rx_phy + (((rx_q->dirty_rx) + 1) % - DMA_RX_SIZE) * + priv->dma_conf.dma_rx_size) * sizeof(struct dma_desc))); } @@ -154,7 +154,8 @@ static void clean_desc3(void *priv_ptr, struct dma_desc *p) * to keep explicit chaining in the descriptor. */ p->des3 = cpu_to_le32((unsigned int)((tx_q->dma_tx_phy + - ((tx_q->dirty_tx + 1) % DMA_TX_SIZE)) + ((tx_q->dirty_tx + 1) % + priv->dma_conf.dma_tx_size)) * sizeof(struct dma_desc))); } diff --git a/drivers/net/ethernet/stmicro/stmmac/common.h b/drivers/net/ethernet/stmicro/stmmac/common.h index 487099092693..6b5d96bced47 100644 --- a/drivers/net/ethernet/stmicro/stmmac/common.h +++ b/drivers/net/ethernet/stmicro/stmmac/common.h @@ -15,6 +15,7 @@ #include <linux/netdevice.h> #include <linux/stmmac.h> #include <linux/phy.h> +#include <linux/pcs/pcs-xpcs.h> #include <linux/module.h> #if IS_ENABLED(CONFIG_VLAN_8021Q) #define STMMAC_VLAN_TAG_USED @@ -32,18 +33,41 @@ #define DWMAC_CORE_4_10 0x41 #define DWMAC_CORE_5_00 0x50 #define DWMAC_CORE_5_10 0x51 +#define DWMAC_CORE_5_20 0x52 #define DWXGMAC_CORE_2_10 0x21 +#define DWXLGMAC_CORE_2_00 0x20 + +/* Device ID */ +#define DWXGMAC_ID 0x76 +#define DWXLGMAC_ID 0x27 #define STMMAC_CHAN0 0 /* Always supported and default for all chips */ -/* These need to be power of two, and >= 4 */ -#define DMA_TX_SIZE 512 -#define DMA_RX_SIZE 512 +/* TX and RX Descriptor Length, these need to be power of two. + * TX descriptor length less than 64 may cause transmit queue timed out error. + * RX descriptor length less than 64 may cause inconsistent Rx chain error. + */ +#define DMA_MIN_TX_SIZE 64 +#define DMA_MAX_TX_SIZE 1024 +#define DMA_DEFAULT_TX_SIZE 512 +#define DMA_MIN_RX_SIZE 64 +#define DMA_MAX_RX_SIZE 1024 +#define DMA_DEFAULT_RX_SIZE 512 #define STMMAC_GET_ENTRY(x, size) ((x + 1) & (size - 1)) #undef FRAME_FILTER_DEBUG /* #define FRAME_FILTER_DEBUG */ +struct stmmac_txq_stats { + unsigned long tx_pkt_n; + unsigned long tx_normal_irq_n; +}; + +struct stmmac_rxq_stats { + unsigned long rx_pkt_n; + unsigned long rx_normal_irq_n; +}; + /* Extra statistic and debug information exposed by ethtool */ struct stmmac_extra_stats { /* Transmit errors */ @@ -169,6 +193,15 @@ struct stmmac_extra_stats { /* TSO */ unsigned long tx_tso_frames; unsigned long tx_tso_nfrags; + /* EST */ + unsigned long mtl_est_cgce; + unsigned long mtl_est_hlbs; + unsigned long mtl_est_hlbf; + unsigned long mtl_est_btre; + unsigned long mtl_est_btrlm; + /* per queue statistics */ + struct stmmac_txq_stats txq_stats[MTL_MAX_TX_QUEUES]; + struct stmmac_rxq_stats rxq_stats[MTL_MAX_RX_QUEUES]; }; /* Safety Feature statistics exposed by ethtool */ @@ -240,6 +273,9 @@ struct stmmac_safety_stats { #define DMA_HW_FEAT_ACTPHYIF 0x70000000 /* Active/selected PHY iface */ #define DEFAULT_DMA_PBL 8 +/* MSI defines */ +#define STMMAC_MSI_VEC_MAX 32 + /* PCS status and mask defines */ #define PCS_ANE_IRQ BIT(2) /* PCS Auto-Negotiation */ #define PCS_LINK_IRQ BIT(1) /* PCS Link */ @@ -281,6 +317,7 @@ enum tx_frame_status { tx_not_ls = 0x1, tx_err = 0x2, tx_dma_own = 0x4, + tx_err_bump_tc = 0x8, }; enum dma_irq_status { @@ -290,12 +327,37 @@ enum dma_irq_status { handle_tx = 0x8, }; +enum dma_irq_dir { + DMA_DIR_RX = 0x1, + DMA_DIR_TX = 0x2, + DMA_DIR_RXTX = 0x3, +}; + +enum request_irq_err { + REQ_IRQ_ERR_ALL, + REQ_IRQ_ERR_TX, + REQ_IRQ_ERR_RX, + REQ_IRQ_ERR_SFTY_UE, + REQ_IRQ_ERR_SFTY_CE, + REQ_IRQ_ERR_LPI, + REQ_IRQ_ERR_WOL, + REQ_IRQ_ERR_MAC, + REQ_IRQ_ERR_NO, +}; + /* EEE and LPI defines */ #define CORE_IRQ_TX_PATH_IN_LPI_MODE (1 << 0) #define CORE_IRQ_TX_PATH_EXIT_LPI_MODE (1 << 1) #define CORE_IRQ_RX_PATH_IN_LPI_MODE (1 << 2) #define CORE_IRQ_RX_PATH_EXIT_LPI_MODE (1 << 3) +/* FPE defines */ +#define FPE_EVENT_UNKNOWN 0 +#define FPE_EVENT_TRSP BIT(0) +#define FPE_EVENT_TVER BIT(1) +#define FPE_EVENT_RRSP BIT(2) +#define FPE_EVENT_RVER BIT(3) + #define CORE_IRQ_MTL_RX_OVERFLOW BIT(8) /* Physical Coding Sublayer */ @@ -369,6 +431,8 @@ struct dma_features { unsigned int estsel; unsigned int fpesel; unsigned int tbssel; + /* Numbers of Auxiliary Snapshot Inputs */ + unsigned int aux_snapshot_n; }; /* RX Buffer size must be multiple of 4/8/16 bytes */ @@ -389,6 +453,7 @@ struct dma_features { /* Default LPI timers */ #define STMMAC_DEFAULT_LIT_LS 0x3E8 #define STMMAC_DEFAULT_TWT_LS 0x1E +#define STMMAC_ET_MAX 0xFFFFF #define STMMAC_CHAIN_MODE 0x1 #define STMMAC_RING_MODE 0x2 @@ -425,6 +490,12 @@ struct mac_link { u32 speed5000; u32 speed10000; } xgmii; + struct { + u32 speed25000; + u32 speed40000; + u32 speed50000; + u32 speed100000; + } xlgmii; }; struct mii_regs { @@ -446,6 +517,7 @@ struct mac_device_info { const struct stmmac_hwtimestamp *ptp; const struct stmmac_tc_ops *tc; const struct stmmac_mmc_ops *mmc; + struct dw_xpcs *xpcs; struct mii_regs mii; /* MII register Addresses */ struct mac_link link; void __iomem *pcsr; /* vpointer to device CSRs */ @@ -456,6 +528,12 @@ struct mac_device_info { unsigned int pcs; unsigned int pmt; unsigned int ps; + unsigned int xlgmac; + unsigned int num_vlan; + u32 vlan_filter[32]; + unsigned int promisc; + bool vlan_fail_q_en; + u8 vlan_fail_q; }; struct stmmac_rx_routing { @@ -467,14 +545,15 @@ int dwmac100_setup(struct stmmac_priv *priv); int dwmac1000_setup(struct stmmac_priv *priv); int dwmac4_setup(struct stmmac_priv *priv); int dwxgmac2_setup(struct stmmac_priv *priv); +int dwxlgmac2_setup(struct stmmac_priv *priv); -void stmmac_set_mac_addr(void __iomem *ioaddr, u8 addr[6], +void stmmac_set_mac_addr(void __iomem *ioaddr, const u8 addr[6], unsigned int high, unsigned int low); void stmmac_get_mac_addr(void __iomem *ioaddr, unsigned char *addr, unsigned int high, unsigned int low); void stmmac_set_mac(void __iomem *ioaddr, bool enable); -void stmmac_dwmac4_set_mac_addr(void __iomem *ioaddr, u8 addr[6], +void stmmac_dwmac4_set_mac_addr(void __iomem *ioaddr, const u8 addr[6], unsigned int high, unsigned int low); void stmmac_dwmac4_get_mac_addr(void __iomem *ioaddr, unsigned char *addr, unsigned int high, unsigned int low); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-anarion.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-anarion.c index d0d2d0fc5f0a..dfbaea06d108 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-anarion.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-anarion.c @@ -84,9 +84,10 @@ static struct anarion_gmac *anarion_config_dt(struct platform_device *pdev) return ERR_PTR(err); switch (phy_mode) { - case PHY_INTERFACE_MODE_RGMII: /* Fall through */ - case PHY_INTERFACE_MODE_RGMII_ID /* Fall through */: - case PHY_INTERFACE_MODE_RGMII_RXID: /* Fall through */ + case PHY_INTERFACE_MODE_RGMII: + fallthrough; + case PHY_INTERFACE_MODE_RGMII_ID: + case PHY_INTERFACE_MODE_RGMII_RXID: case PHY_INTERFACE_MODE_RGMII_TXID: gmac->phy_intf_sel = GMAC_CONFIG_INTF_RGMII; break; @@ -114,7 +115,7 @@ static int anarion_dwmac_probe(struct platform_device *pdev) if (IS_ERR(gmac)) return PTR_ERR(gmac); - plat_dat = stmmac_probe_config_dt(pdev, &stmmac_res.mac); + plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac); if (IS_ERR(plat_dat)) return PTR_ERR(plat_dat); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c index 2342d497348e..80efdeeb0b59 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c @@ -119,23 +119,23 @@ static int dwc_eth_dwmac_config_dt(struct platform_device *pdev, return 0; } -static void *dwc_qos_probe(struct platform_device *pdev, - struct plat_stmmacenet_data *plat_dat, - struct stmmac_resources *stmmac_res) +static int dwc_qos_probe(struct platform_device *pdev, + struct plat_stmmacenet_data *plat_dat, + struct stmmac_resources *stmmac_res) { int err; plat_dat->stmmac_clk = devm_clk_get(&pdev->dev, "apb_pclk"); if (IS_ERR(plat_dat->stmmac_clk)) { dev_err(&pdev->dev, "apb_pclk clock not found.\n"); - return ERR_CAST(plat_dat->stmmac_clk); + return PTR_ERR(plat_dat->stmmac_clk); } err = clk_prepare_enable(plat_dat->stmmac_clk); if (err < 0) { dev_err(&pdev->dev, "failed to enable apb_pclk clock: %d\n", err); - return ERR_PTR(err); + return err; } plat_dat->pclk = devm_clk_get(&pdev->dev, "phy_ref_clk"); @@ -152,11 +152,11 @@ static void *dwc_qos_probe(struct platform_device *pdev, goto disable; } - return NULL; + return 0; disable: clk_disable_unprepare(plat_dat->stmmac_clk); - return ERR_PTR(err); + return err; } static int dwc_qos_remove(struct platform_device *pdev) @@ -267,19 +267,17 @@ static int tegra_eqos_init(struct platform_device *pdev, void *priv) return 0; } -static void *tegra_eqos_probe(struct platform_device *pdev, - struct plat_stmmacenet_data *data, - struct stmmac_resources *res) +static int tegra_eqos_probe(struct platform_device *pdev, + struct plat_stmmacenet_data *data, + struct stmmac_resources *res) { struct device *dev = &pdev->dev; struct tegra_eqos *eqos; int err; eqos = devm_kzalloc(&pdev->dev, sizeof(*eqos), GFP_KERNEL); - if (!eqos) { - err = -ENOMEM; - goto error; - } + if (!eqos) + return -ENOMEM; eqos->dev = &pdev->dev; eqos->regs = res->addr; @@ -363,14 +361,13 @@ bypass_clk_reset_gpio: data->fix_mac_speed = tegra_eqos_fix_speed; data->init = tegra_eqos_init; data->bsp_priv = eqos; + data->sph_disable = 1; err = tegra_eqos_init(pdev, eqos); if (err < 0) goto reset; -out: - return eqos; - + return 0; reset: reset_control_assert(eqos->rst); reset_phy: @@ -384,8 +381,7 @@ disable_slave: disable_master: clk_disable_unprepare(eqos->clk_master); error: - eqos = ERR_PTR(err); - goto out; + return err; } static int tegra_eqos_remove(struct platform_device *pdev) @@ -403,9 +399,9 @@ static int tegra_eqos_remove(struct platform_device *pdev) } struct dwc_eth_dwmac_data { - void *(*probe)(struct platform_device *pdev, - struct plat_stmmacenet_data *data, - struct stmmac_resources *res); + int (*probe)(struct platform_device *pdev, + struct plat_stmmacenet_data *data, + struct stmmac_resources *res); int (*remove)(struct platform_device *pdev); }; @@ -424,7 +420,6 @@ static int dwc_eth_dwmac_probe(struct platform_device *pdev) const struct dwc_eth_dwmac_data *data; struct plat_stmmacenet_data *plat_dat; struct stmmac_resources stmmac_res; - void *priv; int ret; data = device_get_match_data(&pdev->dev); @@ -444,17 +439,13 @@ static int dwc_eth_dwmac_probe(struct platform_device *pdev) if (IS_ERR(stmmac_res.addr)) return PTR_ERR(stmmac_res.addr); - plat_dat = stmmac_probe_config_dt(pdev, &stmmac_res.mac); + plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac); if (IS_ERR(plat_dat)) return PTR_ERR(plat_dat); - priv = data->probe(pdev, plat_dat, &stmmac_res); - if (IS_ERR(priv)) { - ret = PTR_ERR(priv); - - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, "failed to probe subdriver: %d\n", - ret); + ret = data->probe(pdev, plat_dat, &stmmac_res); + if (ret < 0) { + dev_err_probe(&pdev->dev, ret, "failed to probe subdriver\n"); goto remove_config; } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-generic.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-generic.c index fad503820e04..5e731a72cce8 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-generic.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-generic.c @@ -27,7 +27,7 @@ static int dwmac_generic_probe(struct platform_device *pdev) return ret; if (pdev->dev.of_node) { - plat_dat = stmmac_probe_config_dt(pdev, &stmmac_res.mac); + plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac); if (IS_ERR(plat_dat)) { dev_err(&pdev->dev, "dt configuration failed\n"); return PTR_ERR(plat_dat); @@ -71,6 +71,7 @@ err_remove_config_dt: static const struct of_device_id dwmac_generic_match[] = { { .compatible = "st,spear600-gmac"}, + { .compatible = "snps,dwmac-3.40a"}, { .compatible = "snps,dwmac-3.50a"}, { .compatible = "snps,dwmac-3.610"}, { .compatible = "snps,dwmac-3.70a"}, diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-imx.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-imx.c new file mode 100644 index 000000000000..bd52fb7cf486 --- /dev/null +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-imx.c @@ -0,0 +1,319 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * dwmac-imx.c - DWMAC Specific Glue layer for NXP imx8 + * + * Copyright 2020 NXP + * + */ + +#include <linux/clk.h> +#include <linux/gpio/consumer.h> +#include <linux/kernel.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/of_net.h> +#include <linux/phy.h> +#include <linux/platform_device.h> +#include <linux/pm_wakeirq.h> +#include <linux/regmap.h> +#include <linux/slab.h> +#include <linux/stmmac.h> + +#include "stmmac_platform.h" + +#define GPR_ENET_QOS_INTF_MODE_MASK GENMASK(21, 16) +#define GPR_ENET_QOS_INTF_SEL_MII (0x0 << 16) +#define GPR_ENET_QOS_INTF_SEL_RMII (0x4 << 16) +#define GPR_ENET_QOS_INTF_SEL_RGMII (0x1 << 16) +#define GPR_ENET_QOS_CLK_GEN_EN (0x1 << 19) +#define GPR_ENET_QOS_CLK_TX_CLK_SEL (0x1 << 20) +#define GPR_ENET_QOS_RGMII_EN (0x1 << 21) + +struct imx_dwmac_ops { + u32 addr_width; + bool mac_rgmii_txclk_auto_adj; + + int (*set_intf_mode)(struct plat_stmmacenet_data *plat_dat); +}; + +struct imx_priv_data { + struct device *dev; + struct clk *clk_tx; + struct clk *clk_mem; + struct regmap *intf_regmap; + u32 intf_reg_off; + bool rmii_refclk_ext; + + const struct imx_dwmac_ops *ops; + struct plat_stmmacenet_data *plat_dat; +}; + +static int imx8mp_set_intf_mode(struct plat_stmmacenet_data *plat_dat) +{ + struct imx_priv_data *dwmac = plat_dat->bsp_priv; + int val; + + switch (plat_dat->interface) { + case PHY_INTERFACE_MODE_MII: + val = GPR_ENET_QOS_INTF_SEL_MII; + break; + case PHY_INTERFACE_MODE_RMII: + val = GPR_ENET_QOS_INTF_SEL_RMII; + val |= (dwmac->rmii_refclk_ext ? 0 : GPR_ENET_QOS_CLK_TX_CLK_SEL); + break; + case PHY_INTERFACE_MODE_RGMII: + case PHY_INTERFACE_MODE_RGMII_ID: + case PHY_INTERFACE_MODE_RGMII_RXID: + case PHY_INTERFACE_MODE_RGMII_TXID: + val = GPR_ENET_QOS_INTF_SEL_RGMII | + GPR_ENET_QOS_RGMII_EN; + break; + default: + pr_debug("imx dwmac doesn't support %d interface\n", + plat_dat->interface); + return -EINVAL; + } + + val |= GPR_ENET_QOS_CLK_GEN_EN; + return regmap_update_bits(dwmac->intf_regmap, dwmac->intf_reg_off, + GPR_ENET_QOS_INTF_MODE_MASK, val); +}; + +static int +imx8dxl_set_intf_mode(struct plat_stmmacenet_data *plat_dat) +{ + int ret = 0; + + /* TBD: depends on imx8dxl scu interfaces to be upstreamed */ + return ret; +} + +static int imx_dwmac_clks_config(void *priv, bool enabled) +{ + struct imx_priv_data *dwmac = priv; + int ret = 0; + + if (enabled) { + ret = clk_prepare_enable(dwmac->clk_mem); + if (ret) { + dev_err(dwmac->dev, "mem clock enable failed\n"); + return ret; + } + + ret = clk_prepare_enable(dwmac->clk_tx); + if (ret) { + dev_err(dwmac->dev, "tx clock enable failed\n"); + clk_disable_unprepare(dwmac->clk_mem); + return ret; + } + } else { + clk_disable_unprepare(dwmac->clk_tx); + clk_disable_unprepare(dwmac->clk_mem); + } + + return ret; +} + +static int imx_dwmac_init(struct platform_device *pdev, void *priv) +{ + struct plat_stmmacenet_data *plat_dat; + struct imx_priv_data *dwmac = priv; + int ret; + + plat_dat = dwmac->plat_dat; + + if (dwmac->ops->set_intf_mode) { + ret = dwmac->ops->set_intf_mode(plat_dat); + if (ret) + return ret; + } + + return 0; +} + +static void imx_dwmac_exit(struct platform_device *pdev, void *priv) +{ + /* nothing to do now */ +} + +static void imx_dwmac_fix_speed(void *priv, unsigned int speed) +{ + struct plat_stmmacenet_data *plat_dat; + struct imx_priv_data *dwmac = priv; + unsigned long rate; + int err; + + plat_dat = dwmac->plat_dat; + + if (dwmac->ops->mac_rgmii_txclk_auto_adj || + (plat_dat->interface == PHY_INTERFACE_MODE_RMII) || + (plat_dat->interface == PHY_INTERFACE_MODE_MII)) + return; + + switch (speed) { + case SPEED_1000: + rate = 125000000; + break; + case SPEED_100: + rate = 25000000; + break; + case SPEED_10: + rate = 2500000; + break; + default: + dev_err(dwmac->dev, "invalid speed %u\n", speed); + return; + } + + err = clk_set_rate(dwmac->clk_tx, rate); + if (err < 0) + dev_err(dwmac->dev, "failed to set tx rate %lu\n", rate); +} + +static int +imx_dwmac_parse_dt(struct imx_priv_data *dwmac, struct device *dev) +{ + struct device_node *np = dev->of_node; + int err = 0; + + if (of_get_property(np, "snps,rmii_refclk_ext", NULL)) + dwmac->rmii_refclk_ext = true; + + dwmac->clk_tx = devm_clk_get(dev, "tx"); + if (IS_ERR(dwmac->clk_tx)) { + dev_err(dev, "failed to get tx clock\n"); + return PTR_ERR(dwmac->clk_tx); + } + + dwmac->clk_mem = NULL; + if (of_machine_is_compatible("fsl,imx8dxl")) { + dwmac->clk_mem = devm_clk_get(dev, "mem"); + if (IS_ERR(dwmac->clk_mem)) { + dev_err(dev, "failed to get mem clock\n"); + return PTR_ERR(dwmac->clk_mem); + } + } + + if (of_machine_is_compatible("fsl,imx8mp")) { + /* Binding doc describes the property: + is required by i.MX8MP. + is optional for i.MX8DXL. + */ + dwmac->intf_regmap = syscon_regmap_lookup_by_phandle(np, "intf_mode"); + if (IS_ERR(dwmac->intf_regmap)) + return PTR_ERR(dwmac->intf_regmap); + + err = of_property_read_u32_index(np, "intf_mode", 1, &dwmac->intf_reg_off); + if (err) { + dev_err(dev, "Can't get intf mode reg offset (%d)\n", err); + return err; + } + } + + return err; +} + +static int imx_dwmac_probe(struct platform_device *pdev) +{ + struct plat_stmmacenet_data *plat_dat; + struct stmmac_resources stmmac_res; + struct imx_priv_data *dwmac; + const struct imx_dwmac_ops *data; + int ret; + + ret = stmmac_get_platform_resources(pdev, &stmmac_res); + if (ret) + return ret; + + dwmac = devm_kzalloc(&pdev->dev, sizeof(*dwmac), GFP_KERNEL); + if (!dwmac) + return -ENOMEM; + + plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac); + if (IS_ERR(plat_dat)) + return PTR_ERR(plat_dat); + + data = of_device_get_match_data(&pdev->dev); + if (!data) { + dev_err(&pdev->dev, "failed to get match data\n"); + ret = -EINVAL; + goto err_match_data; + } + + dwmac->ops = data; + dwmac->dev = &pdev->dev; + + ret = imx_dwmac_parse_dt(dwmac, &pdev->dev); + if (ret) { + dev_err(&pdev->dev, "failed to parse OF data\n"); + goto err_parse_dt; + } + + plat_dat->addr64 = dwmac->ops->addr_width; + plat_dat->init = imx_dwmac_init; + plat_dat->exit = imx_dwmac_exit; + plat_dat->clks_config = imx_dwmac_clks_config; + plat_dat->fix_mac_speed = imx_dwmac_fix_speed; + plat_dat->bsp_priv = dwmac; + dwmac->plat_dat = plat_dat; + + ret = imx_dwmac_clks_config(dwmac, true); + if (ret) + goto err_clks_config; + + ret = imx_dwmac_init(pdev, dwmac); + if (ret) + goto err_dwmac_init; + + ret = stmmac_dvr_probe(&pdev->dev, plat_dat, &stmmac_res); + if (ret) + goto err_drv_probe; + + return 0; + +err_drv_probe: + imx_dwmac_exit(pdev, plat_dat->bsp_priv); +err_dwmac_init: + imx_dwmac_clks_config(dwmac, false); +err_clks_config: +err_parse_dt: +err_match_data: + stmmac_remove_config_dt(pdev, plat_dat); + return ret; +} + +static struct imx_dwmac_ops imx8mp_dwmac_data = { + .addr_width = 34, + .mac_rgmii_txclk_auto_adj = false, + .set_intf_mode = imx8mp_set_intf_mode, +}; + +static struct imx_dwmac_ops imx8dxl_dwmac_data = { + .addr_width = 32, + .mac_rgmii_txclk_auto_adj = true, + .set_intf_mode = imx8dxl_set_intf_mode, +}; + +static const struct of_device_id imx_dwmac_match[] = { + { .compatible = "nxp,imx8mp-dwmac-eqos", .data = &imx8mp_dwmac_data }, + { .compatible = "nxp,imx8dxl-dwmac-eqos", .data = &imx8dxl_dwmac_data }, + { } +}; +MODULE_DEVICE_TABLE(of, imx_dwmac_match); + +static struct platform_driver imx_dwmac_driver = { + .probe = imx_dwmac_probe, + .remove = stmmac_pltfr_remove, + .driver = { + .name = "imx-dwmac", + .pm = &stmmac_pltfr_pm_ops, + .of_match_table = imx_dwmac_match, + }, +}; +module_platform_driver(imx_dwmac_driver); + +MODULE_AUTHOR("NXP"); +MODULE_DESCRIPTION("NXP imx8 DWMAC Specific Glue layer"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-ingenic.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-ingenic.c new file mode 100644 index 000000000000..378b4dd826bb --- /dev/null +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-ingenic.c @@ -0,0 +1,400 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * dwmac-ingenic.c - Ingenic SoCs DWMAC specific glue layer + * + * Copyright (c) 2021 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com> + */ + +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/kernel.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/of_net.h> +#include <linux/phy.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> +#include <linux/slab.h> +#include <linux/stmmac.h> + +#include "stmmac_platform.h" + +#define MACPHYC_TXCLK_SEL_MASK GENMASK(31, 31) +#define MACPHYC_TXCLK_SEL_OUTPUT 0x1 +#define MACPHYC_TXCLK_SEL_INPUT 0x0 +#define MACPHYC_MODE_SEL_MASK GENMASK(31, 31) +#define MACPHYC_MODE_SEL_RMII 0x0 +#define MACPHYC_TX_SEL_MASK GENMASK(19, 19) +#define MACPHYC_TX_SEL_ORIGIN 0x0 +#define MACPHYC_TX_SEL_DELAY 0x1 +#define MACPHYC_TX_DELAY_MASK GENMASK(18, 12) +#define MACPHYC_RX_SEL_MASK GENMASK(11, 11) +#define MACPHYC_RX_SEL_ORIGIN 0x0 +#define MACPHYC_RX_SEL_DELAY 0x1 +#define MACPHYC_RX_DELAY_MASK GENMASK(10, 4) +#define MACPHYC_SOFT_RST_MASK GENMASK(3, 3) +#define MACPHYC_PHY_INFT_MASK GENMASK(2, 0) +#define MACPHYC_PHY_INFT_RMII 0x4 +#define MACPHYC_PHY_INFT_RGMII 0x1 +#define MACPHYC_PHY_INFT_GMII 0x0 +#define MACPHYC_PHY_INFT_MII 0x0 + +#define MACPHYC_TX_DELAY_PS_MAX 2496 +#define MACPHYC_TX_DELAY_PS_MIN 20 + +#define MACPHYC_RX_DELAY_PS_MAX 2496 +#define MACPHYC_RX_DELAY_PS_MIN 20 + +enum ingenic_mac_version { + ID_JZ4775, + ID_X1000, + ID_X1600, + ID_X1830, + ID_X2000, +}; + +struct ingenic_mac { + const struct ingenic_soc_info *soc_info; + struct device *dev; + struct regmap *regmap; + + int rx_delay; + int tx_delay; +}; + +struct ingenic_soc_info { + enum ingenic_mac_version version; + u32 mask; + + int (*set_mode)(struct plat_stmmacenet_data *plat_dat); +}; + +static int ingenic_mac_init(struct plat_stmmacenet_data *plat_dat) +{ + struct ingenic_mac *mac = plat_dat->bsp_priv; + int ret; + + if (mac->soc_info->set_mode) { + ret = mac->soc_info->set_mode(plat_dat); + if (ret) + return ret; + } + + return 0; +} + +static int jz4775_mac_set_mode(struct plat_stmmacenet_data *plat_dat) +{ + struct ingenic_mac *mac = plat_dat->bsp_priv; + unsigned int val; + + switch (plat_dat->interface) { + case PHY_INTERFACE_MODE_MII: + val = FIELD_PREP(MACPHYC_TXCLK_SEL_MASK, MACPHYC_TXCLK_SEL_INPUT) | + FIELD_PREP(MACPHYC_PHY_INFT_MASK, MACPHYC_PHY_INFT_MII); + dev_dbg(mac->dev, "MAC PHY Control Register: PHY_INTERFACE_MODE_MII\n"); + break; + + case PHY_INTERFACE_MODE_GMII: + val = FIELD_PREP(MACPHYC_TXCLK_SEL_MASK, MACPHYC_TXCLK_SEL_INPUT) | + FIELD_PREP(MACPHYC_PHY_INFT_MASK, MACPHYC_PHY_INFT_GMII); + dev_dbg(mac->dev, "MAC PHY Control Register: PHY_INTERFACE_MODE_GMII\n"); + break; + + case PHY_INTERFACE_MODE_RMII: + val = FIELD_PREP(MACPHYC_TXCLK_SEL_MASK, MACPHYC_TXCLK_SEL_INPUT) | + FIELD_PREP(MACPHYC_PHY_INFT_MASK, MACPHYC_PHY_INFT_RMII); + dev_dbg(mac->dev, "MAC PHY Control Register: PHY_INTERFACE_MODE_RMII\n"); + break; + + case PHY_INTERFACE_MODE_RGMII: + case PHY_INTERFACE_MODE_RGMII_ID: + case PHY_INTERFACE_MODE_RGMII_TXID: + case PHY_INTERFACE_MODE_RGMII_RXID: + val = FIELD_PREP(MACPHYC_TXCLK_SEL_MASK, MACPHYC_TXCLK_SEL_INPUT) | + FIELD_PREP(MACPHYC_PHY_INFT_MASK, MACPHYC_PHY_INFT_RGMII); + dev_dbg(mac->dev, "MAC PHY Control Register: PHY_INTERFACE_MODE_RGMII\n"); + break; + + default: + dev_err(mac->dev, "Unsupported interface %d", plat_dat->interface); + return -EINVAL; + } + + /* Update MAC PHY control register */ + return regmap_update_bits(mac->regmap, 0, mac->soc_info->mask, val); +} + +static int x1000_mac_set_mode(struct plat_stmmacenet_data *plat_dat) +{ + struct ingenic_mac *mac = plat_dat->bsp_priv; + + switch (plat_dat->interface) { + case PHY_INTERFACE_MODE_RMII: + dev_dbg(mac->dev, "MAC PHY Control Register: PHY_INTERFACE_MODE_RMII\n"); + break; + + default: + dev_err(mac->dev, "Unsupported interface %d", plat_dat->interface); + return -EINVAL; + } + + /* Update MAC PHY control register */ + return regmap_update_bits(mac->regmap, 0, mac->soc_info->mask, 0); +} + +static int x1600_mac_set_mode(struct plat_stmmacenet_data *plat_dat) +{ + struct ingenic_mac *mac = plat_dat->bsp_priv; + unsigned int val; + + switch (plat_dat->interface) { + case PHY_INTERFACE_MODE_RMII: + val = FIELD_PREP(MACPHYC_PHY_INFT_MASK, MACPHYC_PHY_INFT_RMII); + dev_dbg(mac->dev, "MAC PHY Control Register: PHY_INTERFACE_MODE_RMII\n"); + break; + + default: + dev_err(mac->dev, "Unsupported interface %d", plat_dat->interface); + return -EINVAL; + } + + /* Update MAC PHY control register */ + return regmap_update_bits(mac->regmap, 0, mac->soc_info->mask, val); +} + +static int x1830_mac_set_mode(struct plat_stmmacenet_data *plat_dat) +{ + struct ingenic_mac *mac = plat_dat->bsp_priv; + unsigned int val; + + switch (plat_dat->interface) { + case PHY_INTERFACE_MODE_RMII: + val = FIELD_PREP(MACPHYC_MODE_SEL_MASK, MACPHYC_MODE_SEL_RMII) | + FIELD_PREP(MACPHYC_PHY_INFT_MASK, MACPHYC_PHY_INFT_RMII); + dev_dbg(mac->dev, "MAC PHY Control Register: PHY_INTERFACE_MODE_RMII\n"); + break; + + default: + dev_err(mac->dev, "Unsupported interface %d", plat_dat->interface); + return -EINVAL; + } + + /* Update MAC PHY control register */ + return regmap_update_bits(mac->regmap, 0, mac->soc_info->mask, val); +} + +static int x2000_mac_set_mode(struct plat_stmmacenet_data *plat_dat) +{ + struct ingenic_mac *mac = plat_dat->bsp_priv; + unsigned int val; + + switch (plat_dat->interface) { + case PHY_INTERFACE_MODE_RMII: + val = FIELD_PREP(MACPHYC_TX_SEL_MASK, MACPHYC_TX_SEL_ORIGIN) | + FIELD_PREP(MACPHYC_RX_SEL_MASK, MACPHYC_RX_SEL_ORIGIN) | + FIELD_PREP(MACPHYC_PHY_INFT_MASK, MACPHYC_PHY_INFT_RMII); + dev_dbg(mac->dev, "MAC PHY Control Register: PHY_INTERFACE_MODE_RMII\n"); + break; + + case PHY_INTERFACE_MODE_RGMII: + case PHY_INTERFACE_MODE_RGMII_ID: + case PHY_INTERFACE_MODE_RGMII_TXID: + case PHY_INTERFACE_MODE_RGMII_RXID: + val = FIELD_PREP(MACPHYC_PHY_INFT_MASK, MACPHYC_PHY_INFT_RGMII); + + if (mac->tx_delay == 0) + val |= FIELD_PREP(MACPHYC_TX_SEL_MASK, MACPHYC_TX_SEL_ORIGIN); + else + val |= FIELD_PREP(MACPHYC_TX_SEL_MASK, MACPHYC_TX_SEL_DELAY) | + FIELD_PREP(MACPHYC_TX_DELAY_MASK, (mac->tx_delay + 9750) / 19500 - 1); + + if (mac->rx_delay == 0) + val |= FIELD_PREP(MACPHYC_RX_SEL_MASK, MACPHYC_RX_SEL_ORIGIN); + else + val |= FIELD_PREP(MACPHYC_RX_SEL_MASK, MACPHYC_RX_SEL_DELAY) | + FIELD_PREP(MACPHYC_RX_DELAY_MASK, (mac->rx_delay + 9750) / 19500 - 1); + + dev_dbg(mac->dev, "MAC PHY Control Register: PHY_INTERFACE_MODE_RGMII\n"); + break; + + default: + dev_err(mac->dev, "Unsupported interface %d", plat_dat->interface); + return -EINVAL; + } + + /* Update MAC PHY control register */ + return regmap_update_bits(mac->regmap, 0, mac->soc_info->mask, val); +} + +static int ingenic_mac_probe(struct platform_device *pdev) +{ + struct plat_stmmacenet_data *plat_dat; + struct stmmac_resources stmmac_res; + struct ingenic_mac *mac; + const struct ingenic_soc_info *data; + u32 tx_delay_ps, rx_delay_ps; + int ret; + + ret = stmmac_get_platform_resources(pdev, &stmmac_res); + if (ret) + return ret; + + plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac); + if (IS_ERR(plat_dat)) + return PTR_ERR(plat_dat); + + mac = devm_kzalloc(&pdev->dev, sizeof(*mac), GFP_KERNEL); + if (!mac) { + ret = -ENOMEM; + goto err_remove_config_dt; + } + + data = of_device_get_match_data(&pdev->dev); + if (!data) { + dev_err(&pdev->dev, "No of match data provided\n"); + ret = -EINVAL; + goto err_remove_config_dt; + } + + /* Get MAC PHY control register */ + mac->regmap = syscon_regmap_lookup_by_phandle(pdev->dev.of_node, "mode-reg"); + if (IS_ERR(mac->regmap)) { + dev_err(&pdev->dev, "%s: Failed to get syscon regmap\n", __func__); + ret = PTR_ERR(mac->regmap); + goto err_remove_config_dt; + } + + if (!of_property_read_u32(pdev->dev.of_node, "tx-clk-delay-ps", &tx_delay_ps)) { + if (tx_delay_ps >= MACPHYC_TX_DELAY_PS_MIN && + tx_delay_ps <= MACPHYC_TX_DELAY_PS_MAX) { + mac->tx_delay = tx_delay_ps * 1000; + } else { + dev_err(&pdev->dev, "Invalid TX clock delay: %dps\n", tx_delay_ps); + ret = -EINVAL; + goto err_remove_config_dt; + } + } + + if (!of_property_read_u32(pdev->dev.of_node, "rx-clk-delay-ps", &rx_delay_ps)) { + if (rx_delay_ps >= MACPHYC_RX_DELAY_PS_MIN && + rx_delay_ps <= MACPHYC_RX_DELAY_PS_MAX) { + mac->rx_delay = rx_delay_ps * 1000; + } else { + dev_err(&pdev->dev, "Invalid RX clock delay: %dps\n", rx_delay_ps); + ret = -EINVAL; + goto err_remove_config_dt; + } + } + + mac->soc_info = data; + mac->dev = &pdev->dev; + + plat_dat->bsp_priv = mac; + + ret = ingenic_mac_init(plat_dat); + if (ret) + goto err_remove_config_dt; + + ret = stmmac_dvr_probe(&pdev->dev, plat_dat, &stmmac_res); + if (ret) + goto err_remove_config_dt; + + return 0; + +err_remove_config_dt: + stmmac_remove_config_dt(pdev, plat_dat); + + return ret; +} + +#ifdef CONFIG_PM_SLEEP +static int ingenic_mac_suspend(struct device *dev) +{ + int ret; + + ret = stmmac_suspend(dev); + + return ret; +} + +static int ingenic_mac_resume(struct device *dev) +{ + struct net_device *ndev = dev_get_drvdata(dev); + struct stmmac_priv *priv = netdev_priv(ndev); + int ret; + + ret = ingenic_mac_init(priv->plat); + if (ret) + return ret; + + ret = stmmac_resume(dev); + + return ret; +} +#endif /* CONFIG_PM_SLEEP */ + +static SIMPLE_DEV_PM_OPS(ingenic_mac_pm_ops, ingenic_mac_suspend, ingenic_mac_resume); + +static struct ingenic_soc_info jz4775_soc_info = { + .version = ID_JZ4775, + .mask = MACPHYC_TXCLK_SEL_MASK | MACPHYC_SOFT_RST_MASK | MACPHYC_PHY_INFT_MASK, + + .set_mode = jz4775_mac_set_mode, +}; + +static struct ingenic_soc_info x1000_soc_info = { + .version = ID_X1000, + .mask = MACPHYC_SOFT_RST_MASK, + + .set_mode = x1000_mac_set_mode, +}; + +static struct ingenic_soc_info x1600_soc_info = { + .version = ID_X1600, + .mask = MACPHYC_SOFT_RST_MASK | MACPHYC_PHY_INFT_MASK, + + .set_mode = x1600_mac_set_mode, +}; + +static struct ingenic_soc_info x1830_soc_info = { + .version = ID_X1830, + .mask = MACPHYC_MODE_SEL_MASK | MACPHYC_SOFT_RST_MASK | MACPHYC_PHY_INFT_MASK, + + .set_mode = x1830_mac_set_mode, +}; + +static struct ingenic_soc_info x2000_soc_info = { + .version = ID_X2000, + .mask = MACPHYC_TX_SEL_MASK | MACPHYC_TX_DELAY_MASK | MACPHYC_RX_SEL_MASK | + MACPHYC_RX_DELAY_MASK | MACPHYC_SOFT_RST_MASK | MACPHYC_PHY_INFT_MASK, + + .set_mode = x2000_mac_set_mode, +}; + +static const struct of_device_id ingenic_mac_of_matches[] = { + { .compatible = "ingenic,jz4775-mac", .data = &jz4775_soc_info }, + { .compatible = "ingenic,x1000-mac", .data = &x1000_soc_info }, + { .compatible = "ingenic,x1600-mac", .data = &x1600_soc_info }, + { .compatible = "ingenic,x1830-mac", .data = &x1830_soc_info }, + { .compatible = "ingenic,x2000-mac", .data = &x2000_soc_info }, + { } +}; +MODULE_DEVICE_TABLE(of, ingenic_mac_of_matches); + +static struct platform_driver ingenic_mac_driver = { + .probe = ingenic_mac_probe, + .remove = stmmac_pltfr_remove, + .driver = { + .name = "ingenic-mac", + .pm = pm_ptr(&ingenic_mac_pm_ops), + .of_match_table = ingenic_mac_of_matches, + }, +}; +module_platform_driver(ingenic_mac_driver); + +MODULE_AUTHOR("周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>"); +MODULE_DESCRIPTION("Ingenic SoCs DWMAC specific glue layer"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel-plat.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel-plat.c new file mode 100644 index 000000000000..06d287f104be --- /dev/null +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel-plat.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Intel DWMAC platform driver + * + * Copyright(C) 2020 Intel Corporation + */ + +#include <linux/ethtool.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> +#include <linux/stmmac.h> + +#include "dwmac4.h" +#include "stmmac.h" +#include "stmmac_platform.h" + +struct intel_dwmac { + struct device *dev; + struct clk *tx_clk; + const struct intel_dwmac_data *data; +}; + +struct intel_dwmac_data { + void (*fix_mac_speed)(void *priv, unsigned int speed); + unsigned long ptp_ref_clk_rate; + unsigned long tx_clk_rate; + bool tx_clk_en; +}; + +static void kmb_eth_fix_mac_speed(void *priv, unsigned int speed) +{ + struct intel_dwmac *dwmac = priv; + unsigned long rate; + int ret; + + rate = clk_get_rate(dwmac->tx_clk); + + switch (speed) { + case SPEED_1000: + rate = 125000000; + break; + + case SPEED_100: + rate = 25000000; + break; + + case SPEED_10: + rate = 2500000; + break; + + default: + dev_err(dwmac->dev, "Invalid speed\n"); + break; + } + + ret = clk_set_rate(dwmac->tx_clk, rate); + if (ret) + dev_err(dwmac->dev, "Failed to configure tx clock rate\n"); +} + +static const struct intel_dwmac_data kmb_data = { + .fix_mac_speed = kmb_eth_fix_mac_speed, + .ptp_ref_clk_rate = 200000000, + .tx_clk_rate = 125000000, + .tx_clk_en = true, +}; + +static const struct of_device_id intel_eth_plat_match[] = { + { .compatible = "intel,keembay-dwmac", .data = &kmb_data }, + { } +}; +MODULE_DEVICE_TABLE(of, intel_eth_plat_match); + +static int intel_eth_plat_probe(struct platform_device *pdev) +{ + struct plat_stmmacenet_data *plat_dat; + struct stmmac_resources stmmac_res; + const struct of_device_id *match; + struct intel_dwmac *dwmac; + unsigned long rate; + int ret; + + ret = stmmac_get_platform_resources(pdev, &stmmac_res); + if (ret) + return ret; + + plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac); + if (IS_ERR(plat_dat)) { + dev_err(&pdev->dev, "dt configuration failed\n"); + return PTR_ERR(plat_dat); + } + + dwmac = devm_kzalloc(&pdev->dev, sizeof(*dwmac), GFP_KERNEL); + if (!dwmac) { + ret = -ENOMEM; + goto err_remove_config_dt; + } + + dwmac->dev = &pdev->dev; + dwmac->tx_clk = NULL; + + match = of_match_device(intel_eth_plat_match, &pdev->dev); + if (match && match->data) { + dwmac->data = (const struct intel_dwmac_data *)match->data; + + if (dwmac->data->fix_mac_speed) + plat_dat->fix_mac_speed = dwmac->data->fix_mac_speed; + + /* Enable TX clock */ + if (dwmac->data->tx_clk_en) { + dwmac->tx_clk = devm_clk_get(&pdev->dev, "tx_clk"); + if (IS_ERR(dwmac->tx_clk)) { + ret = PTR_ERR(dwmac->tx_clk); + goto err_remove_config_dt; + } + + clk_prepare_enable(dwmac->tx_clk); + + /* Check and configure TX clock rate */ + rate = clk_get_rate(dwmac->tx_clk); + if (dwmac->data->tx_clk_rate && + rate != dwmac->data->tx_clk_rate) { + rate = dwmac->data->tx_clk_rate; + ret = clk_set_rate(dwmac->tx_clk, rate); + if (ret) { + dev_err(&pdev->dev, + "Failed to set tx_clk\n"); + goto err_remove_config_dt; + } + } + } + + /* Check and configure PTP ref clock rate */ + rate = clk_get_rate(plat_dat->clk_ptp_ref); + if (dwmac->data->ptp_ref_clk_rate && + rate != dwmac->data->ptp_ref_clk_rate) { + rate = dwmac->data->ptp_ref_clk_rate; + ret = clk_set_rate(plat_dat->clk_ptp_ref, rate); + if (ret) { + dev_err(&pdev->dev, + "Failed to set clk_ptp_ref\n"); + goto err_remove_config_dt; + } + } + } + + plat_dat->bsp_priv = dwmac; + plat_dat->eee_usecs_rate = plat_dat->clk_ptp_rate; + + if (plat_dat->eee_usecs_rate > 0) { + u32 tx_lpi_usec; + + tx_lpi_usec = (plat_dat->eee_usecs_rate / 1000000) - 1; + writel(tx_lpi_usec, stmmac_res.addr + GMAC_1US_TIC_COUNTER); + } + + ret = stmmac_dvr_probe(&pdev->dev, plat_dat, &stmmac_res); + if (ret) { + clk_disable_unprepare(dwmac->tx_clk); + goto err_remove_config_dt; + } + + return 0; + +err_remove_config_dt: + stmmac_remove_config_dt(pdev, plat_dat); + + return ret; +} + +static int intel_eth_plat_remove(struct platform_device *pdev) +{ + struct intel_dwmac *dwmac = get_stmmac_bsp_priv(&pdev->dev); + int ret; + + ret = stmmac_pltfr_remove(pdev); + clk_disable_unprepare(dwmac->tx_clk); + + return ret; +} + +static struct platform_driver intel_eth_plat_driver = { + .probe = intel_eth_plat_probe, + .remove = intel_eth_plat_remove, + .driver = { + .name = "intel-eth-plat", + .pm = &stmmac_pltfr_pm_ops, + .of_match_table = intel_eth_plat_match, + }, +}; +module_platform_driver(intel_eth_plat_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Intel DWMAC platform driver"); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c new file mode 100644 index 000000000000..7deb1f817dac --- /dev/null +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c @@ -0,0 +1,1242 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2020, Intel Corporation + */ + +#include <linux/clk-provider.h> +#include <linux/pci.h> +#include <linux/dmi.h> +#include "dwmac-intel.h" +#include "dwmac4.h" +#include "stmmac.h" +#include "stmmac_ptp.h" + +struct intel_priv_data { + int mdio_adhoc_addr; /* mdio address for serdes & etc */ + unsigned long crossts_adj; + bool is_pse; +}; + +/* This struct is used to associate PCI Function of MAC controller on a board, + * discovered via DMI, with the address of PHY connected to the MAC. The + * negative value of the address means that MAC controller is not connected + * with PHY. + */ +struct stmmac_pci_func_data { + unsigned int func; + int phy_addr; +}; + +struct stmmac_pci_dmi_data { + const struct stmmac_pci_func_data *func; + size_t nfuncs; +}; + +struct stmmac_pci_info { + int (*setup)(struct pci_dev *pdev, struct plat_stmmacenet_data *plat); +}; + +static int stmmac_pci_find_phy_addr(struct pci_dev *pdev, + const struct dmi_system_id *dmi_list) +{ + const struct stmmac_pci_func_data *func_data; + const struct stmmac_pci_dmi_data *dmi_data; + const struct dmi_system_id *dmi_id; + int func = PCI_FUNC(pdev->devfn); + size_t n; + + dmi_id = dmi_first_match(dmi_list); + if (!dmi_id) + return -ENODEV; + + dmi_data = dmi_id->driver_data; + func_data = dmi_data->func; + + for (n = 0; n < dmi_data->nfuncs; n++, func_data++) + if (func_data->func == func) + return func_data->phy_addr; + + return -ENODEV; +} + +static int serdes_status_poll(struct stmmac_priv *priv, int phyaddr, + int phyreg, u32 mask, u32 val) +{ + unsigned int retries = 10; + int val_rd; + + do { + val_rd = mdiobus_read(priv->mii, phyaddr, phyreg); + if ((val_rd & mask) == (val & mask)) + return 0; + udelay(POLL_DELAY_US); + } while (--retries); + + return -ETIMEDOUT; +} + +static int intel_serdes_powerup(struct net_device *ndev, void *priv_data) +{ + struct intel_priv_data *intel_priv = priv_data; + struct stmmac_priv *priv = netdev_priv(ndev); + int serdes_phy_addr = 0; + u32 data = 0; + + if (!intel_priv->mdio_adhoc_addr) + return 0; + + serdes_phy_addr = intel_priv->mdio_adhoc_addr; + + /* Set the serdes rate and the PCLK rate */ + data = mdiobus_read(priv->mii, serdes_phy_addr, + SERDES_GCR0); + + data &= ~SERDES_RATE_MASK; + data &= ~SERDES_PCLK_MASK; + + if (priv->plat->max_speed == 2500) + data |= SERDES_RATE_PCIE_GEN2 << SERDES_RATE_PCIE_SHIFT | + SERDES_PCLK_37p5MHZ << SERDES_PCLK_SHIFT; + else + data |= SERDES_RATE_PCIE_GEN1 << SERDES_RATE_PCIE_SHIFT | + SERDES_PCLK_70MHZ << SERDES_PCLK_SHIFT; + + mdiobus_write(priv->mii, serdes_phy_addr, SERDES_GCR0, data); + + /* assert clk_req */ + data = mdiobus_read(priv->mii, serdes_phy_addr, SERDES_GCR0); + data |= SERDES_PLL_CLK; + mdiobus_write(priv->mii, serdes_phy_addr, SERDES_GCR0, data); + + /* check for clk_ack assertion */ + data = serdes_status_poll(priv, serdes_phy_addr, + SERDES_GSR0, + SERDES_PLL_CLK, + SERDES_PLL_CLK); + + if (data) { + dev_err(priv->device, "Serdes PLL clk request timeout\n"); + return data; + } + + /* assert lane reset */ + data = mdiobus_read(priv->mii, serdes_phy_addr, SERDES_GCR0); + data |= SERDES_RST; + mdiobus_write(priv->mii, serdes_phy_addr, SERDES_GCR0, data); + + /* check for assert lane reset reflection */ + data = serdes_status_poll(priv, serdes_phy_addr, + SERDES_GSR0, + SERDES_RST, + SERDES_RST); + + if (data) { + dev_err(priv->device, "Serdes assert lane reset timeout\n"); + return data; + } + + /* move power state to P0 */ + data = mdiobus_read(priv->mii, serdes_phy_addr, SERDES_GCR0); + + data &= ~SERDES_PWR_ST_MASK; + data |= SERDES_PWR_ST_P0 << SERDES_PWR_ST_SHIFT; + + mdiobus_write(priv->mii, serdes_phy_addr, SERDES_GCR0, data); + + /* Check for P0 state */ + data = serdes_status_poll(priv, serdes_phy_addr, + SERDES_GSR0, + SERDES_PWR_ST_MASK, + SERDES_PWR_ST_P0 << SERDES_PWR_ST_SHIFT); + + if (data) { + dev_err(priv->device, "Serdes power state P0 timeout.\n"); + return data; + } + + /* PSE only - ungate SGMII PHY Rx Clock */ + if (intel_priv->is_pse) + mdiobus_modify(priv->mii, serdes_phy_addr, SERDES_GCR0, + 0, SERDES_PHY_RX_CLK); + + return 0; +} + +static void intel_serdes_powerdown(struct net_device *ndev, void *intel_data) +{ + struct intel_priv_data *intel_priv = intel_data; + struct stmmac_priv *priv = netdev_priv(ndev); + int serdes_phy_addr = 0; + u32 data = 0; + + if (!intel_priv->mdio_adhoc_addr) + return; + + serdes_phy_addr = intel_priv->mdio_adhoc_addr; + + /* PSE only - gate SGMII PHY Rx Clock */ + if (intel_priv->is_pse) + mdiobus_modify(priv->mii, serdes_phy_addr, SERDES_GCR0, + SERDES_PHY_RX_CLK, 0); + + /* move power state to P3 */ + data = mdiobus_read(priv->mii, serdes_phy_addr, SERDES_GCR0); + + data &= ~SERDES_PWR_ST_MASK; + data |= SERDES_PWR_ST_P3 << SERDES_PWR_ST_SHIFT; + + mdiobus_write(priv->mii, serdes_phy_addr, SERDES_GCR0, data); + + /* Check for P3 state */ + data = serdes_status_poll(priv, serdes_phy_addr, + SERDES_GSR0, + SERDES_PWR_ST_MASK, + SERDES_PWR_ST_P3 << SERDES_PWR_ST_SHIFT); + + if (data) { + dev_err(priv->device, "Serdes power state P3 timeout\n"); + return; + } + + /* de-assert clk_req */ + data = mdiobus_read(priv->mii, serdes_phy_addr, SERDES_GCR0); + data &= ~SERDES_PLL_CLK; + mdiobus_write(priv->mii, serdes_phy_addr, SERDES_GCR0, data); + + /* check for clk_ack de-assert */ + data = serdes_status_poll(priv, serdes_phy_addr, + SERDES_GSR0, + SERDES_PLL_CLK, + (u32)~SERDES_PLL_CLK); + + if (data) { + dev_err(priv->device, "Serdes PLL clk de-assert timeout\n"); + return; + } + + /* de-assert lane reset */ + data = mdiobus_read(priv->mii, serdes_phy_addr, SERDES_GCR0); + data &= ~SERDES_RST; + mdiobus_write(priv->mii, serdes_phy_addr, SERDES_GCR0, data); + + /* check for de-assert lane reset reflection */ + data = serdes_status_poll(priv, serdes_phy_addr, + SERDES_GSR0, + SERDES_RST, + (u32)~SERDES_RST); + + if (data) { + dev_err(priv->device, "Serdes de-assert lane reset timeout\n"); + return; + } +} + +static void intel_speed_mode_2500(struct net_device *ndev, void *intel_data) +{ + struct intel_priv_data *intel_priv = intel_data; + struct stmmac_priv *priv = netdev_priv(ndev); + int serdes_phy_addr = 0; + u32 data = 0; + + serdes_phy_addr = intel_priv->mdio_adhoc_addr; + + /* Determine the link speed mode: 2.5Gbps/1Gbps */ + data = mdiobus_read(priv->mii, serdes_phy_addr, + SERDES_GCR); + + if (((data & SERDES_LINK_MODE_MASK) >> SERDES_LINK_MODE_SHIFT) == + SERDES_LINK_MODE_2G5) { + dev_info(priv->device, "Link Speed Mode: 2.5Gbps\n"); + priv->plat->max_speed = 2500; + priv->plat->phy_interface = PHY_INTERFACE_MODE_2500BASEX; + priv->plat->mdio_bus_data->xpcs_an_inband = false; + } else { + priv->plat->max_speed = 1000; + priv->plat->mdio_bus_data->xpcs_an_inband = true; + } +} + +/* Program PTP Clock Frequency for different variant of + * Intel mGBE that has slightly different GPO mapping + */ +static void intel_mgbe_ptp_clk_freq_config(void *npriv) +{ + struct stmmac_priv *priv = (struct stmmac_priv *)npriv; + struct intel_priv_data *intel_priv; + u32 gpio_value; + + intel_priv = (struct intel_priv_data *)priv->plat->bsp_priv; + + gpio_value = readl(priv->ioaddr + GMAC_GPIO_STATUS); + + if (intel_priv->is_pse) { + /* For PSE GbE, use 200MHz */ + gpio_value &= ~PSE_PTP_CLK_FREQ_MASK; + gpio_value |= PSE_PTP_CLK_FREQ_200MHZ; + } else { + /* For PCH GbE, use 200MHz */ + gpio_value &= ~PCH_PTP_CLK_FREQ_MASK; + gpio_value |= PCH_PTP_CLK_FREQ_200MHZ; + } + + writel(gpio_value, priv->ioaddr + GMAC_GPIO_STATUS); +} + +static void get_arttime(struct mii_bus *mii, int intel_adhoc_addr, + u64 *art_time) +{ + u64 ns; + + ns = mdiobus_read(mii, intel_adhoc_addr, PMC_ART_VALUE3); + ns <<= GMAC4_ART_TIME_SHIFT; + ns |= mdiobus_read(mii, intel_adhoc_addr, PMC_ART_VALUE2); + ns <<= GMAC4_ART_TIME_SHIFT; + ns |= mdiobus_read(mii, intel_adhoc_addr, PMC_ART_VALUE1); + ns <<= GMAC4_ART_TIME_SHIFT; + ns |= mdiobus_read(mii, intel_adhoc_addr, PMC_ART_VALUE0); + + *art_time = ns; +} + +static int stmmac_cross_ts_isr(struct stmmac_priv *priv) +{ + return (readl(priv->ioaddr + GMAC_INT_STATUS) & GMAC_INT_TSIE); +} + +static int intel_crosststamp(ktime_t *device, + struct system_counterval_t *system, + void *ctx) +{ + struct intel_priv_data *intel_priv; + + struct stmmac_priv *priv = (struct stmmac_priv *)ctx; + void __iomem *ptpaddr = priv->ptpaddr; + void __iomem *ioaddr = priv->hw->pcsr; + unsigned long flags; + u64 art_time = 0; + u64 ptp_time = 0; + u32 num_snapshot; + u32 gpio_value; + u32 acr_value; + int i; + + if (!boot_cpu_has(X86_FEATURE_ART)) + return -EOPNOTSUPP; + + intel_priv = priv->plat->bsp_priv; + + /* Both internal crosstimestamping and external triggered event + * timestamping cannot be run concurrently. + */ + if (priv->plat->ext_snapshot_en) + return -EBUSY; + + priv->plat->int_snapshot_en = 1; + + mutex_lock(&priv->aux_ts_lock); + /* Enable Internal snapshot trigger */ + acr_value = readl(ptpaddr + PTP_ACR); + acr_value &= ~PTP_ACR_MASK; + switch (priv->plat->int_snapshot_num) { + case AUX_SNAPSHOT0: + acr_value |= PTP_ACR_ATSEN0; + break; + case AUX_SNAPSHOT1: + acr_value |= PTP_ACR_ATSEN1; + break; + case AUX_SNAPSHOT2: + acr_value |= PTP_ACR_ATSEN2; + break; + case AUX_SNAPSHOT3: + acr_value |= PTP_ACR_ATSEN3; + break; + default: + mutex_unlock(&priv->aux_ts_lock); + priv->plat->int_snapshot_en = 0; + return -EINVAL; + } + writel(acr_value, ptpaddr + PTP_ACR); + + /* Clear FIFO */ + acr_value = readl(ptpaddr + PTP_ACR); + acr_value |= PTP_ACR_ATSFC; + writel(acr_value, ptpaddr + PTP_ACR); + /* Release the mutex */ + mutex_unlock(&priv->aux_ts_lock); + + /* Trigger Internal snapshot signal + * Create a rising edge by just toggle the GPO1 to low + * and back to high. + */ + gpio_value = readl(ioaddr + GMAC_GPIO_STATUS); + gpio_value &= ~GMAC_GPO1; + writel(gpio_value, ioaddr + GMAC_GPIO_STATUS); + gpio_value |= GMAC_GPO1; + writel(gpio_value, ioaddr + GMAC_GPIO_STATUS); + + /* Time sync done Indication - Interrupt method */ + if (!wait_event_interruptible_timeout(priv->tstamp_busy_wait, + stmmac_cross_ts_isr(priv), + HZ / 100)) { + priv->plat->int_snapshot_en = 0; + return -ETIMEDOUT; + } + + num_snapshot = (readl(ioaddr + GMAC_TIMESTAMP_STATUS) & + GMAC_TIMESTAMP_ATSNS_MASK) >> + GMAC_TIMESTAMP_ATSNS_SHIFT; + + /* Repeat until the timestamps are from the FIFO last segment */ + for (i = 0; i < num_snapshot; i++) { + read_lock_irqsave(&priv->ptp_lock, flags); + stmmac_get_ptptime(priv, ptpaddr, &ptp_time); + *device = ns_to_ktime(ptp_time); + read_unlock_irqrestore(&priv->ptp_lock, flags); + get_arttime(priv->mii, intel_priv->mdio_adhoc_addr, &art_time); + *system = convert_art_to_tsc(art_time); + } + + system->cycles *= intel_priv->crossts_adj; + priv->plat->int_snapshot_en = 0; + + return 0; +} + +static void intel_mgbe_pse_crossts_adj(struct intel_priv_data *intel_priv, + int base) +{ + if (boot_cpu_has(X86_FEATURE_ART)) { + unsigned int art_freq; + + /* On systems that support ART, ART frequency can be obtained + * from ECX register of CPUID leaf (0x15). + */ + art_freq = cpuid_ecx(ART_CPUID_LEAF); + do_div(art_freq, base); + intel_priv->crossts_adj = art_freq; + } +} + +static void common_default_data(struct plat_stmmacenet_data *plat) +{ + plat->clk_csr = 2; /* clk_csr_i = 20-35MHz & MDC = clk_csr_i/16 */ + plat->has_gmac = 1; + plat->force_sf_dma_mode = 1; + + plat->mdio_bus_data->needs_reset = true; + + /* Set default value for multicast hash bins */ + plat->multicast_filter_bins = HASH_TABLE_SIZE; + + /* Set default value for unicast filter entries */ + plat->unicast_filter_entries = 1; + + /* Set the maxmtu to a default of JUMBO_LEN */ + plat->maxmtu = JUMBO_LEN; + + /* Set default number of RX and TX queues to use */ + plat->tx_queues_to_use = 1; + plat->rx_queues_to_use = 1; + + /* Disable Priority config by default */ + plat->tx_queues_cfg[0].use_prio = false; + plat->rx_queues_cfg[0].use_prio = false; + + /* Disable RX queues routing by default */ + plat->rx_queues_cfg[0].pkt_route = 0x0; +} + +static int intel_mgbe_common_data(struct pci_dev *pdev, + struct plat_stmmacenet_data *plat) +{ + struct fwnode_handle *fwnode; + char clk_name[20]; + int ret; + int i; + + plat->pdev = pdev; + plat->phy_addr = -1; + plat->clk_csr = 5; + plat->has_gmac = 0; + plat->has_gmac4 = 1; + plat->force_sf_dma_mode = 0; + plat->tso_en = 1; + plat->sph_disable = 1; + + /* Multiplying factor to the clk_eee_i clock time + * period to make it closer to 100 ns. This value + * should be programmed such that the clk_eee_time_period * + * (MULT_FACT_100NS + 1) should be within 80 ns to 120 ns + * clk_eee frequency is 19.2Mhz + * clk_eee_time_period is 52ns + * 52ns * (1 + 1) = 104ns + * MULT_FACT_100NS = 1 + */ + plat->mult_fact_100ns = 1; + + plat->rx_sched_algorithm = MTL_RX_ALGORITHM_SP; + + for (i = 0; i < plat->rx_queues_to_use; i++) { + plat->rx_queues_cfg[i].mode_to_use = MTL_QUEUE_DCB; + plat->rx_queues_cfg[i].chan = i; + + /* Disable Priority config by default */ + plat->rx_queues_cfg[i].use_prio = false; + + /* Disable RX queues routing by default */ + plat->rx_queues_cfg[i].pkt_route = 0x0; + } + + for (i = 0; i < plat->tx_queues_to_use; i++) { + plat->tx_queues_cfg[i].mode_to_use = MTL_QUEUE_DCB; + + /* Disable Priority config by default */ + plat->tx_queues_cfg[i].use_prio = false; + /* Default TX Q0 to use TSO and rest TXQ for TBS */ + if (i > 0) + plat->tx_queues_cfg[i].tbs_en = 1; + } + + /* FIFO size is 4096 bytes for 1 tx/rx queue */ + plat->tx_fifo_size = plat->tx_queues_to_use * 4096; + plat->rx_fifo_size = plat->rx_queues_to_use * 4096; + + plat->tx_sched_algorithm = MTL_TX_ALGORITHM_WRR; + plat->tx_queues_cfg[0].weight = 0x09; + plat->tx_queues_cfg[1].weight = 0x0A; + plat->tx_queues_cfg[2].weight = 0x0B; + plat->tx_queues_cfg[3].weight = 0x0C; + plat->tx_queues_cfg[4].weight = 0x0D; + plat->tx_queues_cfg[5].weight = 0x0E; + plat->tx_queues_cfg[6].weight = 0x0F; + plat->tx_queues_cfg[7].weight = 0x10; + + plat->dma_cfg->pbl = 32; + plat->dma_cfg->pblx8 = true; + plat->dma_cfg->fixed_burst = 0; + plat->dma_cfg->mixed_burst = 0; + plat->dma_cfg->aal = 0; + plat->dma_cfg->dche = true; + + plat->axi = devm_kzalloc(&pdev->dev, sizeof(*plat->axi), + GFP_KERNEL); + if (!plat->axi) + return -ENOMEM; + + plat->axi->axi_lpi_en = 0; + plat->axi->axi_xit_frm = 0; + plat->axi->axi_wr_osr_lmt = 1; + plat->axi->axi_rd_osr_lmt = 1; + plat->axi->axi_blen[0] = 4; + plat->axi->axi_blen[1] = 8; + plat->axi->axi_blen[2] = 16; + + plat->ptp_max_adj = plat->clk_ptp_rate; + plat->eee_usecs_rate = plat->clk_ptp_rate; + + /* Set system clock */ + sprintf(clk_name, "%s-%s", "stmmac", pci_name(pdev)); + + plat->stmmac_clk = clk_register_fixed_rate(&pdev->dev, + clk_name, NULL, 0, + plat->clk_ptp_rate); + + if (IS_ERR(plat->stmmac_clk)) { + dev_warn(&pdev->dev, "Fail to register stmmac-clk\n"); + plat->stmmac_clk = NULL; + } + + ret = clk_prepare_enable(plat->stmmac_clk); + if (ret) { + clk_unregister_fixed_rate(plat->stmmac_clk); + return ret; + } + + plat->ptp_clk_freq_config = intel_mgbe_ptp_clk_freq_config; + + /* Set default value for multicast hash bins */ + plat->multicast_filter_bins = HASH_TABLE_SIZE; + + /* Set default value for unicast filter entries */ + plat->unicast_filter_entries = 1; + + /* Set the maxmtu to a default of JUMBO_LEN */ + plat->maxmtu = JUMBO_LEN; + + plat->vlan_fail_q_en = true; + + /* Use the last Rx queue */ + plat->vlan_fail_q = plat->rx_queues_to_use - 1; + + /* For fixed-link setup, we allow phy-mode setting */ + fwnode = dev_fwnode(&pdev->dev); + if (fwnode) { + int phy_mode; + + /* "phy-mode" setting is optional. If it is set, + * we allow either sgmii or 1000base-x for now. + */ + phy_mode = fwnode_get_phy_mode(fwnode); + if (phy_mode >= 0) { + if (phy_mode == PHY_INTERFACE_MODE_SGMII || + phy_mode == PHY_INTERFACE_MODE_1000BASEX) + plat->phy_interface = phy_mode; + else + dev_warn(&pdev->dev, "Invalid phy-mode\n"); + } + } + + /* Intel mgbe SGMII interface uses pcs-xcps */ + if (plat->phy_interface == PHY_INTERFACE_MODE_SGMII || + plat->phy_interface == PHY_INTERFACE_MODE_1000BASEX) { + plat->mdio_bus_data->has_xpcs = true; + plat->mdio_bus_data->xpcs_an_inband = true; + } + + /* For fixed-link setup, we clear xpcs_an_inband */ + if (fwnode) { + struct fwnode_handle *fixed_node; + + fixed_node = fwnode_get_named_child_node(fwnode, "fixed-link"); + if (fixed_node) + plat->mdio_bus_data->xpcs_an_inband = false; + + fwnode_handle_put(fixed_node); + } + + /* Ensure mdio bus scan skips intel serdes and pcs-xpcs */ + plat->mdio_bus_data->phy_mask = 1 << INTEL_MGBE_ADHOC_ADDR; + plat->mdio_bus_data->phy_mask |= 1 << INTEL_MGBE_XPCS_ADDR; + + plat->int_snapshot_num = AUX_SNAPSHOT1; + plat->ext_snapshot_num = AUX_SNAPSHOT0; + + plat->crosststamp = intel_crosststamp; + plat->int_snapshot_en = 0; + + /* Setup MSI vector offset specific to Intel mGbE controller */ + plat->msi_mac_vec = 29; + plat->msi_lpi_vec = 28; + plat->msi_sfty_ce_vec = 27; + plat->msi_sfty_ue_vec = 26; + plat->msi_rx_base_vec = 0; + plat->msi_tx_base_vec = 1; + + return 0; +} + +static int ehl_common_data(struct pci_dev *pdev, + struct plat_stmmacenet_data *plat) +{ + plat->rx_queues_to_use = 8; + plat->tx_queues_to_use = 8; + plat->use_phy_wol = 1; + + plat->safety_feat_cfg->tsoee = 1; + plat->safety_feat_cfg->mrxpee = 1; + plat->safety_feat_cfg->mestee = 1; + plat->safety_feat_cfg->mrxee = 1; + plat->safety_feat_cfg->mtxee = 1; + plat->safety_feat_cfg->epsi = 0; + plat->safety_feat_cfg->edpp = 0; + plat->safety_feat_cfg->prtyen = 0; + plat->safety_feat_cfg->tmouten = 0; + + return intel_mgbe_common_data(pdev, plat); +} + +static int ehl_sgmii_data(struct pci_dev *pdev, + struct plat_stmmacenet_data *plat) +{ + plat->bus_id = 1; + plat->phy_interface = PHY_INTERFACE_MODE_SGMII; + plat->speed_mode_2500 = intel_speed_mode_2500; + plat->serdes_powerup = intel_serdes_powerup; + plat->serdes_powerdown = intel_serdes_powerdown; + + plat->clk_ptp_rate = 204800000; + + return ehl_common_data(pdev, plat); +} + +static struct stmmac_pci_info ehl_sgmii1g_info = { + .setup = ehl_sgmii_data, +}; + +static int ehl_rgmii_data(struct pci_dev *pdev, + struct plat_stmmacenet_data *plat) +{ + plat->bus_id = 1; + plat->phy_interface = PHY_INTERFACE_MODE_RGMII; + + plat->clk_ptp_rate = 204800000; + + return ehl_common_data(pdev, plat); +} + +static struct stmmac_pci_info ehl_rgmii1g_info = { + .setup = ehl_rgmii_data, +}; + +static int ehl_pse0_common_data(struct pci_dev *pdev, + struct plat_stmmacenet_data *plat) +{ + struct intel_priv_data *intel_priv = plat->bsp_priv; + + intel_priv->is_pse = true; + plat->bus_id = 2; + plat->addr64 = 32; + + plat->clk_ptp_rate = 200000000; + + intel_mgbe_pse_crossts_adj(intel_priv, EHL_PSE_ART_MHZ); + + return ehl_common_data(pdev, plat); +} + +static int ehl_pse0_rgmii1g_data(struct pci_dev *pdev, + struct plat_stmmacenet_data *plat) +{ + plat->phy_interface = PHY_INTERFACE_MODE_RGMII_ID; + return ehl_pse0_common_data(pdev, plat); +} + +static struct stmmac_pci_info ehl_pse0_rgmii1g_info = { + .setup = ehl_pse0_rgmii1g_data, +}; + +static int ehl_pse0_sgmii1g_data(struct pci_dev *pdev, + struct plat_stmmacenet_data *plat) +{ + plat->phy_interface = PHY_INTERFACE_MODE_SGMII; + plat->speed_mode_2500 = intel_speed_mode_2500; + plat->serdes_powerup = intel_serdes_powerup; + plat->serdes_powerdown = intel_serdes_powerdown; + return ehl_pse0_common_data(pdev, plat); +} + +static struct stmmac_pci_info ehl_pse0_sgmii1g_info = { + .setup = ehl_pse0_sgmii1g_data, +}; + +static int ehl_pse1_common_data(struct pci_dev *pdev, + struct plat_stmmacenet_data *plat) +{ + struct intel_priv_data *intel_priv = plat->bsp_priv; + + intel_priv->is_pse = true; + plat->bus_id = 3; + plat->addr64 = 32; + + plat->clk_ptp_rate = 200000000; + + intel_mgbe_pse_crossts_adj(intel_priv, EHL_PSE_ART_MHZ); + + return ehl_common_data(pdev, plat); +} + +static int ehl_pse1_rgmii1g_data(struct pci_dev *pdev, + struct plat_stmmacenet_data *plat) +{ + plat->phy_interface = PHY_INTERFACE_MODE_RGMII_ID; + return ehl_pse1_common_data(pdev, plat); +} + +static struct stmmac_pci_info ehl_pse1_rgmii1g_info = { + .setup = ehl_pse1_rgmii1g_data, +}; + +static int ehl_pse1_sgmii1g_data(struct pci_dev *pdev, + struct plat_stmmacenet_data *plat) +{ + plat->phy_interface = PHY_INTERFACE_MODE_SGMII; + plat->speed_mode_2500 = intel_speed_mode_2500; + plat->serdes_powerup = intel_serdes_powerup; + plat->serdes_powerdown = intel_serdes_powerdown; + return ehl_pse1_common_data(pdev, plat); +} + +static struct stmmac_pci_info ehl_pse1_sgmii1g_info = { + .setup = ehl_pse1_sgmii1g_data, +}; + +static int tgl_common_data(struct pci_dev *pdev, + struct plat_stmmacenet_data *plat) +{ + plat->rx_queues_to_use = 6; + plat->tx_queues_to_use = 4; + plat->clk_ptp_rate = 204800000; + plat->speed_mode_2500 = intel_speed_mode_2500; + + plat->safety_feat_cfg->tsoee = 1; + plat->safety_feat_cfg->mrxpee = 0; + plat->safety_feat_cfg->mestee = 1; + plat->safety_feat_cfg->mrxee = 1; + plat->safety_feat_cfg->mtxee = 1; + plat->safety_feat_cfg->epsi = 0; + plat->safety_feat_cfg->edpp = 0; + plat->safety_feat_cfg->prtyen = 0; + plat->safety_feat_cfg->tmouten = 0; + + return intel_mgbe_common_data(pdev, plat); +} + +static int tgl_sgmii_phy0_data(struct pci_dev *pdev, + struct plat_stmmacenet_data *plat) +{ + plat->bus_id = 1; + plat->phy_interface = PHY_INTERFACE_MODE_SGMII; + plat->serdes_powerup = intel_serdes_powerup; + plat->serdes_powerdown = intel_serdes_powerdown; + return tgl_common_data(pdev, plat); +} + +static struct stmmac_pci_info tgl_sgmii1g_phy0_info = { + .setup = tgl_sgmii_phy0_data, +}; + +static int tgl_sgmii_phy1_data(struct pci_dev *pdev, + struct plat_stmmacenet_data *plat) +{ + plat->bus_id = 2; + plat->phy_interface = PHY_INTERFACE_MODE_SGMII; + plat->serdes_powerup = intel_serdes_powerup; + plat->serdes_powerdown = intel_serdes_powerdown; + return tgl_common_data(pdev, plat); +} + +static struct stmmac_pci_info tgl_sgmii1g_phy1_info = { + .setup = tgl_sgmii_phy1_data, +}; + +static int adls_sgmii_phy0_data(struct pci_dev *pdev, + struct plat_stmmacenet_data *plat) +{ + plat->bus_id = 1; + plat->phy_interface = PHY_INTERFACE_MODE_SGMII; + + /* SerDes power up and power down are done in BIOS for ADL */ + + return tgl_common_data(pdev, plat); +} + +static struct stmmac_pci_info adls_sgmii1g_phy0_info = { + .setup = adls_sgmii_phy0_data, +}; + +static int adls_sgmii_phy1_data(struct pci_dev *pdev, + struct plat_stmmacenet_data *plat) +{ + plat->bus_id = 2; + plat->phy_interface = PHY_INTERFACE_MODE_SGMII; + + /* SerDes power up and power down are done in BIOS for ADL */ + + return tgl_common_data(pdev, plat); +} + +static struct stmmac_pci_info adls_sgmii1g_phy1_info = { + .setup = adls_sgmii_phy1_data, +}; +static const struct stmmac_pci_func_data galileo_stmmac_func_data[] = { + { + .func = 6, + .phy_addr = 1, + }, +}; + +static const struct stmmac_pci_dmi_data galileo_stmmac_dmi_data = { + .func = galileo_stmmac_func_data, + .nfuncs = ARRAY_SIZE(galileo_stmmac_func_data), +}; + +static const struct stmmac_pci_func_data iot2040_stmmac_func_data[] = { + { + .func = 6, + .phy_addr = 1, + }, + { + .func = 7, + .phy_addr = 1, + }, +}; + +static const struct stmmac_pci_dmi_data iot2040_stmmac_dmi_data = { + .func = iot2040_stmmac_func_data, + .nfuncs = ARRAY_SIZE(iot2040_stmmac_func_data), +}; + +static const struct dmi_system_id quark_pci_dmi[] = { + { + .matches = { + DMI_EXACT_MATCH(DMI_BOARD_NAME, "Galileo"), + }, + .driver_data = (void *)&galileo_stmmac_dmi_data, + }, + { + .matches = { + DMI_EXACT_MATCH(DMI_BOARD_NAME, "GalileoGen2"), + }, + .driver_data = (void *)&galileo_stmmac_dmi_data, + }, + /* There are 2 types of SIMATIC IOT2000: IOT2020 and IOT2040. + * The asset tag "6ES7647-0AA00-0YA2" is only for IOT2020 which + * has only one pci network device while other asset tags are + * for IOT2040 which has two. + */ + { + .matches = { + DMI_EXACT_MATCH(DMI_BOARD_NAME, "SIMATIC IOT2000"), + DMI_EXACT_MATCH(DMI_BOARD_ASSET_TAG, + "6ES7647-0AA00-0YA2"), + }, + .driver_data = (void *)&galileo_stmmac_dmi_data, + }, + { + .matches = { + DMI_EXACT_MATCH(DMI_BOARD_NAME, "SIMATIC IOT2000"), + }, + .driver_data = (void *)&iot2040_stmmac_dmi_data, + }, + {} +}; + +static int quark_default_data(struct pci_dev *pdev, + struct plat_stmmacenet_data *plat) +{ + int ret; + + /* Set common default data first */ + common_default_data(plat); + + /* Refuse to load the driver and register net device if MAC controller + * does not connect to any PHY interface. + */ + ret = stmmac_pci_find_phy_addr(pdev, quark_pci_dmi); + if (ret < 0) { + /* Return error to the caller on DMI enabled boards. */ + if (dmi_get_system_info(DMI_BOARD_NAME)) + return ret; + + /* Galileo boards with old firmware don't support DMI. We always + * use 1 here as PHY address, so at least the first found MAC + * controller would be probed. + */ + ret = 1; + } + + plat->bus_id = pci_dev_id(pdev); + plat->phy_addr = ret; + plat->phy_interface = PHY_INTERFACE_MODE_RMII; + + plat->dma_cfg->pbl = 16; + plat->dma_cfg->pblx8 = true; + plat->dma_cfg->fixed_burst = 1; + /* AXI (TODO) */ + + return 0; +} + +static const struct stmmac_pci_info quark_info = { + .setup = quark_default_data, +}; + +static int stmmac_config_single_msi(struct pci_dev *pdev, + struct plat_stmmacenet_data *plat, + struct stmmac_resources *res) +{ + int ret; + + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES); + if (ret < 0) { + dev_info(&pdev->dev, "%s: Single IRQ enablement failed\n", + __func__); + return ret; + } + + res->irq = pci_irq_vector(pdev, 0); + res->wol_irq = res->irq; + plat->multi_msi_en = 0; + dev_info(&pdev->dev, "%s: Single IRQ enablement successful\n", + __func__); + + return 0; +} + +static int stmmac_config_multi_msi(struct pci_dev *pdev, + struct plat_stmmacenet_data *plat, + struct stmmac_resources *res) +{ + int ret; + int i; + + if (plat->msi_rx_base_vec >= STMMAC_MSI_VEC_MAX || + plat->msi_tx_base_vec >= STMMAC_MSI_VEC_MAX) { + dev_info(&pdev->dev, "%s: Invalid RX & TX vector defined\n", + __func__); + return -1; + } + + ret = pci_alloc_irq_vectors(pdev, 2, STMMAC_MSI_VEC_MAX, + PCI_IRQ_MSI | PCI_IRQ_MSIX); + if (ret < 0) { + dev_info(&pdev->dev, "%s: multi MSI enablement failed\n", + __func__); + return ret; + } + + /* For RX MSI */ + for (i = 0; i < plat->rx_queues_to_use; i++) { + res->rx_irq[i] = pci_irq_vector(pdev, + plat->msi_rx_base_vec + i * 2); + } + + /* For TX MSI */ + for (i = 0; i < plat->tx_queues_to_use; i++) { + res->tx_irq[i] = pci_irq_vector(pdev, + plat->msi_tx_base_vec + i * 2); + } + + if (plat->msi_mac_vec < STMMAC_MSI_VEC_MAX) + res->irq = pci_irq_vector(pdev, plat->msi_mac_vec); + if (plat->msi_wol_vec < STMMAC_MSI_VEC_MAX) + res->wol_irq = pci_irq_vector(pdev, plat->msi_wol_vec); + if (plat->msi_lpi_vec < STMMAC_MSI_VEC_MAX) + res->lpi_irq = pci_irq_vector(pdev, plat->msi_lpi_vec); + if (plat->msi_sfty_ce_vec < STMMAC_MSI_VEC_MAX) + res->sfty_ce_irq = pci_irq_vector(pdev, plat->msi_sfty_ce_vec); + if (plat->msi_sfty_ue_vec < STMMAC_MSI_VEC_MAX) + res->sfty_ue_irq = pci_irq_vector(pdev, plat->msi_sfty_ue_vec); + + plat->multi_msi_en = 1; + dev_info(&pdev->dev, "%s: multi MSI enablement successful\n", __func__); + + return 0; +} + +/** + * intel_eth_pci_probe + * + * @pdev: pci device pointer + * @id: pointer to table of device id/id's. + * + * Description: This probing function gets called for all PCI devices which + * match the ID table and are not "owned" by other driver yet. This function + * gets passed a "struct pci_dev *" for each device whose entry in the ID table + * matches the device. The probe functions returns zero when the driver choose + * to take "ownership" of the device or an error code(-ve no) otherwise. + */ +static int intel_eth_pci_probe(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + struct stmmac_pci_info *info = (struct stmmac_pci_info *)id->driver_data; + struct intel_priv_data *intel_priv; + struct plat_stmmacenet_data *plat; + struct stmmac_resources res; + int ret; + + intel_priv = devm_kzalloc(&pdev->dev, sizeof(*intel_priv), GFP_KERNEL); + if (!intel_priv) + return -ENOMEM; + + plat = devm_kzalloc(&pdev->dev, sizeof(*plat), GFP_KERNEL); + if (!plat) + return -ENOMEM; + + plat->mdio_bus_data = devm_kzalloc(&pdev->dev, + sizeof(*plat->mdio_bus_data), + GFP_KERNEL); + if (!plat->mdio_bus_data) + return -ENOMEM; + + plat->dma_cfg = devm_kzalloc(&pdev->dev, sizeof(*plat->dma_cfg), + GFP_KERNEL); + if (!plat->dma_cfg) + return -ENOMEM; + + plat->safety_feat_cfg = devm_kzalloc(&pdev->dev, + sizeof(*plat->safety_feat_cfg), + GFP_KERNEL); + if (!plat->safety_feat_cfg) + return -ENOMEM; + + /* Enable pci device */ + ret = pcim_enable_device(pdev); + if (ret) { + dev_err(&pdev->dev, "%s: ERROR: failed to enable device\n", + __func__); + return ret; + } + + ret = pcim_iomap_regions(pdev, BIT(0), pci_name(pdev)); + if (ret) + return ret; + + pci_set_master(pdev); + + plat->bsp_priv = intel_priv; + intel_priv->mdio_adhoc_addr = INTEL_MGBE_ADHOC_ADDR; + intel_priv->crossts_adj = 1; + + /* Initialize all MSI vectors to invalid so that it can be set + * according to platform data settings below. + * Note: MSI vector takes value from 0 upto 31 (STMMAC_MSI_VEC_MAX) + */ + plat->msi_mac_vec = STMMAC_MSI_VEC_MAX; + plat->msi_wol_vec = STMMAC_MSI_VEC_MAX; + plat->msi_lpi_vec = STMMAC_MSI_VEC_MAX; + plat->msi_sfty_ce_vec = STMMAC_MSI_VEC_MAX; + plat->msi_sfty_ue_vec = STMMAC_MSI_VEC_MAX; + plat->msi_rx_base_vec = STMMAC_MSI_VEC_MAX; + plat->msi_tx_base_vec = STMMAC_MSI_VEC_MAX; + + ret = info->setup(pdev, plat); + if (ret) + return ret; + + memset(&res, 0, sizeof(res)); + res.addr = pcim_iomap_table(pdev)[0]; + + if (plat->eee_usecs_rate > 0) { + u32 tx_lpi_usec; + + tx_lpi_usec = (plat->eee_usecs_rate / 1000000) - 1; + writel(tx_lpi_usec, res.addr + GMAC_1US_TIC_COUNTER); + } + + ret = stmmac_config_multi_msi(pdev, plat, &res); + if (ret) { + ret = stmmac_config_single_msi(pdev, plat, &res); + if (ret) { + dev_err(&pdev->dev, "%s: ERROR: failed to enable IRQ\n", + __func__); + goto err_alloc_irq; + } + } + + ret = stmmac_dvr_probe(&pdev->dev, plat, &res); + if (ret) { + goto err_alloc_irq; + } + + return 0; + +err_alloc_irq: + clk_disable_unprepare(plat->stmmac_clk); + clk_unregister_fixed_rate(plat->stmmac_clk); + return ret; +} + +/** + * intel_eth_pci_remove + * + * @pdev: pci device pointer + * Description: this function calls the main to free the net resources + * and releases the PCI resources. + */ +static void intel_eth_pci_remove(struct pci_dev *pdev) +{ + struct net_device *ndev = dev_get_drvdata(&pdev->dev); + struct stmmac_priv *priv = netdev_priv(ndev); + + stmmac_dvr_remove(&pdev->dev); + + clk_disable_unprepare(priv->plat->stmmac_clk); + clk_unregister_fixed_rate(priv->plat->stmmac_clk); +} + +static int __maybe_unused intel_eth_pci_suspend(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + int ret; + + ret = stmmac_suspend(dev); + if (ret) + return ret; + + ret = pci_save_state(pdev); + if (ret) + return ret; + + pci_wake_from_d3(pdev, true); + pci_set_power_state(pdev, PCI_D3hot); + return 0; +} + +static int __maybe_unused intel_eth_pci_resume(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + int ret; + + pci_restore_state(pdev); + pci_set_power_state(pdev, PCI_D0); + + ret = pcim_enable_device(pdev); + if (ret) + return ret; + + pci_set_master(pdev); + + return stmmac_resume(dev); +} + +static SIMPLE_DEV_PM_OPS(intel_eth_pm_ops, intel_eth_pci_suspend, + intel_eth_pci_resume); + +#define PCI_DEVICE_ID_INTEL_QUARK 0x0937 +#define PCI_DEVICE_ID_INTEL_EHL_RGMII1G 0x4b30 +#define PCI_DEVICE_ID_INTEL_EHL_SGMII1G 0x4b31 +#define PCI_DEVICE_ID_INTEL_EHL_SGMII2G5 0x4b32 +/* Intel(R) Programmable Services Engine (Intel(R) PSE) consist of 2 MAC + * which are named PSE0 and PSE1 + */ +#define PCI_DEVICE_ID_INTEL_EHL_PSE0_RGMII1G 0x4ba0 +#define PCI_DEVICE_ID_INTEL_EHL_PSE0_SGMII1G 0x4ba1 +#define PCI_DEVICE_ID_INTEL_EHL_PSE0_SGMII2G5 0x4ba2 +#define PCI_DEVICE_ID_INTEL_EHL_PSE1_RGMII1G 0x4bb0 +#define PCI_DEVICE_ID_INTEL_EHL_PSE1_SGMII1G 0x4bb1 +#define PCI_DEVICE_ID_INTEL_EHL_PSE1_SGMII2G5 0x4bb2 +#define PCI_DEVICE_ID_INTEL_TGLH_SGMII1G_0 0x43ac +#define PCI_DEVICE_ID_INTEL_TGLH_SGMII1G_1 0x43a2 +#define PCI_DEVICE_ID_INTEL_TGL_SGMII1G 0xa0ac +#define PCI_DEVICE_ID_INTEL_ADLS_SGMII1G_0 0x7aac +#define PCI_DEVICE_ID_INTEL_ADLS_SGMII1G_1 0x7aad +#define PCI_DEVICE_ID_INTEL_ADLN_SGMII1G 0x54ac +#define PCI_DEVICE_ID_INTEL_RPLP_SGMII1G 0x51ac + +static const struct pci_device_id intel_eth_pci_id_table[] = { + { PCI_DEVICE_DATA(INTEL, QUARK, &quark_info) }, + { PCI_DEVICE_DATA(INTEL, EHL_RGMII1G, &ehl_rgmii1g_info) }, + { PCI_DEVICE_DATA(INTEL, EHL_SGMII1G, &ehl_sgmii1g_info) }, + { PCI_DEVICE_DATA(INTEL, EHL_SGMII2G5, &ehl_sgmii1g_info) }, + { PCI_DEVICE_DATA(INTEL, EHL_PSE0_RGMII1G, &ehl_pse0_rgmii1g_info) }, + { PCI_DEVICE_DATA(INTEL, EHL_PSE0_SGMII1G, &ehl_pse0_sgmii1g_info) }, + { PCI_DEVICE_DATA(INTEL, EHL_PSE0_SGMII2G5, &ehl_pse0_sgmii1g_info) }, + { PCI_DEVICE_DATA(INTEL, EHL_PSE1_RGMII1G, &ehl_pse1_rgmii1g_info) }, + { PCI_DEVICE_DATA(INTEL, EHL_PSE1_SGMII1G, &ehl_pse1_sgmii1g_info) }, + { PCI_DEVICE_DATA(INTEL, EHL_PSE1_SGMII2G5, &ehl_pse1_sgmii1g_info) }, + { PCI_DEVICE_DATA(INTEL, TGL_SGMII1G, &tgl_sgmii1g_phy0_info) }, + { PCI_DEVICE_DATA(INTEL, TGLH_SGMII1G_0, &tgl_sgmii1g_phy0_info) }, + { PCI_DEVICE_DATA(INTEL, TGLH_SGMII1G_1, &tgl_sgmii1g_phy1_info) }, + { PCI_DEVICE_DATA(INTEL, ADLS_SGMII1G_0, &adls_sgmii1g_phy0_info) }, + { PCI_DEVICE_DATA(INTEL, ADLS_SGMII1G_1, &adls_sgmii1g_phy1_info) }, + { PCI_DEVICE_DATA(INTEL, ADLN_SGMII1G, &tgl_sgmii1g_phy0_info) }, + { PCI_DEVICE_DATA(INTEL, RPLP_SGMII1G, &tgl_sgmii1g_phy0_info) }, + {} +}; +MODULE_DEVICE_TABLE(pci, intel_eth_pci_id_table); + +static struct pci_driver intel_eth_pci_driver = { + .name = "intel-eth-pci", + .id_table = intel_eth_pci_id_table, + .probe = intel_eth_pci_probe, + .remove = intel_eth_pci_remove, + .driver = { + .pm = &intel_eth_pm_ops, + }, +}; + +module_pci_driver(intel_eth_pci_driver); + +MODULE_DESCRIPTION("INTEL 10/100/1000 Ethernet PCI driver"); +MODULE_AUTHOR("Voon Weifeng <weifeng.voon@intel.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.h b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.h new file mode 100644 index 000000000000..0a37987478c1 --- /dev/null +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2020, Intel Corporation + * DWMAC Intel header file + */ + +#ifndef __DWMAC_INTEL_H__ +#define __DWMAC_INTEL_H__ + +#define POLL_DELAY_US 8 + +/* SERDES Register */ +#define SERDES_GCR 0x0 /* Global Conguration */ +#define SERDES_GSR0 0x5 /* Global Status Reg0 */ +#define SERDES_GCR0 0xb /* Global Configuration Reg0 */ + +/* SERDES defines */ +#define SERDES_PLL_CLK BIT(0) /* PLL clk valid signal */ +#define SERDES_PHY_RX_CLK BIT(1) /* PSE SGMII PHY rx clk */ +#define SERDES_RST BIT(2) /* Serdes Reset */ +#define SERDES_PWR_ST_MASK GENMASK(6, 4) /* Serdes Power state*/ +#define SERDES_RATE_MASK GENMASK(9, 8) +#define SERDES_PCLK_MASK GENMASK(14, 12) /* PCLK rate to PHY */ +#define SERDES_LINK_MODE_MASK GENMASK(2, 1) +#define SERDES_LINK_MODE_SHIFT 1 +#define SERDES_PWR_ST_SHIFT 4 +#define SERDES_PWR_ST_P0 0x0 +#define SERDES_PWR_ST_P3 0x3 +#define SERDES_LINK_MODE_2G5 0x3 +#define SERSED_LINK_MODE_1G 0x2 +#define SERDES_PCLK_37p5MHZ 0x0 +#define SERDES_PCLK_70MHZ 0x1 +#define SERDES_RATE_PCIE_GEN1 0x0 +#define SERDES_RATE_PCIE_GEN2 0x1 +#define SERDES_RATE_PCIE_SHIFT 8 +#define SERDES_PCLK_SHIFT 12 + +#define INTEL_MGBE_ADHOC_ADDR 0x15 +#define INTEL_MGBE_XPCS_ADDR 0x16 + +/* Cross-timestamping defines */ +#define ART_CPUID_LEAF 0x15 +#define EHL_PSE_ART_MHZ 19200000 + +/* Selection for PTP Clock Freq belongs to PSE & PCH GbE */ +#define PSE_PTP_CLK_FREQ_MASK (GMAC_GPO0 | GMAC_GPO3) +#define PSE_PTP_CLK_FREQ_19_2MHZ (GMAC_GPO0) +#define PSE_PTP_CLK_FREQ_200MHZ (GMAC_GPO0 | GMAC_GPO3) +#define PSE_PTP_CLK_FREQ_256MHZ (0) +#define PCH_PTP_CLK_FREQ_MASK (GMAC_GPO0) +#define PCH_PTP_CLK_FREQ_19_2MHZ (GMAC_GPO0) +#define PCH_PTP_CLK_FREQ_200MHZ (0) + +#endif /* __DWMAC_INTEL_H__ */ diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-ipq806x.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-ipq806x.c index 6ae13dc19510..e888c8a9c830 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-ipq806x.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-ipq806x.c @@ -27,6 +27,8 @@ #include <linux/stmmac.h> #include <linux/of_mdio.h> #include <linux/module.h> +#include <linux/sys_soc.h> +#include <linux/bitfield.h> #include "stmmac_platform.h" @@ -64,6 +66,17 @@ #define NSS_COMMON_CLK_DIV_SGMII_100 4 #define NSS_COMMON_CLK_DIV_SGMII_10 49 +#define QSGMII_PCS_ALL_CH_CTL 0x80 +#define QSGMII_PCS_CH_SPEED_FORCE BIT(1) +#define QSGMII_PCS_CH_SPEED_10 0x0 +#define QSGMII_PCS_CH_SPEED_100 BIT(2) +#define QSGMII_PCS_CH_SPEED_1000 BIT(3) +#define QSGMII_PCS_CH_SPEED_MASK (QSGMII_PCS_CH_SPEED_FORCE | \ + QSGMII_PCS_CH_SPEED_10 | \ + QSGMII_PCS_CH_SPEED_100 | \ + QSGMII_PCS_CH_SPEED_1000) +#define QSGMII_PCS_CH_SPEED_SHIFT(x) ((x) * 4) + #define QSGMII_PCS_CAL_LCKDT_CTL 0x120 #define QSGMII_PCS_CAL_LCKDT_CTL_RST BIT(19) @@ -75,11 +88,20 @@ #define QSGMII_PHY_RX_SIGNAL_DETECT_EN BIT(2) #define QSGMII_PHY_TX_DRIVER_EN BIT(3) #define QSGMII_PHY_QSGMII_EN BIT(7) -#define QSGMII_PHY_PHASE_LOOP_GAIN_OFFSET 12 -#define QSGMII_PHY_RX_DC_BIAS_OFFSET 18 -#define QSGMII_PHY_RX_INPUT_EQU_OFFSET 20 -#define QSGMII_PHY_CDR_PI_SLEW_OFFSET 22 -#define QSGMII_PHY_TX_DRV_AMP_OFFSET 28 +#define QSGMII_PHY_DEEMPHASIS_LVL_MASK GENMASK(11, 10) +#define QSGMII_PHY_DEEMPHASIS_LVL(x) FIELD_PREP(QSGMII_PHY_DEEMPHASIS_LVL_MASK, (x)) +#define QSGMII_PHY_PHASE_LOOP_GAIN_MASK GENMASK(14, 12) +#define QSGMII_PHY_PHASE_LOOP_GAIN(x) FIELD_PREP(QSGMII_PHY_PHASE_LOOP_GAIN_MASK, (x)) +#define QSGMII_PHY_RX_DC_BIAS_MASK GENMASK(19, 18) +#define QSGMII_PHY_RX_DC_BIAS(x) FIELD_PREP(QSGMII_PHY_RX_DC_BIAS_MASK, (x)) +#define QSGMII_PHY_RX_INPUT_EQU_MASK GENMASK(21, 20) +#define QSGMII_PHY_RX_INPUT_EQU(x) FIELD_PREP(QSGMII_PHY_RX_INPUT_EQU_MASK, (x)) +#define QSGMII_PHY_CDR_PI_SLEW_MASK GENMASK(23, 22) +#define QSGMII_PHY_CDR_PI_SLEW(x) FIELD_PREP(QSGMII_PHY_CDR_PI_SLEW_MASK, (x)) +#define QSGMII_PHY_TX_SLEW_MASK GENMASK(27, 26) +#define QSGMII_PHY_TX_SLEW(x) FIELD_PREP(QSGMII_PHY_TX_SLEW_MASK, (x)) +#define QSGMII_PHY_TX_DRV_AMP_MASK GENMASK(31, 28) +#define QSGMII_PHY_TX_DRV_AMP(x) FIELD_PREP(QSGMII_PHY_TX_DRV_AMP_MASK, (x)) struct ipq806x_gmac { struct platform_device *pdev; @@ -242,6 +264,113 @@ static void ipq806x_gmac_fix_mac_speed(void *priv, unsigned int speed) ipq806x_gmac_set_speed(gmac, speed); } +static int +ipq806x_gmac_configure_qsgmii_pcs_speed(struct ipq806x_gmac *gmac) +{ + struct platform_device *pdev = gmac->pdev; + struct device *dev = &pdev->dev; + struct device_node *dn; + int link_speed; + int val = 0; + int ret; + + /* Some bootloader may apply wrong configuration and cause + * not functioning port. If fixed link is not set, + * reset the force speed bit. + */ + if (!of_phy_is_fixed_link(pdev->dev.of_node)) + goto write; + + dn = of_get_child_by_name(pdev->dev.of_node, "fixed-link"); + ret = of_property_read_u32(dn, "speed", &link_speed); + of_node_put(dn); + if (ret) { + dev_err(dev, "found fixed-link node with no speed"); + return ret; + } + + val = QSGMII_PCS_CH_SPEED_FORCE; + + switch (link_speed) { + case SPEED_1000: + val |= QSGMII_PCS_CH_SPEED_1000; + break; + case SPEED_100: + val |= QSGMII_PCS_CH_SPEED_100; + break; + case SPEED_10: + val |= QSGMII_PCS_CH_SPEED_10; + break; + } + +write: + regmap_update_bits(gmac->qsgmii_csr, QSGMII_PCS_ALL_CH_CTL, + QSGMII_PCS_CH_SPEED_MASK << + QSGMII_PCS_CH_SPEED_SHIFT(gmac->id), + val << + QSGMII_PCS_CH_SPEED_SHIFT(gmac->id)); + + return 0; +} + +static const struct soc_device_attribute ipq806x_gmac_soc_v1[] = { + { + .revision = "1.*", + }, + { + /* sentinel */ + } +}; + +static int +ipq806x_gmac_configure_qsgmii_params(struct ipq806x_gmac *gmac) +{ + struct platform_device *pdev = gmac->pdev; + const struct soc_device_attribute *soc; + struct device *dev = &pdev->dev; + u32 qsgmii_param; + + switch (gmac->id) { + case 1: + soc = soc_device_match(ipq806x_gmac_soc_v1); + + if (soc) + qsgmii_param = QSGMII_PHY_TX_DRV_AMP(0xc) | + QSGMII_PHY_TX_SLEW(0x2) | + QSGMII_PHY_DEEMPHASIS_LVL(0x2); + else + qsgmii_param = QSGMII_PHY_TX_DRV_AMP(0xd) | + QSGMII_PHY_TX_SLEW(0x0) | + QSGMII_PHY_DEEMPHASIS_LVL(0x0); + + qsgmii_param |= QSGMII_PHY_RX_DC_BIAS(0x2); + break; + case 2: + case 3: + qsgmii_param = QSGMII_PHY_RX_DC_BIAS(0x3) | + QSGMII_PHY_TX_DRV_AMP(0xc); + break; + default: /* gmac 0 can't be set in SGMII mode */ + dev_err(dev, "gmac id %d can't be in SGMII mode", gmac->id); + return -EINVAL; + } + + /* Common params across all gmac id */ + qsgmii_param |= QSGMII_PHY_CDR_EN | + QSGMII_PHY_RX_FRONT_EN | + QSGMII_PHY_RX_SIGNAL_DETECT_EN | + QSGMII_PHY_TX_DRIVER_EN | + QSGMII_PHY_QSGMII_EN | + QSGMII_PHY_PHASE_LOOP_GAIN(0x4) | + QSGMII_PHY_RX_INPUT_EQU(0x1) | + QSGMII_PHY_CDR_PI_SLEW(0x2); + + regmap_write(gmac->qsgmii_csr, QSGMII_PHY_SGMII_CTL(gmac->id), + qsgmii_param); + + return 0; +} + static int ipq806x_gmac_probe(struct platform_device *pdev) { struct plat_stmmacenet_data *plat_dat; @@ -255,7 +384,7 @@ static int ipq806x_gmac_probe(struct platform_device *pdev) if (val) return val; - plat_dat = stmmac_probe_config_dt(pdev, &stmmac_res.mac); + plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac); if (IS_ERR(plat_dat)) return PTR_ERR(plat_dat); @@ -289,10 +418,7 @@ static int ipq806x_gmac_probe(struct platform_device *pdev) val &= ~NSS_COMMON_GMAC_CTL_PHY_IFACE_SEL; break; default: - dev_err(&pdev->dev, "Unsupported PHY mode: \"%s\"\n", - phy_modes(gmac->phy_mode)); - err = -EINVAL; - goto err_remove_config_dt; + goto err_unsupported_phy; } regmap_write(gmac->nss_common, NSS_COMMON_GMAC_CTL(gmac->id), val); @@ -309,35 +435,43 @@ static int ipq806x_gmac_probe(struct platform_device *pdev) NSS_COMMON_CLK_SRC_CTRL_OFFSET(gmac->id); break; default: - dev_err(&pdev->dev, "Unsupported PHY mode: \"%s\"\n", - phy_modes(gmac->phy_mode)); - err = -EINVAL; - goto err_remove_config_dt; + goto err_unsupported_phy; } regmap_write(gmac->nss_common, NSS_COMMON_CLK_SRC_CTRL, val); /* Enable PTP clock */ regmap_read(gmac->nss_common, NSS_COMMON_CLK_GATE, &val); val |= NSS_COMMON_CLK_GATE_PTP_EN(gmac->id); + switch (gmac->phy_mode) { + case PHY_INTERFACE_MODE_RGMII: + val |= NSS_COMMON_CLK_GATE_RGMII_RX_EN(gmac->id) | + NSS_COMMON_CLK_GATE_RGMII_TX_EN(gmac->id); + break; + case PHY_INTERFACE_MODE_SGMII: + val |= NSS_COMMON_CLK_GATE_GMII_RX_EN(gmac->id) | + NSS_COMMON_CLK_GATE_GMII_TX_EN(gmac->id); + break; + default: + goto err_unsupported_phy; + } regmap_write(gmac->nss_common, NSS_COMMON_CLK_GATE, val); if (gmac->phy_mode == PHY_INTERFACE_MODE_SGMII) { - regmap_write(gmac->qsgmii_csr, QSGMII_PHY_SGMII_CTL(gmac->id), - QSGMII_PHY_CDR_EN | - QSGMII_PHY_RX_FRONT_EN | - QSGMII_PHY_RX_SIGNAL_DETECT_EN | - QSGMII_PHY_TX_DRIVER_EN | - QSGMII_PHY_QSGMII_EN | - 0x4ul << QSGMII_PHY_PHASE_LOOP_GAIN_OFFSET | - 0x3ul << QSGMII_PHY_RX_DC_BIAS_OFFSET | - 0x1ul << QSGMII_PHY_RX_INPUT_EQU_OFFSET | - 0x2ul << QSGMII_PHY_CDR_PI_SLEW_OFFSET | - 0xCul << QSGMII_PHY_TX_DRV_AMP_OFFSET); + err = ipq806x_gmac_configure_qsgmii_params(gmac); + if (err) + goto err_remove_config_dt; + + err = ipq806x_gmac_configure_qsgmii_pcs_speed(gmac); + if (err) + goto err_remove_config_dt; } plat_dat->has_gmac = true; plat_dat->bsp_priv = gmac; plat_dat->fix_mac_speed = ipq806x_gmac_fix_mac_speed; + plat_dat->multicast_filter_bins = 0; + plat_dat->tx_fifo_size = 8192; + plat_dat->rx_fifo_size = 8192; err = stmmac_dvr_probe(&pdev->dev, plat_dat, &stmmac_res); if (err) @@ -345,6 +479,11 @@ static int ipq806x_gmac_probe(struct platform_device *pdev) return 0; +err_unsupported_phy: + dev_err(&pdev->dev, "Unsupported PHY mode: \"%s\"\n", + phy_modes(gmac->phy_mode)); + err = -EINVAL; + err_remove_config_dt: stmmac_remove_config_dt(pdev, plat_dat); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c new file mode 100644 index 000000000000..a25c187d3185 --- /dev/null +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2020, Loongson Corporation + */ + +#include <linux/clk-provider.h> +#include <linux/pci.h> +#include <linux/dmi.h> +#include <linux/device.h> +#include <linux/of_irq.h> +#include "stmmac.h" + +static int loongson_default_data(struct plat_stmmacenet_data *plat) +{ + plat->clk_csr = 2; /* clk_csr_i = 20-35MHz & MDC = clk_csr_i/16 */ + plat->has_gmac = 1; + plat->force_sf_dma_mode = 1; + + /* Set default value for multicast hash bins */ + plat->multicast_filter_bins = HASH_TABLE_SIZE; + + /* Set default value for unicast filter entries */ + plat->unicast_filter_entries = 1; + + /* Set the maxmtu to a default of JUMBO_LEN */ + plat->maxmtu = JUMBO_LEN; + + /* Set default number of RX and TX queues to use */ + plat->tx_queues_to_use = 1; + plat->rx_queues_to_use = 1; + + /* Disable Priority config by default */ + plat->tx_queues_cfg[0].use_prio = false; + plat->rx_queues_cfg[0].use_prio = false; + + /* Disable RX queues routing by default */ + plat->rx_queues_cfg[0].pkt_route = 0x0; + + /* Default to phy auto-detection */ + plat->phy_addr = -1; + + plat->dma_cfg->pbl = 32; + plat->dma_cfg->pblx8 = true; + + plat->multicast_filter_bins = 256; + return 0; +} + +static int loongson_dwmac_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct plat_stmmacenet_data *plat; + struct stmmac_resources res; + struct device_node *np; + int ret, i, phy_mode; + + np = dev_of_node(&pdev->dev); + + if (!np) { + pr_info("dwmac_loongson_pci: No OF node\n"); + return -ENODEV; + } + + if (!of_device_is_compatible(np, "loongson, pci-gmac")) { + pr_info("dwmac_loongson_pci: Incompatible OF node\n"); + return -ENODEV; + } + + plat = devm_kzalloc(&pdev->dev, sizeof(*plat), GFP_KERNEL); + if (!plat) + return -ENOMEM; + + plat->mdio_node = of_get_child_by_name(np, "mdio"); + if (plat->mdio_node) { + dev_info(&pdev->dev, "Found MDIO subnode\n"); + + plat->mdio_bus_data = devm_kzalloc(&pdev->dev, + sizeof(*plat->mdio_bus_data), + GFP_KERNEL); + if (!plat->mdio_bus_data) { + ret = -ENOMEM; + goto err_put_node; + } + plat->mdio_bus_data->needs_reset = true; + } + + plat->dma_cfg = devm_kzalloc(&pdev->dev, sizeof(*plat->dma_cfg), GFP_KERNEL); + if (!plat->dma_cfg) { + ret = -ENOMEM; + goto err_put_node; + } + + /* Enable pci device */ + ret = pci_enable_device(pdev); + if (ret) { + dev_err(&pdev->dev, "%s: ERROR: failed to enable device\n", __func__); + goto err_put_node; + } + + /* Get the base address of device */ + for (i = 0; i < PCI_STD_NUM_BARS; i++) { + if (pci_resource_len(pdev, i) == 0) + continue; + ret = pcim_iomap_regions(pdev, BIT(0), pci_name(pdev)); + if (ret) + goto err_disable_device; + break; + } + + plat->bus_id = of_alias_get_id(np, "ethernet"); + if (plat->bus_id < 0) + plat->bus_id = pci_dev_id(pdev); + + phy_mode = device_get_phy_mode(&pdev->dev); + if (phy_mode < 0) { + dev_err(&pdev->dev, "phy_mode not found\n"); + ret = phy_mode; + goto err_disable_device; + } + + plat->phy_interface = phy_mode; + plat->interface = PHY_INTERFACE_MODE_GMII; + + pci_set_master(pdev); + + loongson_default_data(plat); + pci_enable_msi(pdev); + memset(&res, 0, sizeof(res)); + res.addr = pcim_iomap_table(pdev)[0]; + + res.irq = of_irq_get_byname(np, "macirq"); + if (res.irq < 0) { + dev_err(&pdev->dev, "IRQ macirq not found\n"); + ret = -ENODEV; + goto err_disable_msi; + } + + res.wol_irq = of_irq_get_byname(np, "eth_wake_irq"); + if (res.wol_irq < 0) { + dev_info(&pdev->dev, "IRQ eth_wake_irq not found, using macirq\n"); + res.wol_irq = res.irq; + } + + res.lpi_irq = of_irq_get_byname(np, "eth_lpi"); + if (res.lpi_irq < 0) { + dev_err(&pdev->dev, "IRQ eth_lpi not found\n"); + ret = -ENODEV; + goto err_disable_msi; + } + + ret = stmmac_dvr_probe(&pdev->dev, plat, &res); + if (ret) + goto err_disable_msi; + + return ret; + +err_disable_msi: + pci_disable_msi(pdev); +err_disable_device: + pci_disable_device(pdev); +err_put_node: + of_node_put(plat->mdio_node); + return ret; +} + +static void loongson_dwmac_remove(struct pci_dev *pdev) +{ + struct net_device *ndev = dev_get_drvdata(&pdev->dev); + struct stmmac_priv *priv = netdev_priv(ndev); + int i; + + of_node_put(priv->plat->mdio_node); + stmmac_dvr_remove(&pdev->dev); + + for (i = 0; i < PCI_STD_NUM_BARS; i++) { + if (pci_resource_len(pdev, i) == 0) + continue; + pcim_iounmap_regions(pdev, BIT(i)); + break; + } + + pci_disable_msi(pdev); + pci_disable_device(pdev); +} + +static int __maybe_unused loongson_dwmac_suspend(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + int ret; + + ret = stmmac_suspend(dev); + if (ret) + return ret; + + ret = pci_save_state(pdev); + if (ret) + return ret; + + pci_disable_device(pdev); + pci_wake_from_d3(pdev, true); + return 0; +} + +static int __maybe_unused loongson_dwmac_resume(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + int ret; + + pci_restore_state(pdev); + pci_set_power_state(pdev, PCI_D0); + + ret = pci_enable_device(pdev); + if (ret) + return ret; + + pci_set_master(pdev); + + return stmmac_resume(dev); +} + +static SIMPLE_DEV_PM_OPS(loongson_dwmac_pm_ops, loongson_dwmac_suspend, + loongson_dwmac_resume); + +static const struct pci_device_id loongson_dwmac_id_table[] = { + { PCI_VDEVICE(LOONGSON, 0x7a03) }, + {} +}; +MODULE_DEVICE_TABLE(pci, loongson_dwmac_id_table); + +static struct pci_driver loongson_dwmac_driver = { + .name = "dwmac-loongson-pci", + .id_table = loongson_dwmac_id_table, + .probe = loongson_dwmac_probe, + .remove = loongson_dwmac_remove, + .driver = { + .pm = &loongson_dwmac_pm_ops, + }, +}; + +module_pci_driver(loongson_dwmac_driver); + +MODULE_DESCRIPTION("Loongson DWMAC PCI driver"); +MODULE_AUTHOR("Qing Zhang <zhangqing@loongson.cn>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-lpc18xx.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-lpc18xx.c index 3d3f43d91b98..9d77c647badd 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-lpc18xx.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-lpc18xx.c @@ -37,7 +37,7 @@ static int lpc18xx_dwmac_probe(struct platform_device *pdev) if (ret) return ret; - plat_dat = stmmac_probe_config_dt(pdev, &stmmac_res.mac); + plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac); if (IS_ERR(plat_dat)) return PTR_ERR(plat_dat); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c index 9e4b83832938..d42e1afb6521 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c @@ -9,7 +9,6 @@ #include <linux/of.h> #include <linux/of_device.h> #include <linux/of_net.h> -#include <linux/pm_runtime.h> #include <linux/regmap.h> #include <linux/stmmac.h> @@ -40,6 +39,33 @@ #define ETH_FINE_DLY_GTXC BIT(1) #define ETH_FINE_DLY_RXC BIT(0) +/* Peri Configuration register for mt8195 */ +#define MT8195_PERI_ETH_CTRL0 0xFD0 +#define MT8195_RMII_CLK_SRC_INTERNAL BIT(28) +#define MT8195_RMII_CLK_SRC_RXC BIT(27) +#define MT8195_ETH_INTF_SEL GENMASK(26, 24) +#define MT8195_RGMII_TXC_PHASE_CTRL BIT(22) +#define MT8195_EXT_PHY_MODE BIT(21) +#define MT8195_DLY_GTXC_INV BIT(12) +#define MT8195_DLY_GTXC_ENABLE BIT(5) +#define MT8195_DLY_GTXC_STAGES GENMASK(4, 0) + +#define MT8195_PERI_ETH_CTRL1 0xFD4 +#define MT8195_DLY_RXC_INV BIT(25) +#define MT8195_DLY_RXC_ENABLE BIT(18) +#define MT8195_DLY_RXC_STAGES GENMASK(17, 13) +#define MT8195_DLY_TXC_INV BIT(12) +#define MT8195_DLY_TXC_ENABLE BIT(5) +#define MT8195_DLY_TXC_STAGES GENMASK(4, 0) + +#define MT8195_PERI_ETH_CTRL2 0xFD8 +#define MT8195_DLY_RMII_RXC_INV BIT(25) +#define MT8195_DLY_RMII_RXC_ENABLE BIT(18) +#define MT8195_DLY_RMII_RXC_STAGES GENMASK(17, 13) +#define MT8195_DLY_RMII_TXC_INV BIT(12) +#define MT8195_DLY_RMII_TXC_ENABLE BIT(5) +#define MT8195_DLY_RMII_TXC_STAGES GENMASK(4, 0) + struct mac_delay_struct { u32 tx_delay; u32 rx_delay; @@ -50,19 +76,21 @@ struct mac_delay_struct { struct mediatek_dwmac_plat_data { const struct mediatek_dwmac_variant *variant; struct mac_delay_struct mac_delay; + struct clk *rmii_internal_clk; struct clk_bulk_data *clks; - struct device_node *np; struct regmap *peri_regmap; + struct device_node *np; struct device *dev; phy_interface_t phy_mode; - int num_clks_to_config; bool rmii_clk_from_mac; bool rmii_rxc; + bool mac_wol; }; struct mediatek_dwmac_variant { int (*dwmac_set_phy_interface)(struct mediatek_dwmac_plat_data *plat); int (*dwmac_set_delay)(struct mediatek_dwmac_plat_data *plat); + void (*dwmac_fix_mac_speed)(void *priv, unsigned int speed); /* clock ids to be requested */ const char * const *clk_list; @@ -75,7 +103,11 @@ struct mediatek_dwmac_variant { /* list of clocks required for mac */ static const char * const mt2712_dwmac_clk_l[] = { - "axi", "apb", "mac_main", "ptp_ref", "rmii_internal" + "axi", "apb", "mac_main", "ptp_ref" +}; + +static const char * const mt8195_dwmac_clk_l[] = { + "axi", "apb", "mac_cg", "mac_main", "ptp_ref" }; static int mt2712_set_interface(struct mediatek_dwmac_plat_data *plat) @@ -84,23 +116,12 @@ static int mt2712_set_interface(struct mediatek_dwmac_plat_data *plat) int rmii_rxc = plat->rmii_rxc ? RMII_CLK_SRC_RXC : 0; u32 intf_val = 0; - /* The clock labeled as "rmii_internal" in mt2712_dwmac_clk_l is needed - * only in RMII(when MAC provides the reference clock), and useless for - * RGMII/MII/RMII(when PHY provides the reference clock). - * num_clks_to_config indicates the real number of clocks should be - * configured, equals to (plat->variant->num_clks - 1) in default for all the case, - * then +1 for rmii_clk_from_mac case. - */ - plat->num_clks_to_config = plat->variant->num_clks - 1; - /* select phy interface in top control domain */ switch (plat->phy_mode) { case PHY_INTERFACE_MODE_MII: intf_val |= PHY_INTF_MII; break; case PHY_INTERFACE_MODE_RMII: - if (plat->rmii_clk_from_mac) - plat->num_clks_to_config++; intf_val |= (PHY_INTF_RMII | rmii_rxc | rmii_clk_from_mac); break; case PHY_INTERFACE_MODE_RGMII: @@ -268,6 +289,193 @@ static const struct mediatek_dwmac_variant mt2712_gmac_variant = { .tx_delay_max = 17600, }; +static int mt8195_set_interface(struct mediatek_dwmac_plat_data *plat) +{ + int rmii_clk_from_mac = plat->rmii_clk_from_mac ? MT8195_RMII_CLK_SRC_INTERNAL : 0; + int rmii_rxc = plat->rmii_rxc ? MT8195_RMII_CLK_SRC_RXC : 0; + u32 intf_val = 0; + + /* select phy interface in top control domain */ + switch (plat->phy_mode) { + case PHY_INTERFACE_MODE_MII: + intf_val |= FIELD_PREP(MT8195_ETH_INTF_SEL, PHY_INTF_MII); + break; + case PHY_INTERFACE_MODE_RMII: + intf_val |= (rmii_rxc | rmii_clk_from_mac); + intf_val |= FIELD_PREP(MT8195_ETH_INTF_SEL, PHY_INTF_RMII); + break; + case PHY_INTERFACE_MODE_RGMII: + case PHY_INTERFACE_MODE_RGMII_TXID: + case PHY_INTERFACE_MODE_RGMII_RXID: + case PHY_INTERFACE_MODE_RGMII_ID: + intf_val |= FIELD_PREP(MT8195_ETH_INTF_SEL, PHY_INTF_RGMII); + break; + default: + dev_err(plat->dev, "phy interface not supported\n"); + return -EINVAL; + } + + /* MT8195 only support external PHY */ + intf_val |= MT8195_EXT_PHY_MODE; + + regmap_write(plat->peri_regmap, MT8195_PERI_ETH_CTRL0, intf_val); + + return 0; +} + +static void mt8195_delay_ps2stage(struct mediatek_dwmac_plat_data *plat) +{ + struct mac_delay_struct *mac_delay = &plat->mac_delay; + + /* 290ps per stage */ + mac_delay->tx_delay /= 290; + mac_delay->rx_delay /= 290; +} + +static void mt8195_delay_stage2ps(struct mediatek_dwmac_plat_data *plat) +{ + struct mac_delay_struct *mac_delay = &plat->mac_delay; + + /* 290ps per stage */ + mac_delay->tx_delay *= 290; + mac_delay->rx_delay *= 290; +} + +static int mt8195_set_delay(struct mediatek_dwmac_plat_data *plat) +{ + struct mac_delay_struct *mac_delay = &plat->mac_delay; + u32 gtxc_delay_val = 0, delay_val = 0, rmii_delay_val = 0; + + mt8195_delay_ps2stage(plat); + + switch (plat->phy_mode) { + case PHY_INTERFACE_MODE_MII: + delay_val |= FIELD_PREP(MT8195_DLY_TXC_ENABLE, !!mac_delay->tx_delay); + delay_val |= FIELD_PREP(MT8195_DLY_TXC_STAGES, mac_delay->tx_delay); + delay_val |= FIELD_PREP(MT8195_DLY_TXC_INV, mac_delay->tx_inv); + + delay_val |= FIELD_PREP(MT8195_DLY_RXC_ENABLE, !!mac_delay->rx_delay); + delay_val |= FIELD_PREP(MT8195_DLY_RXC_STAGES, mac_delay->rx_delay); + delay_val |= FIELD_PREP(MT8195_DLY_RXC_INV, mac_delay->rx_inv); + break; + case PHY_INTERFACE_MODE_RMII: + if (plat->rmii_clk_from_mac) { + /* case 1: mac provides the rmii reference clock, + * and the clock output to TXC pin. + * The egress timing can be adjusted by RMII_TXC delay macro circuit. + * The ingress timing can be adjusted by RMII_RXC delay macro circuit. + */ + rmii_delay_val |= FIELD_PREP(MT8195_DLY_RMII_TXC_ENABLE, + !!mac_delay->tx_delay); + rmii_delay_val |= FIELD_PREP(MT8195_DLY_RMII_TXC_STAGES, + mac_delay->tx_delay); + rmii_delay_val |= FIELD_PREP(MT8195_DLY_RMII_TXC_INV, + mac_delay->tx_inv); + + rmii_delay_val |= FIELD_PREP(MT8195_DLY_RMII_RXC_ENABLE, + !!mac_delay->rx_delay); + rmii_delay_val |= FIELD_PREP(MT8195_DLY_RMII_RXC_STAGES, + mac_delay->rx_delay); + rmii_delay_val |= FIELD_PREP(MT8195_DLY_RMII_RXC_INV, + mac_delay->rx_inv); + } else { + /* case 2: the rmii reference clock is from external phy, + * and the property "rmii_rxc" indicates which pin(TXC/RXC) + * the reference clk is connected to. The reference clock is a + * received signal, so rx_delay/rx_inv are used to indicate + * the reference clock timing adjustment + */ + if (plat->rmii_rxc) { + /* the rmii reference clock from outside is connected + * to RXC pin, the reference clock will be adjusted + * by RXC delay macro circuit. + */ + delay_val |= FIELD_PREP(MT8195_DLY_RXC_ENABLE, + !!mac_delay->rx_delay); + delay_val |= FIELD_PREP(MT8195_DLY_RXC_STAGES, + mac_delay->rx_delay); + delay_val |= FIELD_PREP(MT8195_DLY_RXC_INV, + mac_delay->rx_inv); + } else { + /* the rmii reference clock from outside is connected + * to TXC pin, the reference clock will be adjusted + * by TXC delay macro circuit. + */ + delay_val |= FIELD_PREP(MT8195_DLY_TXC_ENABLE, + !!mac_delay->rx_delay); + delay_val |= FIELD_PREP(MT8195_DLY_TXC_STAGES, + mac_delay->rx_delay); + delay_val |= FIELD_PREP(MT8195_DLY_TXC_INV, + mac_delay->rx_inv); + } + } + break; + case PHY_INTERFACE_MODE_RGMII: + case PHY_INTERFACE_MODE_RGMII_TXID: + case PHY_INTERFACE_MODE_RGMII_RXID: + case PHY_INTERFACE_MODE_RGMII_ID: + gtxc_delay_val |= FIELD_PREP(MT8195_DLY_GTXC_ENABLE, !!mac_delay->tx_delay); + gtxc_delay_val |= FIELD_PREP(MT8195_DLY_GTXC_STAGES, mac_delay->tx_delay); + gtxc_delay_val |= FIELD_PREP(MT8195_DLY_GTXC_INV, mac_delay->tx_inv); + + delay_val |= FIELD_PREP(MT8195_DLY_RXC_ENABLE, !!mac_delay->rx_delay); + delay_val |= FIELD_PREP(MT8195_DLY_RXC_STAGES, mac_delay->rx_delay); + delay_val |= FIELD_PREP(MT8195_DLY_RXC_INV, mac_delay->rx_inv); + + break; + default: + dev_err(plat->dev, "phy interface not supported\n"); + return -EINVAL; + } + + regmap_update_bits(plat->peri_regmap, + MT8195_PERI_ETH_CTRL0, + MT8195_RGMII_TXC_PHASE_CTRL | + MT8195_DLY_GTXC_INV | + MT8195_DLY_GTXC_ENABLE | + MT8195_DLY_GTXC_STAGES, + gtxc_delay_val); + regmap_write(plat->peri_regmap, MT8195_PERI_ETH_CTRL1, delay_val); + regmap_write(plat->peri_regmap, MT8195_PERI_ETH_CTRL2, rmii_delay_val); + + mt8195_delay_stage2ps(plat); + + return 0; +} + +static void mt8195_fix_mac_speed(void *priv, unsigned int speed) +{ + struct mediatek_dwmac_plat_data *priv_plat = priv; + + if ((phy_interface_mode_is_rgmii(priv_plat->phy_mode))) { + /* prefer 2ns fixed delay which is controlled by TXC_PHASE_CTRL, + * when link speed is 1Gbps with RGMII interface, + * Fall back to delay macro circuit for 10/100Mbps link speed. + */ + if (speed == SPEED_1000) + regmap_update_bits(priv_plat->peri_regmap, + MT8195_PERI_ETH_CTRL0, + MT8195_RGMII_TXC_PHASE_CTRL | + MT8195_DLY_GTXC_ENABLE | + MT8195_DLY_GTXC_INV | + MT8195_DLY_GTXC_STAGES, + MT8195_RGMII_TXC_PHASE_CTRL); + else + mt8195_set_delay(priv_plat); + } +} + +static const struct mediatek_dwmac_variant mt8195_gmac_variant = { + .dwmac_set_phy_interface = mt8195_set_interface, + .dwmac_set_delay = mt8195_set_delay, + .dwmac_fix_mac_speed = mt8195_fix_mac_speed, + .clk_list = mt8195_dwmac_clk_l, + .num_clks = ARRAY_SIZE(mt8195_dwmac_clk_l), + .dma_bit_mask = 35, + .rx_delay_max = 9280, + .tx_delay_max = 9280, +}; + static int mediatek_dwmac_config_dt(struct mediatek_dwmac_plat_data *plat) { struct mac_delay_struct *mac_delay = &plat->mac_delay; @@ -308,6 +516,7 @@ static int mediatek_dwmac_config_dt(struct mediatek_dwmac_plat_data *plat) mac_delay->rx_inv = of_property_read_bool(plat->np, "mediatek,rxc-inverse"); plat->rmii_rxc = of_property_read_bool(plat->np, "mediatek,rmii-rxc"); plat->rmii_clk_from_mac = of_property_read_bool(plat->np, "mediatek,rmii-clk-from-mac"); + plat->mac_wol = of_property_read_bool(plat->np, "mediatek,mac-wol"); return 0; } @@ -315,18 +524,34 @@ static int mediatek_dwmac_config_dt(struct mediatek_dwmac_plat_data *plat) static int mediatek_dwmac_clk_init(struct mediatek_dwmac_plat_data *plat) { const struct mediatek_dwmac_variant *variant = plat->variant; - int i, num = variant->num_clks; + int i, ret; - plat->clks = devm_kcalloc(plat->dev, num, sizeof(*plat->clks), GFP_KERNEL); + plat->clks = devm_kcalloc(plat->dev, variant->num_clks, sizeof(*plat->clks), GFP_KERNEL); if (!plat->clks) return -ENOMEM; - for (i = 0; i < num; i++) + for (i = 0; i < variant->num_clks; i++) plat->clks[i].id = variant->clk_list[i]; - plat->num_clks_to_config = variant->num_clks; + ret = devm_clk_bulk_get(plat->dev, variant->num_clks, plat->clks); + if (ret) + return ret; + + /* The clock labeled as "rmii_internal" is needed only in RMII(when + * MAC provides the reference clock), and useless for RGMII/MII or + * RMII(when PHY provides the reference clock). + * So, "rmii_internal" clock is got and configured only when + * reference clock of RMII is from MAC. + */ + if (plat->rmii_clk_from_mac) { + plat->rmii_internal_clk = devm_clk_get(plat->dev, "rmii_internal"); + if (IS_ERR(plat->rmii_internal_clk)) + ret = PTR_ERR(plat->rmii_internal_clk); + } else { + plat->rmii_internal_clk = NULL; + } - return devm_clk_bulk_get(plat->dev, num, plat->clks); + return ret; } static int mediatek_dwmac_init(struct platform_device *pdev, void *priv) @@ -335,44 +560,91 @@ static int mediatek_dwmac_init(struct platform_device *pdev, void *priv) const struct mediatek_dwmac_variant *variant = plat->variant; int ret; - ret = dma_set_mask_and_coherent(plat->dev, DMA_BIT_MASK(variant->dma_bit_mask)); - if (ret) { - dev_err(plat->dev, "No suitable DMA available, err = %d\n", ret); - return ret; + if (variant->dwmac_set_phy_interface) { + ret = variant->dwmac_set_phy_interface(plat); + if (ret) { + dev_err(plat->dev, "failed to set phy interface, err = %d\n", ret); + return ret; + } } - ret = variant->dwmac_set_phy_interface(plat); - if (ret) { - dev_err(plat->dev, "failed to set phy interface, err = %d\n", ret); - return ret; + if (variant->dwmac_set_delay) { + ret = variant->dwmac_set_delay(plat); + if (ret) { + dev_err(plat->dev, "failed to set delay value, err = %d\n", ret); + return ret; + } } - ret = variant->dwmac_set_delay(plat); - if (ret) { - dev_err(plat->dev, "failed to set delay value, err = %d\n", ret); - return ret; - } + return 0; +} - ret = clk_bulk_prepare_enable(plat->num_clks_to_config, plat->clks); - if (ret) { - dev_err(plat->dev, "failed to enable clks, err = %d\n", ret); - return ret; - } +static int mediatek_dwmac_clks_config(void *priv, bool enabled) +{ + struct mediatek_dwmac_plat_data *plat = priv; + const struct mediatek_dwmac_variant *variant = plat->variant; + int ret = 0; + + if (enabled) { + ret = clk_bulk_prepare_enable(variant->num_clks, plat->clks); + if (ret) { + dev_err(plat->dev, "failed to enable clks, err = %d\n", ret); + return ret; + } - pm_runtime_enable(&pdev->dev); - pm_runtime_get_sync(&pdev->dev); + ret = clk_prepare_enable(plat->rmii_internal_clk); + if (ret) { + dev_err(plat->dev, "failed to enable rmii internal clk, err = %d\n", ret); + return ret; + } + } else { + clk_disable_unprepare(plat->rmii_internal_clk); + clk_bulk_disable_unprepare(variant->num_clks, plat->clks); + } - return 0; + return ret; } -static void mediatek_dwmac_exit(struct platform_device *pdev, void *priv) +static int mediatek_dwmac_common_data(struct platform_device *pdev, + struct plat_stmmacenet_data *plat, + struct mediatek_dwmac_plat_data *priv_plat) { - struct mediatek_dwmac_plat_data *plat = priv; + int i; + + plat->interface = priv_plat->phy_mode; + plat->use_phy_wol = priv_plat->mac_wol ? 0 : 1; + plat->riwt_off = 1; + plat->maxmtu = ETH_DATA_LEN; + plat->addr64 = priv_plat->variant->dma_bit_mask; + plat->bsp_priv = priv_plat; + plat->init = mediatek_dwmac_init; + plat->clks_config = mediatek_dwmac_clks_config; + if (priv_plat->variant->dwmac_fix_mac_speed) + plat->fix_mac_speed = priv_plat->variant->dwmac_fix_mac_speed; + + plat->safety_feat_cfg = devm_kzalloc(&pdev->dev, + sizeof(*plat->safety_feat_cfg), + GFP_KERNEL); + if (!plat->safety_feat_cfg) + return -ENOMEM; - clk_bulk_disable_unprepare(plat->num_clks_to_config, plat->clks); + plat->safety_feat_cfg->tsoee = 1; + plat->safety_feat_cfg->mrxpee = 0; + plat->safety_feat_cfg->mestee = 1; + plat->safety_feat_cfg->mrxee = 1; + plat->safety_feat_cfg->mtxee = 1; + plat->safety_feat_cfg->epsi = 0; + plat->safety_feat_cfg->edpp = 1; + plat->safety_feat_cfg->prtyen = 1; + plat->safety_feat_cfg->tmouten = 1; + + for (i = 0; i < plat->tx_queues_to_use; i++) { + /* Default TX Q0 to use TSO and rest TXQ for TBS */ + if (i > 0) + plat->tx_queues_cfg[i].tbs_en = 1; + } - pm_runtime_put_sync(&pdev->dev); - pm_runtime_disable(&pdev->dev); + return 0; } static int mediatek_dwmac_probe(struct platform_device *pdev) @@ -407,33 +679,47 @@ static int mediatek_dwmac_probe(struct platform_device *pdev) if (ret) return ret; - plat_dat = stmmac_probe_config_dt(pdev, &stmmac_res.mac); + plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac); if (IS_ERR(plat_dat)) return PTR_ERR(plat_dat); - plat_dat->interface = priv_plat->phy_mode; - plat_dat->has_gmac4 = 1; - plat_dat->has_gmac = 0; - plat_dat->pmt = 0; - plat_dat->riwt_off = 1; - plat_dat->maxmtu = ETH_DATA_LEN; - plat_dat->bsp_priv = priv_plat; - plat_dat->init = mediatek_dwmac_init; - plat_dat->exit = mediatek_dwmac_exit; + mediatek_dwmac_common_data(pdev, plat_dat, priv_plat); mediatek_dwmac_init(pdev, priv_plat); + ret = mediatek_dwmac_clks_config(priv_plat, true); + if (ret) + goto err_remove_config_dt; + ret = stmmac_dvr_probe(&pdev->dev, plat_dat, &stmmac_res); - if (ret) { - stmmac_remove_config_dt(pdev, plat_dat); - return ret; - } + if (ret) + goto err_drv_probe; return 0; + +err_drv_probe: + mediatek_dwmac_clks_config(priv_plat, false); +err_remove_config_dt: + stmmac_remove_config_dt(pdev, plat_dat); + + return ret; +} + +static int mediatek_dwmac_remove(struct platform_device *pdev) +{ + struct mediatek_dwmac_plat_data *priv_plat = get_stmmac_bsp_priv(&pdev->dev); + int ret; + + ret = stmmac_pltfr_remove(pdev); + mediatek_dwmac_clks_config(priv_plat, false); + + return ret; } static const struct of_device_id mediatek_dwmac_match[] = { { .compatible = "mediatek,mt2712-gmac", .data = &mt2712_gmac_variant }, + { .compatible = "mediatek,mt8195-gmac", + .data = &mt8195_gmac_variant }, { } }; @@ -441,7 +727,7 @@ MODULE_DEVICE_TABLE(of, mediatek_dwmac_match); static struct platform_driver mediatek_dwmac_driver = { .probe = mediatek_dwmac_probe, - .remove = stmmac_pltfr_remove, + .remove = mediatek_dwmac_remove, .driver = { .name = "dwmac-mediatek", .pm = &stmmac_pltfr_pm_ops, diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-meson.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-meson.c index bbc16b5a410a..16fb66a0ca72 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-meson.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-meson.c @@ -52,7 +52,7 @@ static int meson6_dwmac_probe(struct platform_device *pdev) if (ret) return ret; - plat_dat = stmmac_probe_config_dt(pdev, &stmmac_res.mac); + plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac); if (IS_ERR(plat_dat)) return PTR_ERR(plat_dat); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c index 0e2fa14f1423..e8b507f88fbc 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c @@ -5,6 +5,7 @@ * Copyright (C) 2016 Martin Blumenstingl <martin.blumenstingl@googlemail.com> */ +#include <linux/bitfield.h> #include <linux/clk.h> #include <linux/clk-provider.h> #include <linux/device.h> @@ -29,10 +30,12 @@ #define PRG_ETH0_EXT_RMII_MODE 4 /* mux to choose between fclk_div2 (bit unset) and mpll2 (bit set) */ -#define PRG_ETH0_CLK_M250_SEL_SHIFT 4 #define PRG_ETH0_CLK_M250_SEL_MASK GENMASK(4, 4) -#define PRG_ETH0_TXDLY_SHIFT 5 +/* TX clock delay in ns = "8ns / 4 * tx_dly_val" (where 8ns are exactly one + * cycle of the 125MHz RGMII TX clock): + * 0ns = 0x0, 2ns = 0x1, 4ns = 0x2, 6ns = 0x3 + */ #define PRG_ETH0_TXDLY_MASK GENMASK(6, 5) /* divider for the result of m250_sel */ @@ -44,12 +47,42 @@ #define PRG_ETH0_INVERTED_RMII_CLK BIT(11) #define PRG_ETH0_TX_AND_PHY_REF_CLK BIT(12) -#define MUX_CLK_NUM_PARENTS 2 +/* Bypass (= 0, the signal from the GPIO input directly connects to the + * internal sampling) or enable (= 1) the internal logic for RXEN and RXD[3:0] + * timing tuning. + */ +#define PRG_ETH0_ADJ_ENABLE BIT(13) +/* Controls whether the RXEN and RXD[3:0] signals should be aligned with the + * input RX rising/falling edge and sent to the Ethernet internals. This sets + * the automatically delay and skew automatically (internally). + */ +#define PRG_ETH0_ADJ_SETUP BIT(14) +/* An internal counter based on the "timing-adjustment" clock. The counter is + * cleared on both, the falling and rising edge of the RX_CLK. This selects the + * delay (= the counter value) when to start sampling RXEN and RXD[3:0]. + */ +#define PRG_ETH0_ADJ_DELAY GENMASK(19, 15) +/* Adjusts the skew between each bit of RXEN and RXD[3:0]. If a signal has a + * large input delay, the bit for that signal (RXEN = bit 0, RXD[3] = bit 1, + * ...) can be configured to be 1 to compensate for a delay of about 1ns. + */ +#define PRG_ETH0_ADJ_SKEW GENMASK(24, 20) + +#define PRG_ETH1 0x4 + +/* Defined for adding a delay to the input RX_CLK for better timing. + * Each step is 200ps. These bits are used with external RGMII PHYs + * because RGMII RX only has the small window. cfg_rxclk_dly can + * adjust the window between RX_CLK and RX_DATA and improve the stability + * of "rx data valid". + */ +#define PRG_ETH1_CFG_RXCLK_DLY GENMASK(19, 16) struct meson8b_dwmac; struct meson8b_dwmac_data { int (*set_phy_mode)(struct meson8b_dwmac *dwmac); + bool has_prg_eth1_rgmii_rx_delay; }; struct meson8b_dwmac { @@ -60,6 +93,8 @@ struct meson8b_dwmac { phy_interface_t phy_mode; struct clk *rgmii_tx_clk; u32 tx_delay_ns; + u32 rx_delay_ps; + struct clk *timing_adj_clk; }; struct meson8b_dwmac_clk_configs { @@ -83,12 +118,12 @@ static void meson8b_dwmac_mask_bits(struct meson8b_dwmac *dwmac, u32 reg, static struct clk *meson8b_dwmac_register_clk(struct meson8b_dwmac *dwmac, const char *name_suffix, - const char **parent_names, + const struct clk_parent_data *parents, int num_parents, const struct clk_ops *ops, struct clk_hw *hw) { - struct clk_init_data init; + struct clk_init_data init = { }; char clk_name[32]; snprintf(clk_name, sizeof(clk_name), "%s#%s", dev_name(dwmac->dev), @@ -97,7 +132,7 @@ static struct clk *meson8b_dwmac_register_clk(struct meson8b_dwmac *dwmac, init.name = clk_name; init.ops = ops; init.flags = CLK_SET_RATE_PARENT; - init.parent_names = parent_names; + init.parent_data = parents; init.num_parents = num_parents; hw->init = &init; @@ -107,11 +142,12 @@ static struct clk *meson8b_dwmac_register_clk(struct meson8b_dwmac *dwmac, static int meson8b_init_rgmii_tx_clk(struct meson8b_dwmac *dwmac) { - int i, ret; struct clk *clk; struct device *dev = dwmac->dev; - const char *parent_name, *mux_parent_names[MUX_CLK_NUM_PARENTS]; - struct meson8b_dwmac_clk_configs *clk_configs; + static const struct clk_parent_data mux_parents[] = { + { .fw_name = "clkin0", }, + { .index = -1, }, + }; static const struct clk_div_table div_table[] = { { .div = 2, .val = 2, }, { .div = 3, .val = 3, }, @@ -119,63 +155,51 @@ static int meson8b_init_rgmii_tx_clk(struct meson8b_dwmac *dwmac) { .div = 5, .val = 5, }, { .div = 6, .val = 6, }, { .div = 7, .val = 7, }, + { /* end of array */ } }; + struct meson8b_dwmac_clk_configs *clk_configs; + struct clk_parent_data parent_data = { }; clk_configs = devm_kzalloc(dev, sizeof(*clk_configs), GFP_KERNEL); if (!clk_configs) return -ENOMEM; - /* get the mux parents from DT */ - for (i = 0; i < MUX_CLK_NUM_PARENTS; i++) { - char name[16]; - - snprintf(name, sizeof(name), "clkin%d", i); - clk = devm_clk_get(dev, name); - if (IS_ERR(clk)) { - ret = PTR_ERR(clk); - if (ret != -EPROBE_DEFER) - dev_err(dev, "Missing clock %s\n", name); - return ret; - } - - mux_parent_names[i] = __clk_get_name(clk); - } - clk_configs->m250_mux.reg = dwmac->regs + PRG_ETH0; - clk_configs->m250_mux.shift = PRG_ETH0_CLK_M250_SEL_SHIFT; - clk_configs->m250_mux.mask = PRG_ETH0_CLK_M250_SEL_MASK; - clk = meson8b_dwmac_register_clk(dwmac, "m250_sel", mux_parent_names, - MUX_CLK_NUM_PARENTS, &clk_mux_ops, + clk_configs->m250_mux.shift = __ffs(PRG_ETH0_CLK_M250_SEL_MASK); + clk_configs->m250_mux.mask = PRG_ETH0_CLK_M250_SEL_MASK >> + clk_configs->m250_mux.shift; + clk = meson8b_dwmac_register_clk(dwmac, "m250_sel", mux_parents, + ARRAY_SIZE(mux_parents), &clk_mux_ops, &clk_configs->m250_mux.hw); if (WARN_ON(IS_ERR(clk))) return PTR_ERR(clk); - parent_name = __clk_get_name(clk); + parent_data.hw = &clk_configs->m250_mux.hw; clk_configs->m250_div.reg = dwmac->regs + PRG_ETH0; clk_configs->m250_div.shift = PRG_ETH0_CLK_M250_DIV_SHIFT; clk_configs->m250_div.width = PRG_ETH0_CLK_M250_DIV_WIDTH; clk_configs->m250_div.table = div_table; clk_configs->m250_div.flags = CLK_DIVIDER_ALLOW_ZERO | CLK_DIVIDER_ROUND_CLOSEST; - clk = meson8b_dwmac_register_clk(dwmac, "m250_div", &parent_name, 1, + clk = meson8b_dwmac_register_clk(dwmac, "m250_div", &parent_data, 1, &clk_divider_ops, &clk_configs->m250_div.hw); if (WARN_ON(IS_ERR(clk))) return PTR_ERR(clk); - parent_name = __clk_get_name(clk); + parent_data.hw = &clk_configs->m250_div.hw; clk_configs->fixed_div2.mult = 1; clk_configs->fixed_div2.div = 2; - clk = meson8b_dwmac_register_clk(dwmac, "fixed_div2", &parent_name, 1, + clk = meson8b_dwmac_register_clk(dwmac, "fixed_div2", &parent_data, 1, &clk_fixed_factor_ops, &clk_configs->fixed_div2.hw); if (WARN_ON(IS_ERR(clk))) return PTR_ERR(clk); - parent_name = __clk_get_name(clk); + parent_data.hw = &clk_configs->fixed_div2.hw; clk_configs->rgmii_tx_en.reg = dwmac->regs + PRG_ETH0; clk_configs->rgmii_tx_en.bit_idx = PRG_ETH0_RGMII_TX_CLK_EN; - clk = meson8b_dwmac_register_clk(dwmac, "rgmii_tx_en", &parent_name, 1, + clk = meson8b_dwmac_register_clk(dwmac, "rgmii_tx_en", &parent_data, 1, &clk_gate_ops, &clk_configs->rgmii_tx_en.hw); if (WARN_ON(IS_ERR(clk))) @@ -239,30 +263,95 @@ static int meson_axg_set_phy_mode(struct meson8b_dwmac *dwmac) return 0; } -static int meson8b_init_prg_eth(struct meson8b_dwmac *dwmac) +static int meson8b_devm_clk_prepare_enable(struct meson8b_dwmac *dwmac, + struct clk *clk) +{ + int ret; + + ret = clk_prepare_enable(clk); + if (ret) + return ret; + + return devm_add_action_or_reset(dwmac->dev, + (void(*)(void *))clk_disable_unprepare, + clk); +} + +static int meson8b_init_rgmii_delays(struct meson8b_dwmac *dwmac) { + u32 tx_dly_config, rx_adj_config, cfg_rxclk_dly, delay_config; int ret; - u8 tx_dly_val = 0; + + rx_adj_config = 0; + cfg_rxclk_dly = 0; + tx_dly_config = FIELD_PREP(PRG_ETH0_TXDLY_MASK, + dwmac->tx_delay_ns >> 1); + + if (dwmac->data->has_prg_eth1_rgmii_rx_delay) + cfg_rxclk_dly = FIELD_PREP(PRG_ETH1_CFG_RXCLK_DLY, + dwmac->rx_delay_ps / 200); + else if (dwmac->rx_delay_ps == 2000) + rx_adj_config = PRG_ETH0_ADJ_ENABLE | PRG_ETH0_ADJ_SETUP; switch (dwmac->phy_mode) { case PHY_INTERFACE_MODE_RGMII: + delay_config = tx_dly_config | rx_adj_config; + break; case PHY_INTERFACE_MODE_RGMII_RXID: - /* TX clock delay in ns = "8ns / 4 * tx_dly_val" (where - * 8ns are exactly one cycle of the 125MHz RGMII TX clock): - * 0ns = 0x0, 2ns = 0x1, 4ns = 0x2, 6ns = 0x3 - */ - tx_dly_val = dwmac->tx_delay_ns >> 1; - /* fall through */ - - case PHY_INTERFACE_MODE_RGMII_ID: + delay_config = tx_dly_config; + cfg_rxclk_dly = 0; + break; case PHY_INTERFACE_MODE_RGMII_TXID: + delay_config = rx_adj_config; + break; + case PHY_INTERFACE_MODE_RGMII_ID: + case PHY_INTERFACE_MODE_RMII: + delay_config = 0; + cfg_rxclk_dly = 0; + break; + default: + dev_err(dwmac->dev, "unsupported phy-mode %s\n", + phy_modes(dwmac->phy_mode)); + return -EINVAL; + } + + if (delay_config & PRG_ETH0_ADJ_ENABLE) { + if (!dwmac->timing_adj_clk) { + dev_err(dwmac->dev, + "The timing-adjustment clock is mandatory for the RX delay re-timing\n"); + return -EINVAL; + } + + /* The timing adjustment logic is driven by a separate clock */ + ret = meson8b_devm_clk_prepare_enable(dwmac, + dwmac->timing_adj_clk); + if (ret) { + dev_err(dwmac->dev, + "Failed to enable the timing-adjustment clock\n"); + return ret; + } + } + + meson8b_dwmac_mask_bits(dwmac, PRG_ETH0, PRG_ETH0_TXDLY_MASK | + PRG_ETH0_ADJ_ENABLE | PRG_ETH0_ADJ_SETUP | + PRG_ETH0_ADJ_DELAY | PRG_ETH0_ADJ_SKEW, + delay_config); + + meson8b_dwmac_mask_bits(dwmac, PRG_ETH1, PRG_ETH1_CFG_RXCLK_DLY, + cfg_rxclk_dly); + + return 0; +} + +static int meson8b_init_prg_eth(struct meson8b_dwmac *dwmac) +{ + int ret; + + if (phy_interface_mode_is_rgmii(dwmac->phy_mode)) { /* only relevant for RMII mode -> disable in RGMII mode */ meson8b_dwmac_mask_bits(dwmac, PRG_ETH0, PRG_ETH0_INVERTED_RMII_CLK, 0); - meson8b_dwmac_mask_bits(dwmac, PRG_ETH0, PRG_ETH0_TXDLY_MASK, - tx_dly_val << PRG_ETH0_TXDLY_SHIFT); - /* Configure the 125MHz RGMII TX clock, the IP block changes * the output automatically (= without us having to configure * a register) based on the line-speed (125MHz for Gbit speeds, @@ -275,34 +364,18 @@ static int meson8b_init_prg_eth(struct meson8b_dwmac *dwmac) return ret; } - ret = clk_prepare_enable(dwmac->rgmii_tx_clk); + ret = meson8b_devm_clk_prepare_enable(dwmac, + dwmac->rgmii_tx_clk); if (ret) { dev_err(dwmac->dev, "failed to enable the RGMII TX clock\n"); return ret; } - - devm_add_action_or_reset(dwmac->dev, - (void(*)(void *))clk_disable_unprepare, - dwmac->rgmii_tx_clk); - break; - - case PHY_INTERFACE_MODE_RMII: + } else { /* invert internal clk_rmii_i to generate 25/2.5 tx_rx_clk */ meson8b_dwmac_mask_bits(dwmac, PRG_ETH0, PRG_ETH0_INVERTED_RMII_CLK, PRG_ETH0_INVERTED_RMII_CLK); - - /* TX clock delay cannot be configured in RMII mode */ - meson8b_dwmac_mask_bits(dwmac, PRG_ETH0, PRG_ETH0_TXDLY_MASK, - 0); - - break; - - default: - dev_err(dwmac->dev, "unsupported phy-mode %s\n", - phy_modes(dwmac->phy_mode)); - return -EINVAL; } /* enable TX_CLK and PHY_REF_CLK generator */ @@ -323,7 +396,7 @@ static int meson8b_dwmac_probe(struct platform_device *pdev) if (ret) return ret; - plat_dat = stmmac_probe_config_dt(pdev, &stmmac_res.mac); + plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac); if (IS_ERR(plat_dat)) return PTR_ERR(plat_dat); @@ -357,6 +430,43 @@ static int meson8b_dwmac_probe(struct platform_device *pdev) &dwmac->tx_delay_ns)) dwmac->tx_delay_ns = 2; + /* RX delay defaults to 0ps since this is what many boards use */ + if (of_property_read_u32(pdev->dev.of_node, "rx-internal-delay-ps", + &dwmac->rx_delay_ps)) { + if (!of_property_read_u32(pdev->dev.of_node, + "amlogic,rx-delay-ns", + &dwmac->rx_delay_ps)) + /* convert ns to ps */ + dwmac->rx_delay_ps *= 1000; + } + + if (dwmac->data->has_prg_eth1_rgmii_rx_delay) { + if (dwmac->rx_delay_ps > 3000 || dwmac->rx_delay_ps % 200) { + dev_err(dwmac->dev, + "The RGMII RX delay range is 0..3000ps in 200ps steps"); + ret = -EINVAL; + goto err_remove_config_dt; + } + } else { + if (dwmac->rx_delay_ps != 0 && dwmac->rx_delay_ps != 2000) { + dev_err(dwmac->dev, + "The only allowed RGMII RX delays values are: 0ps, 2000ps"); + ret = -EINVAL; + goto err_remove_config_dt; + } + } + + dwmac->timing_adj_clk = devm_clk_get_optional(dwmac->dev, + "timing-adjustment"); + if (IS_ERR(dwmac->timing_adj_clk)) { + ret = PTR_ERR(dwmac->timing_adj_clk); + goto err_remove_config_dt; + } + + ret = meson8b_init_rgmii_delays(dwmac); + if (ret) + goto err_remove_config_dt; + ret = meson8b_init_rgmii_tx_clk(dwmac); if (ret) goto err_remove_config_dt; @@ -385,10 +495,17 @@ err_remove_config_dt: static const struct meson8b_dwmac_data meson8b_dwmac_data = { .set_phy_mode = meson8b_set_phy_mode, + .has_prg_eth1_rgmii_rx_delay = false, }; static const struct meson8b_dwmac_data meson_axg_dwmac_data = { .set_phy_mode = meson_axg_set_phy_mode, + .has_prg_eth1_rgmii_rx_delay = false, +}; + +static const struct meson8b_dwmac_data meson_g12a_dwmac_data = { + .set_phy_mode = meson_axg_set_phy_mode, + .has_prg_eth1_rgmii_rx_delay = true, }; static const struct of_device_id meson8b_dwmac_match[] = { @@ -408,6 +525,10 @@ static const struct of_device_id meson8b_dwmac_match[] = { .compatible = "amlogic,meson-axg-dwmac", .data = &meson_axg_dwmac_data, }, + { + .compatible = "amlogic,meson-g12a-dwmac", + .data = &meson_g12a_dwmac_data, + }, { } }; MODULE_DEVICE_TABLE(of, meson8b_dwmac_match); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-oxnas.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-oxnas.c index 8551ea878ba5..62a69a91ab22 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-oxnas.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-oxnas.c @@ -12,6 +12,7 @@ #include <linux/io.h> #include <linux/module.h> #include <linux/of.h> +#include <linux/of_device.h> #include <linux/platform_device.h> #include <linux/regmap.h> #include <linux/mfd/syscon.h> @@ -48,46 +49,60 @@ #define DWMAC_RX_VARDELAY(d) ((d) << DWMAC_RX_VARDELAY_SHIFT) #define DWMAC_RXN_VARDELAY(d) ((d) << DWMAC_RXN_VARDELAY_SHIFT) +struct oxnas_dwmac; + +struct oxnas_dwmac_data { + int (*setup)(struct oxnas_dwmac *dwmac); +}; + struct oxnas_dwmac { struct device *dev; struct clk *clk; struct regmap *regmap; + const struct oxnas_dwmac_data *data; }; -static int oxnas_dwmac_init(struct platform_device *pdev, void *priv) +static int oxnas_dwmac_setup_ox810se(struct oxnas_dwmac *dwmac) { - struct oxnas_dwmac *dwmac = priv; unsigned int value; int ret; - /* Reset HW here before changing the glue configuration */ - ret = device_reset(dwmac->dev); - if (ret) + ret = regmap_read(dwmac->regmap, OXNAS_DWMAC_CTRL_REGOFFSET, &value); + if (ret < 0) return ret; - ret = clk_prepare_enable(dwmac->clk); - if (ret) - return ret; + /* Enable GMII_GTXCLK to follow GMII_REFCLK, required for gigabit PHY */ + value |= BIT(DWMAC_CKEN_GTX) | + /* Use simple mux for 25/125 Mhz clock switching */ + BIT(DWMAC_SIMPLE_MUX); + + regmap_write(dwmac->regmap, OXNAS_DWMAC_CTRL_REGOFFSET, value); + + return 0; +} + +static int oxnas_dwmac_setup_ox820(struct oxnas_dwmac *dwmac) +{ + unsigned int value; + int ret; ret = regmap_read(dwmac->regmap, OXNAS_DWMAC_CTRL_REGOFFSET, &value); - if (ret < 0) { - clk_disable_unprepare(dwmac->clk); + if (ret < 0) return ret; - } /* Enable GMII_GTXCLK to follow GMII_REFCLK, required for gigabit PHY */ value |= BIT(DWMAC_CKEN_GTX) | /* Use simple mux for 25/125 Mhz clock switching */ - BIT(DWMAC_SIMPLE_MUX) | - /* set auto switch tx clock source */ - BIT(DWMAC_AUTO_TX_SOURCE) | - /* enable tx & rx vardelay */ - BIT(DWMAC_CKEN_TX_OUT) | - BIT(DWMAC_CKEN_TXN_OUT) | - BIT(DWMAC_CKEN_TX_IN) | - BIT(DWMAC_CKEN_RX_OUT) | - BIT(DWMAC_CKEN_RXN_OUT) | - BIT(DWMAC_CKEN_RX_IN); + BIT(DWMAC_SIMPLE_MUX) | + /* set auto switch tx clock source */ + BIT(DWMAC_AUTO_TX_SOURCE) | + /* enable tx & rx vardelay */ + BIT(DWMAC_CKEN_TX_OUT) | + BIT(DWMAC_CKEN_TXN_OUT) | + BIT(DWMAC_CKEN_TX_IN) | + BIT(DWMAC_CKEN_RX_OUT) | + BIT(DWMAC_CKEN_RXN_OUT) | + BIT(DWMAC_CKEN_RX_IN); regmap_write(dwmac->regmap, OXNAS_DWMAC_CTRL_REGOFFSET, value); /* set tx & rx vardelay */ @@ -100,6 +115,27 @@ static int oxnas_dwmac_init(struct platform_device *pdev, void *priv) return 0; } +static int oxnas_dwmac_init(struct platform_device *pdev, void *priv) +{ + struct oxnas_dwmac *dwmac = priv; + int ret; + + /* Reset HW here before changing the glue configuration */ + ret = device_reset(dwmac->dev); + if (ret) + return ret; + + ret = clk_prepare_enable(dwmac->clk); + if (ret) + return ret; + + ret = dwmac->data->setup(dwmac); + if (ret) + clk_disable_unprepare(dwmac->clk); + + return ret; +} + static void oxnas_dwmac_exit(struct platform_device *pdev, void *priv) { struct oxnas_dwmac *dwmac = priv; @@ -118,7 +154,7 @@ static int oxnas_dwmac_probe(struct platform_device *pdev) if (ret) return ret; - plat_dat = stmmac_probe_config_dt(pdev, &stmmac_res.mac); + plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac); if (IS_ERR(plat_dat)) return PTR_ERR(plat_dat); @@ -128,6 +164,12 @@ static int oxnas_dwmac_probe(struct platform_device *pdev) goto err_remove_config_dt; } + dwmac->data = (const struct oxnas_dwmac_data *)of_device_get_match_data(&pdev->dev); + if (!dwmac->data) { + ret = -EINVAL; + goto err_remove_config_dt; + } + dwmac->dev = &pdev->dev; plat_dat->bsp_priv = dwmac; plat_dat->init = oxnas_dwmac_init; @@ -166,8 +208,23 @@ err_remove_config_dt: return ret; } +static const struct oxnas_dwmac_data ox810se_dwmac_data = { + .setup = oxnas_dwmac_setup_ox810se, +}; + +static const struct oxnas_dwmac_data ox820_dwmac_data = { + .setup = oxnas_dwmac_setup_ox820, +}; + static const struct of_device_id oxnas_dwmac_match[] = { - { .compatible = "oxsemi,ox820-dwmac" }, + { + .compatible = "oxsemi,ox810se-dwmac", + .data = &ox810se_dwmac_data, + }, + { + .compatible = "oxsemi,ox820-dwmac", + .data = &ox820_dwmac_data, + }, { } }; MODULE_DEVICE_TABLE(of, oxnas_dwmac_match); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c index e0a5fe83d8e0..835caa15d55f 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c @@ -75,6 +75,12 @@ struct ethqos_emac_por { unsigned int value; }; +struct ethqos_emac_driver_data { + const struct ethqos_emac_por *por; + unsigned int num_por; + bool rgmii_config_looback_en; +}; + struct qcom_ethqos { struct platform_device *pdev; void __iomem *rgmii_base; @@ -85,6 +91,7 @@ struct qcom_ethqos { const struct ethqos_emac_por *por; unsigned int num_por; + bool rgmii_config_looback_en; }; static int rgmii_readl(struct qcom_ethqos *ethqos, unsigned int offset) @@ -108,8 +115,10 @@ static void rgmii_updatel(struct qcom_ethqos *ethqos, rgmii_writel(ethqos, temp, offset); } -static void rgmii_dump(struct qcom_ethqos *ethqos) +static void rgmii_dump(void *priv) { + struct qcom_ethqos *ethqos = priv; + dev_dbg(ðqos->pdev->dev, "Rgmii register dump\n"); dev_dbg(ðqos->pdev->dev, "RGMII_IO_MACRO_CONFIG: %x\n", rgmii_readl(ethqos, RGMII_IO_MACRO_CONFIG)); @@ -171,6 +180,27 @@ static const struct ethqos_emac_por emac_v2_3_0_por[] = { { .offset = RGMII_IO_MACRO_CONFIG2, .value = 0x00002060 }, }; +static const struct ethqos_emac_driver_data emac_v2_3_0_data = { + .por = emac_v2_3_0_por, + .num_por = ARRAY_SIZE(emac_v2_3_0_por), + .rgmii_config_looback_en = true, +}; + +static const struct ethqos_emac_por emac_v2_1_0_por[] = { + { .offset = RGMII_IO_MACRO_CONFIG, .value = 0x40C01343 }, + { .offset = SDCC_HC_REG_DLL_CONFIG, .value = 0x2004642C }, + { .offset = SDCC_HC_REG_DDR_CONFIG, .value = 0x00000000 }, + { .offset = SDCC_HC_REG_DLL_CONFIG2, .value = 0x00200000 }, + { .offset = SDCC_USR_CTL, .value = 0x00010800 }, + { .offset = RGMII_IO_MACRO_CONFIG2, .value = 0x00002060 }, +}; + +static const struct ethqos_emac_driver_data emac_v2_1_0_data = { + .por = emac_v2_1_0_por, + .num_por = ARRAY_SIZE(emac_v2_1_0_por), + .rgmii_config_looback_en = false, +}; + static int ethqos_dll_configure(struct qcom_ethqos *ethqos) { unsigned int val; @@ -285,8 +315,12 @@ static int ethqos_rgmii_macro_init(struct qcom_ethqos *ethqos) rgmii_updatel(ethqos, SDCC_DDR_CONFIG_PRG_DLY_EN, SDCC_DDR_CONFIG_PRG_DLY_EN, SDCC_HC_REG_DDR_CONFIG); - rgmii_updatel(ethqos, RGMII_CONFIG_LOOPBACK_EN, - RGMII_CONFIG_LOOPBACK_EN, RGMII_IO_MACRO_CONFIG); + if (ethqos->rgmii_config_looback_en) + rgmii_updatel(ethqos, RGMII_CONFIG_LOOPBACK_EN, + RGMII_CONFIG_LOOPBACK_EN, RGMII_IO_MACRO_CONFIG); + else + rgmii_updatel(ethqos, RGMII_CONFIG_LOOPBACK_EN, + 0, RGMII_IO_MACRO_CONFIG); break; case SPEED_100: @@ -319,8 +353,13 @@ static int ethqos_rgmii_macro_init(struct qcom_ethqos *ethqos) rgmii_updatel(ethqos, SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY_EN, SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY_EN, SDCC_HC_REG_DDR_CONFIG); - rgmii_updatel(ethqos, RGMII_CONFIG_LOOPBACK_EN, - RGMII_CONFIG_LOOPBACK_EN, RGMII_IO_MACRO_CONFIG); + if (ethqos->rgmii_config_looback_en) + rgmii_updatel(ethqos, RGMII_CONFIG_LOOPBACK_EN, + RGMII_CONFIG_LOOPBACK_EN, RGMII_IO_MACRO_CONFIG); + else + rgmii_updatel(ethqos, RGMII_CONFIG_LOOPBACK_EN, + 0, RGMII_IO_MACRO_CONFIG); + break; case SPEED_10: @@ -437,25 +476,52 @@ static void ethqos_fix_mac_speed(void *priv, unsigned int speed) ethqos_configure(ethqos); } +static int ethqos_clks_config(void *priv, bool enabled) +{ + struct qcom_ethqos *ethqos = priv; + int ret = 0; + + if (enabled) { + ret = clk_prepare_enable(ethqos->rgmii_clk); + if (ret) { + dev_err(ðqos->pdev->dev, "rgmii_clk enable failed\n"); + return ret; + } + + /* Enable functional clock to prevent DMA reset to timeout due + * to lacking PHY clock after the hardware block has been power + * cycled. The actual configuration will be adjusted once + * ethqos_fix_mac_speed() is invoked. + */ + ethqos_set_func_clk_en(ethqos); + } else { + clk_disable_unprepare(ethqos->rgmii_clk); + } + + return ret; +} + static int qcom_ethqos_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; struct plat_stmmacenet_data *plat_dat; struct stmmac_resources stmmac_res; + const struct ethqos_emac_driver_data *data; struct qcom_ethqos *ethqos; - struct resource *res; int ret; ret = stmmac_get_platform_resources(pdev, &stmmac_res); if (ret) return ret; - plat_dat = stmmac_probe_config_dt(pdev, &stmmac_res.mac); + plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac); if (IS_ERR(plat_dat)) { dev_err(&pdev->dev, "dt configuration failed\n"); return PTR_ERR(plat_dat); } + plat_dat->clks_config = ethqos_clks_config; + ethqos = devm_kzalloc(&pdev->dev, sizeof(*ethqos), GFP_KERNEL); if (!ethqos) { ret = -ENOMEM; @@ -463,15 +529,16 @@ static int qcom_ethqos_probe(struct platform_device *pdev) } ethqos->pdev = pdev; - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rgmii"); - ethqos->rgmii_base = devm_ioremap_resource(&pdev->dev, res); + ethqos->rgmii_base = devm_platform_ioremap_resource_byname(pdev, "rgmii"); if (IS_ERR(ethqos->rgmii_base)) { - dev_err(&pdev->dev, "Can't get rgmii base\n"); ret = PTR_ERR(ethqos->rgmii_base); goto err_mem; } - ethqos->por = of_device_get_match_data(&pdev->dev); + data = of_device_get_match_data(&pdev->dev); + ethqos->por = data->por; + ethqos->num_por = data->num_por; + ethqos->rgmii_config_looback_en = data->rgmii_config_looback_en; ethqos->rgmii_clk = devm_clk_get(&pdev->dev, "rgmii"); if (IS_ERR(ethqos->rgmii_clk)) { @@ -479,7 +546,7 @@ static int qcom_ethqos_probe(struct platform_device *pdev) goto err_mem; } - ret = clk_prepare_enable(ethqos->rgmii_clk); + ret = ethqos_clks_config(ethqos, true); if (ret) goto err_mem; @@ -489,6 +556,7 @@ static int qcom_ethqos_probe(struct platform_device *pdev) plat_dat->bsp_priv = ethqos; plat_dat->fix_mac_speed = ethqos_fix_mac_speed; + plat_dat->dump_debug_regs = rgmii_dump; plat_dat->has_gmac4 = 1; plat_dat->pmt = 1; plat_dat->tso_en = of_property_read_bool(np, "snps,tso"); @@ -497,12 +565,10 @@ static int qcom_ethqos_probe(struct platform_device *pdev) if (ret) goto err_clk; - rgmii_dump(ethqos); - return ret; err_clk: - clk_disable_unprepare(ethqos->rgmii_clk); + ethqos_clks_config(ethqos, false); err_mem: stmmac_remove_config_dt(pdev, plat_dat); @@ -520,13 +586,14 @@ static int qcom_ethqos_remove(struct platform_device *pdev) return -ENODEV; ret = stmmac_pltfr_remove(pdev); - clk_disable_unprepare(ethqos->rgmii_clk); + ethqos_clks_config(ethqos, false); return ret; } static const struct of_device_id qcom_ethqos_match[] = { - { .compatible = "qcom,qcs404-ethqos", .data = &emac_v2_3_0_por}, + { .compatible = "qcom,qcs404-ethqos", .data = &emac_v2_3_0_data}, + { .compatible = "qcom,sm8150-ethqos", .data = &emac_v2_1_0_data}, { } }; MODULE_DEVICE_TABLE(of, qcom_ethqos_match); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c index dc50ba13a746..6656d76b6766 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-or-later /** - * dwmac-rk.c - Rockchip RK3288 DWMAC specific glue layer + * DOC: dwmac-rk.c - Rockchip RK3288 DWMAC specific glue layer * * Copyright (C) 2014 Chen-Zhi (Roger Chen) * @@ -32,12 +32,17 @@ struct rk_gmac_ops { void (*set_to_rmii)(struct rk_priv_data *bsp_priv); void (*set_rgmii_speed)(struct rk_priv_data *bsp_priv, int speed); void (*set_rmii_speed)(struct rk_priv_data *bsp_priv, int speed); + void (*set_clock_selection)(struct rk_priv_data *bsp_priv, bool input, + bool enable); void (*integrated_phy_powerup)(struct rk_priv_data *bsp_priv); + bool regs_valid; + u32 regs[]; }; struct rk_priv_data { struct platform_device *pdev; phy_interface_t phy_iface; + int id; struct regulator *regulator; bool suspended; const struct rk_gmac_ops *ops; @@ -63,6 +68,7 @@ struct rk_priv_data { int rx_delay; struct regmap *grf; + struct regmap *php_grf; }; #define HIWORD_UPDATE(val, mask, shift) \ @@ -482,6 +488,54 @@ static const struct rk_gmac_ops rk3288_ops = { .set_rmii_speed = rk3288_set_rmii_speed, }; +#define RK3308_GRF_MAC_CON0 0x04a0 + +/* RK3308_GRF_MAC_CON0 */ +#define RK3308_GMAC_PHY_INTF_SEL_RMII (GRF_CLR_BIT(2) | GRF_CLR_BIT(3) | \ + GRF_BIT(4)) +#define RK3308_GMAC_FLOW_CTRL GRF_BIT(3) +#define RK3308_GMAC_FLOW_CTRL_CLR GRF_CLR_BIT(3) +#define RK3308_GMAC_SPEED_10M GRF_CLR_BIT(0) +#define RK3308_GMAC_SPEED_100M GRF_BIT(0) + +static void rk3308_set_to_rmii(struct rk_priv_data *bsp_priv) +{ + struct device *dev = &bsp_priv->pdev->dev; + + if (IS_ERR(bsp_priv->grf)) { + dev_err(dev, "Missing rockchip,grf property\n"); + return; + } + + regmap_write(bsp_priv->grf, RK3308_GRF_MAC_CON0, + RK3308_GMAC_PHY_INTF_SEL_RMII); +} + +static void rk3308_set_rmii_speed(struct rk_priv_data *bsp_priv, int speed) +{ + struct device *dev = &bsp_priv->pdev->dev; + + if (IS_ERR(bsp_priv->grf)) { + dev_err(dev, "Missing rockchip,grf property\n"); + return; + } + + if (speed == 10) { + regmap_write(bsp_priv->grf, RK3308_GRF_MAC_CON0, + RK3308_GMAC_SPEED_10M); + } else if (speed == 100) { + regmap_write(bsp_priv->grf, RK3308_GRF_MAC_CON0, + RK3308_GMAC_SPEED_100M); + } else { + dev_err(dev, "unknown speed value for RMII! speed=%d", speed); + } +} + +static const struct rk_gmac_ops rk3308_ops = { + .set_to_rmii = rk3308_set_to_rmii, + .set_rmii_speed = rk3308_set_rmii_speed, +}; + #define RK3328_GRF_MAC_CON0 0x0900 #define RK3328_GRF_MAC_CON1 0x0904 #define RK3328_GRF_MAC_CON2 0x0908 @@ -948,6 +1002,255 @@ static const struct rk_gmac_ops rk3399_ops = { .set_rmii_speed = rk3399_set_rmii_speed, }; +#define RK3568_GRF_GMAC0_CON0 0x0380 +#define RK3568_GRF_GMAC0_CON1 0x0384 +#define RK3568_GRF_GMAC1_CON0 0x0388 +#define RK3568_GRF_GMAC1_CON1 0x038c + +/* RK3568_GRF_GMAC0_CON1 && RK3568_GRF_GMAC1_CON1 */ +#define RK3568_GMAC_PHY_INTF_SEL_RGMII \ + (GRF_BIT(4) | GRF_CLR_BIT(5) | GRF_CLR_BIT(6)) +#define RK3568_GMAC_PHY_INTF_SEL_RMII \ + (GRF_CLR_BIT(4) | GRF_CLR_BIT(5) | GRF_BIT(6)) +#define RK3568_GMAC_FLOW_CTRL GRF_BIT(3) +#define RK3568_GMAC_FLOW_CTRL_CLR GRF_CLR_BIT(3) +#define RK3568_GMAC_RXCLK_DLY_ENABLE GRF_BIT(1) +#define RK3568_GMAC_RXCLK_DLY_DISABLE GRF_CLR_BIT(1) +#define RK3568_GMAC_TXCLK_DLY_ENABLE GRF_BIT(0) +#define RK3568_GMAC_TXCLK_DLY_DISABLE GRF_CLR_BIT(0) + +/* RK3568_GRF_GMAC0_CON0 && RK3568_GRF_GMAC1_CON0 */ +#define RK3568_GMAC_CLK_RX_DL_CFG(val) HIWORD_UPDATE(val, 0x7F, 8) +#define RK3568_GMAC_CLK_TX_DL_CFG(val) HIWORD_UPDATE(val, 0x7F, 0) + +static void rk3568_set_to_rgmii(struct rk_priv_data *bsp_priv, + int tx_delay, int rx_delay) +{ + struct device *dev = &bsp_priv->pdev->dev; + u32 con0, con1; + + if (IS_ERR(bsp_priv->grf)) { + dev_err(dev, "Missing rockchip,grf property\n"); + return; + } + + con0 = (bsp_priv->id == 1) ? RK3568_GRF_GMAC1_CON0 : + RK3568_GRF_GMAC0_CON0; + con1 = (bsp_priv->id == 1) ? RK3568_GRF_GMAC1_CON1 : + RK3568_GRF_GMAC0_CON1; + + regmap_write(bsp_priv->grf, con0, + RK3568_GMAC_CLK_RX_DL_CFG(rx_delay) | + RK3568_GMAC_CLK_TX_DL_CFG(tx_delay)); + + regmap_write(bsp_priv->grf, con1, + RK3568_GMAC_PHY_INTF_SEL_RGMII | + RK3568_GMAC_RXCLK_DLY_ENABLE | + RK3568_GMAC_TXCLK_DLY_ENABLE); +} + +static void rk3568_set_to_rmii(struct rk_priv_data *bsp_priv) +{ + struct device *dev = &bsp_priv->pdev->dev; + u32 con1; + + if (IS_ERR(bsp_priv->grf)) { + dev_err(dev, "%s: Missing rockchip,grf property\n", __func__); + return; + } + + con1 = (bsp_priv->id == 1) ? RK3568_GRF_GMAC1_CON1 : + RK3568_GRF_GMAC0_CON1; + regmap_write(bsp_priv->grf, con1, RK3568_GMAC_PHY_INTF_SEL_RMII); +} + +static void rk3568_set_gmac_speed(struct rk_priv_data *bsp_priv, int speed) +{ + struct device *dev = &bsp_priv->pdev->dev; + unsigned long rate; + int ret; + + switch (speed) { + case 10: + rate = 2500000; + break; + case 100: + rate = 25000000; + break; + case 1000: + rate = 125000000; + break; + default: + dev_err(dev, "unknown speed value for GMAC speed=%d", speed); + return; + } + + ret = clk_set_rate(bsp_priv->clk_mac_speed, rate); + if (ret) + dev_err(dev, "%s: set clk_mac_speed rate %ld failed %d\n", + __func__, rate, ret); +} + +static const struct rk_gmac_ops rk3568_ops = { + .set_to_rgmii = rk3568_set_to_rgmii, + .set_to_rmii = rk3568_set_to_rmii, + .set_rgmii_speed = rk3568_set_gmac_speed, + .set_rmii_speed = rk3568_set_gmac_speed, + .regs_valid = true, + .regs = { + 0xfe2a0000, /* gmac0 */ + 0xfe010000, /* gmac1 */ + 0x0, /* sentinel */ + }, +}; + +/* sys_grf */ +#define RK3588_GRF_GMAC_CON7 0X031c +#define RK3588_GRF_GMAC_CON8 0X0320 +#define RK3588_GRF_GMAC_CON9 0X0324 + +#define RK3588_GMAC_RXCLK_DLY_ENABLE(id) GRF_BIT(2 * (id) + 3) +#define RK3588_GMAC_RXCLK_DLY_DISABLE(id) GRF_CLR_BIT(2 * (id) + 3) +#define RK3588_GMAC_TXCLK_DLY_ENABLE(id) GRF_BIT(2 * (id) + 2) +#define RK3588_GMAC_TXCLK_DLY_DISABLE(id) GRF_CLR_BIT(2 * (id) + 2) + +#define RK3588_GMAC_CLK_RX_DL_CFG(val) HIWORD_UPDATE(val, 0xFF, 8) +#define RK3588_GMAC_CLK_TX_DL_CFG(val) HIWORD_UPDATE(val, 0xFF, 0) + +/* php_grf */ +#define RK3588_GRF_GMAC_CON0 0X0008 +#define RK3588_GRF_CLK_CON1 0X0070 + +#define RK3588_GMAC_PHY_INTF_SEL_RGMII(id) \ + (GRF_BIT(3 + (id) * 6) | GRF_CLR_BIT(4 + (id) * 6) | GRF_CLR_BIT(5 + (id) * 6)) +#define RK3588_GMAC_PHY_INTF_SEL_RMII(id) \ + (GRF_CLR_BIT(3 + (id) * 6) | GRF_CLR_BIT(4 + (id) * 6) | GRF_BIT(5 + (id) * 6)) + +#define RK3588_GMAC_CLK_RMII_MODE(id) GRF_BIT(5 * (id)) +#define RK3588_GMAC_CLK_RGMII_MODE(id) GRF_CLR_BIT(5 * (id)) + +#define RK3588_GMAC_CLK_SELET_CRU(id) GRF_BIT(5 * (id) + 4) +#define RK3588_GMAC_CLK_SELET_IO(id) GRF_CLR_BIT(5 * (id) + 4) + +#define RK3588_GMA_CLK_RMII_DIV2(id) GRF_BIT(5 * (id) + 2) +#define RK3588_GMA_CLK_RMII_DIV20(id) GRF_CLR_BIT(5 * (id) + 2) + +#define RK3588_GMAC_CLK_RGMII_DIV1(id) \ + (GRF_CLR_BIT(5 * (id) + 2) | GRF_CLR_BIT(5 * (id) + 3)) +#define RK3588_GMAC_CLK_RGMII_DIV5(id) \ + (GRF_BIT(5 * (id) + 2) | GRF_BIT(5 * (id) + 3)) +#define RK3588_GMAC_CLK_RGMII_DIV50(id) \ + (GRF_CLR_BIT(5 * (id) + 2) | GRF_BIT(5 * (id) + 3)) + +#define RK3588_GMAC_CLK_RMII_GATE(id) GRF_BIT(5 * (id) + 1) +#define RK3588_GMAC_CLK_RMII_NOGATE(id) GRF_CLR_BIT(5 * (id) + 1) + +static void rk3588_set_to_rgmii(struct rk_priv_data *bsp_priv, + int tx_delay, int rx_delay) +{ + struct device *dev = &bsp_priv->pdev->dev; + u32 offset_con, id = bsp_priv->id; + + if (IS_ERR(bsp_priv->grf) || IS_ERR(bsp_priv->php_grf)) { + dev_err(dev, "Missing rockchip,grf or rockchip,php_grf property\n"); + return; + } + + offset_con = bsp_priv->id == 1 ? RK3588_GRF_GMAC_CON9 : + RK3588_GRF_GMAC_CON8; + + regmap_write(bsp_priv->php_grf, RK3588_GRF_GMAC_CON0, + RK3588_GMAC_PHY_INTF_SEL_RGMII(id)); + + regmap_write(bsp_priv->php_grf, RK3588_GRF_CLK_CON1, + RK3588_GMAC_CLK_RGMII_MODE(id)); + + regmap_write(bsp_priv->grf, RK3588_GRF_GMAC_CON7, + RK3588_GMAC_RXCLK_DLY_ENABLE(id) | + RK3588_GMAC_TXCLK_DLY_ENABLE(id)); + + regmap_write(bsp_priv->grf, offset_con, + RK3588_GMAC_CLK_RX_DL_CFG(rx_delay) | + RK3588_GMAC_CLK_TX_DL_CFG(tx_delay)); +} + +static void rk3588_set_to_rmii(struct rk_priv_data *bsp_priv) +{ + struct device *dev = &bsp_priv->pdev->dev; + + if (IS_ERR(bsp_priv->php_grf)) { + dev_err(dev, "%s: Missing rockchip,php_grf property\n", __func__); + return; + } + + regmap_write(bsp_priv->php_grf, RK3588_GRF_GMAC_CON0, + RK3588_GMAC_PHY_INTF_SEL_RMII(bsp_priv->id)); + + regmap_write(bsp_priv->php_grf, RK3588_GRF_CLK_CON1, + RK3588_GMAC_CLK_RMII_MODE(bsp_priv->id)); +} + +static void rk3588_set_gmac_speed(struct rk_priv_data *bsp_priv, int speed) +{ + struct device *dev = &bsp_priv->pdev->dev; + unsigned int val = 0, id = bsp_priv->id; + + switch (speed) { + case 10: + if (bsp_priv->phy_iface == PHY_INTERFACE_MODE_RMII) + val = RK3588_GMA_CLK_RMII_DIV20(id); + else + val = RK3588_GMAC_CLK_RGMII_DIV50(id); + break; + case 100: + if (bsp_priv->phy_iface == PHY_INTERFACE_MODE_RMII) + val = RK3588_GMA_CLK_RMII_DIV2(id); + else + val = RK3588_GMAC_CLK_RGMII_DIV5(id); + break; + case 1000: + if (bsp_priv->phy_iface != PHY_INTERFACE_MODE_RMII) + val = RK3588_GMAC_CLK_RGMII_DIV1(id); + else + goto err; + break; + default: + goto err; + } + + regmap_write(bsp_priv->php_grf, RK3588_GRF_CLK_CON1, val); + + return; +err: + dev_err(dev, "unknown speed value for GMAC speed=%d", speed); +} + +static void rk3588_set_clock_selection(struct rk_priv_data *bsp_priv, bool input, + bool enable) +{ + unsigned int val = input ? RK3588_GMAC_CLK_SELET_IO(bsp_priv->id) : + RK3588_GMAC_CLK_SELET_CRU(bsp_priv->id); + + val |= enable ? RK3588_GMAC_CLK_RMII_NOGATE(bsp_priv->id) : + RK3588_GMAC_CLK_RMII_GATE(bsp_priv->id); + + regmap_write(bsp_priv->php_grf, RK3588_GRF_CLK_CON1, val); +} + +static const struct rk_gmac_ops rk3588_ops = { + .set_to_rgmii = rk3588_set_to_rgmii, + .set_to_rmii = rk3588_set_to_rmii, + .set_rgmii_speed = rk3588_set_gmac_speed, + .set_rmii_speed = rk3588_set_gmac_speed, + .set_clock_selection = rk3588_set_clock_selection, + .regs_valid = true, + .regs = { + 0xfe1b0000, /* gmac0 */ + 0xfe1c0000, /* gmac1 */ + 0x0, /* sentinel */ + }, +}; + #define RV1108_GRF_GMAC_CON0 0X0900 /* RV1108_GRF_GMAC_CON0 */ @@ -1000,6 +1303,130 @@ static const struct rk_gmac_ops rv1108_ops = { .set_rmii_speed = rv1108_set_rmii_speed, }; +#define RV1126_GRF_GMAC_CON0 0X0070 +#define RV1126_GRF_GMAC_CON1 0X0074 +#define RV1126_GRF_GMAC_CON2 0X0078 + +/* RV1126_GRF_GMAC_CON0 */ +#define RV1126_GMAC_PHY_INTF_SEL_RGMII \ + (GRF_BIT(4) | GRF_CLR_BIT(5) | GRF_CLR_BIT(6)) +#define RV1126_GMAC_PHY_INTF_SEL_RMII \ + (GRF_CLR_BIT(4) | GRF_CLR_BIT(5) | GRF_BIT(6)) +#define RV1126_GMAC_FLOW_CTRL GRF_BIT(7) +#define RV1126_GMAC_FLOW_CTRL_CLR GRF_CLR_BIT(7) +#define RV1126_GMAC_M0_RXCLK_DLY_ENABLE GRF_BIT(1) +#define RV1126_GMAC_M0_RXCLK_DLY_DISABLE GRF_CLR_BIT(1) +#define RV1126_GMAC_M0_TXCLK_DLY_ENABLE GRF_BIT(0) +#define RV1126_GMAC_M0_TXCLK_DLY_DISABLE GRF_CLR_BIT(0) +#define RV1126_GMAC_M1_RXCLK_DLY_ENABLE GRF_BIT(3) +#define RV1126_GMAC_M1_RXCLK_DLY_DISABLE GRF_CLR_BIT(3) +#define RV1126_GMAC_M1_TXCLK_DLY_ENABLE GRF_BIT(2) +#define RV1126_GMAC_M1_TXCLK_DLY_DISABLE GRF_CLR_BIT(2) + +/* RV1126_GRF_GMAC_CON1 */ +#define RV1126_GMAC_M0_CLK_RX_DL_CFG(val) HIWORD_UPDATE(val, 0x7F, 8) +#define RV1126_GMAC_M0_CLK_TX_DL_CFG(val) HIWORD_UPDATE(val, 0x7F, 0) +/* RV1126_GRF_GMAC_CON2 */ +#define RV1126_GMAC_M1_CLK_RX_DL_CFG(val) HIWORD_UPDATE(val, 0x7F, 8) +#define RV1126_GMAC_M1_CLK_TX_DL_CFG(val) HIWORD_UPDATE(val, 0x7F, 0) + +static void rv1126_set_to_rgmii(struct rk_priv_data *bsp_priv, + int tx_delay, int rx_delay) +{ + struct device *dev = &bsp_priv->pdev->dev; + + if (IS_ERR(bsp_priv->grf)) { + dev_err(dev, "Missing rockchip,grf property\n"); + return; + } + + regmap_write(bsp_priv->grf, RV1126_GRF_GMAC_CON0, + RV1126_GMAC_PHY_INTF_SEL_RGMII | + RV1126_GMAC_M0_RXCLK_DLY_ENABLE | + RV1126_GMAC_M0_TXCLK_DLY_ENABLE | + RV1126_GMAC_M1_RXCLK_DLY_ENABLE | + RV1126_GMAC_M1_TXCLK_DLY_ENABLE); + + regmap_write(bsp_priv->grf, RV1126_GRF_GMAC_CON1, + RV1126_GMAC_M0_CLK_RX_DL_CFG(rx_delay) | + RV1126_GMAC_M0_CLK_TX_DL_CFG(tx_delay)); + + regmap_write(bsp_priv->grf, RV1126_GRF_GMAC_CON2, + RV1126_GMAC_M1_CLK_RX_DL_CFG(rx_delay) | + RV1126_GMAC_M1_CLK_TX_DL_CFG(tx_delay)); +} + +static void rv1126_set_to_rmii(struct rk_priv_data *bsp_priv) +{ + struct device *dev = &bsp_priv->pdev->dev; + + if (IS_ERR(bsp_priv->grf)) { + dev_err(dev, "%s: Missing rockchip,grf property\n", __func__); + return; + } + + regmap_write(bsp_priv->grf, RV1126_GRF_GMAC_CON0, + RV1126_GMAC_PHY_INTF_SEL_RMII); +} + +static void rv1126_set_rgmii_speed(struct rk_priv_data *bsp_priv, int speed) +{ + struct device *dev = &bsp_priv->pdev->dev; + unsigned long rate; + int ret; + + switch (speed) { + case 10: + rate = 2500000; + break; + case 100: + rate = 25000000; + break; + case 1000: + rate = 125000000; + break; + default: + dev_err(dev, "unknown speed value for RGMII speed=%d", speed); + return; + } + + ret = clk_set_rate(bsp_priv->clk_mac_speed, rate); + if (ret) + dev_err(dev, "%s: set clk_mac_speed rate %ld failed %d\n", + __func__, rate, ret); +} + +static void rv1126_set_rmii_speed(struct rk_priv_data *bsp_priv, int speed) +{ + struct device *dev = &bsp_priv->pdev->dev; + unsigned long rate; + int ret; + + switch (speed) { + case 10: + rate = 2500000; + break; + case 100: + rate = 25000000; + break; + default: + dev_err(dev, "unknown speed value for RGMII speed=%d", speed); + return; + } + + ret = clk_set_rate(bsp_priv->clk_mac_speed, rate); + if (ret) + dev_err(dev, "%s: set clk_mac_speed rate %ld failed %d\n", + __func__, rate, ret); +} + +static const struct rk_gmac_ops rv1126_ops = { + .set_to_rgmii = rv1126_set_to_rgmii, + .set_to_rmii = rv1126_set_to_rmii, + .set_rgmii_speed = rv1126_set_rgmii_speed, + .set_rmii_speed = rv1126_set_rmii_speed, +}; + #define RK_GRF_MACPHY_CON0 0xb00 #define RK_GRF_MACPHY_CON1 0xb04 #define RK_GRF_MACPHY_CON2 0xb08 @@ -1151,6 +1578,10 @@ static int gmac_clk_enable(struct rk_priv_data *bsp_priv, bool enable) if (!IS_ERR(bsp_priv->clk_mac_speed)) clk_prepare_enable(bsp_priv->clk_mac_speed); + if (bsp_priv->ops && bsp_priv->ops->set_clock_selection) + bsp_priv->ops->set_clock_selection(bsp_priv, + bsp_priv->clock_input, true); + /** * if (!IS_ERR(bsp_priv->clk_mac)) * clk_prepare_enable(bsp_priv->clk_mac); @@ -1177,6 +1608,10 @@ static int gmac_clk_enable(struct rk_priv_data *bsp_priv, bool enable) clk_disable_unprepare(bsp_priv->mac_clk_tx); clk_disable_unprepare(bsp_priv->clk_mac_speed); + + if (bsp_priv->ops && bsp_priv->ops->set_clock_selection) + bsp_priv->ops->set_clock_selection(bsp_priv, + bsp_priv->clock_input, false); /** * if (!IS_ERR(bsp_priv->clk_mac)) * clk_disable_unprepare(bsp_priv->clk_mac); @@ -1216,6 +1651,7 @@ static struct rk_priv_data *rk_gmac_setup(struct platform_device *pdev, { struct rk_priv_data *bsp_priv; struct device *dev = &pdev->dev; + struct resource *res; int ret; const char *strings = NULL; int value; @@ -1227,6 +1663,22 @@ static struct rk_priv_data *rk_gmac_setup(struct platform_device *pdev, of_get_phy_mode(dev->of_node, &bsp_priv->phy_iface); bsp_priv->ops = ops; + /* Some SoCs have multiple MAC controllers, which need + * to be distinguished. + */ + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (res && ops->regs_valid) { + int i = 0; + + while (ops->regs[i]) { + if (ops->regs[i] == res->start) { + bsp_priv->id = i; + break; + } + i++; + } + } + bsp_priv->regulator = devm_regulator_get_optional(dev, "phy"); if (IS_ERR(bsp_priv->regulator)) { if (PTR_ERR(bsp_priv->regulator) == -EPROBE_DEFER) { @@ -1274,6 +1726,8 @@ static struct rk_priv_data *rk_gmac_setup(struct platform_device *pdev, bsp_priv->grf = syscon_regmap_lookup_by_phandle(dev->of_node, "rockchip,grf"); + bsp_priv->php_grf = syscon_regmap_lookup_by_phandle(dev->of_node, + "rockchip,php-grf"); if (plat->phy_node) { bsp_priv->integrated_phy = of_property_read_bool(plat->phy_node, @@ -1294,11 +1748,36 @@ static struct rk_priv_data *rk_gmac_setup(struct platform_device *pdev, return bsp_priv; } +static int rk_gmac_check_ops(struct rk_priv_data *bsp_priv) +{ + switch (bsp_priv->phy_iface) { + case PHY_INTERFACE_MODE_RGMII: + case PHY_INTERFACE_MODE_RGMII_ID: + case PHY_INTERFACE_MODE_RGMII_RXID: + case PHY_INTERFACE_MODE_RGMII_TXID: + if (!bsp_priv->ops->set_to_rgmii) + return -EINVAL; + break; + case PHY_INTERFACE_MODE_RMII: + if (!bsp_priv->ops->set_to_rmii) + return -EINVAL; + break; + default: + dev_err(&bsp_priv->pdev->dev, + "unsupported interface %d", bsp_priv->phy_iface); + } + return 0; +} + static int rk_gmac_powerup(struct rk_priv_data *bsp_priv) { int ret; struct device *dev = &bsp_priv->pdev->dev; + ret = rk_gmac_check_ops(bsp_priv); + if (ret) + return ret; + ret = gmac_clk_enable(bsp_priv, true); if (ret) return ret; @@ -1336,7 +1815,6 @@ static int rk_gmac_powerup(struct rk_priv_data *bsp_priv) return ret; } - pm_runtime_enable(dev); pm_runtime_get_sync(dev); if (bsp_priv->integrated_phy) @@ -1347,13 +1825,10 @@ static int rk_gmac_powerup(struct rk_priv_data *bsp_priv) static void rk_gmac_powerdown(struct rk_priv_data *gmac) { - struct device *dev = &gmac->pdev->dev; - if (gmac->integrated_phy) rk_gmac_integrated_phy_powerdown(gmac); - pm_runtime_put_sync(dev); - pm_runtime_disable(dev); + pm_runtime_put_sync(&gmac->pdev->dev); phy_power_on(gmac, false); gmac_clk_enable(gmac, false); @@ -1369,10 +1844,12 @@ static void rk_fix_speed(void *priv, unsigned int speed) case PHY_INTERFACE_MODE_RGMII_ID: case PHY_INTERFACE_MODE_RGMII_RXID: case PHY_INTERFACE_MODE_RGMII_TXID: - bsp_priv->ops->set_rgmii_speed(bsp_priv, speed); + if (bsp_priv->ops->set_rgmii_speed) + bsp_priv->ops->set_rgmii_speed(bsp_priv, speed); break; case PHY_INTERFACE_MODE_RMII: - bsp_priv->ops->set_rmii_speed(bsp_priv, speed); + if (bsp_priv->ops->set_rmii_speed) + bsp_priv->ops->set_rmii_speed(bsp_priv, speed); break; default: dev_err(dev, "unsupported interface %d", bsp_priv->phy_iface); @@ -1396,11 +1873,15 @@ static int rk_gmac_probe(struct platform_device *pdev) if (ret) return ret; - plat_dat = stmmac_probe_config_dt(pdev, &stmmac_res.mac); + plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac); if (IS_ERR(plat_dat)) return PTR_ERR(plat_dat); - plat_dat->has_gmac = true; + /* If the stmmac is not already selected as gmac4, + * then make sure we fallback to gmac. + */ + if (!plat_dat->has_gmac4) + plat_dat->has_gmac = true; plat_dat->fix_mac_speed = rk_fix_speed; plat_dat->bsp_priv = rk_gmac_setup(pdev, plat_dat, data); @@ -1411,7 +1892,7 @@ static int rk_gmac_probe(struct platform_device *pdev) ret = rk_gmac_clk_init(plat_dat); if (ret) - return ret; + goto err_remove_config_dt; ret = rk_gmac_powerup(plat_dat->bsp_priv); if (ret) @@ -1477,11 +1958,15 @@ static const struct of_device_id rk_gmac_dwmac_match[] = { { .compatible = "rockchip,rk3128-gmac", .data = &rk3128_ops }, { .compatible = "rockchip,rk3228-gmac", .data = &rk3228_ops }, { .compatible = "rockchip,rk3288-gmac", .data = &rk3288_ops }, + { .compatible = "rockchip,rk3308-gmac", .data = &rk3308_ops }, { .compatible = "rockchip,rk3328-gmac", .data = &rk3328_ops }, { .compatible = "rockchip,rk3366-gmac", .data = &rk3366_ops }, { .compatible = "rockchip,rk3368-gmac", .data = &rk3368_ops }, { .compatible = "rockchip,rk3399-gmac", .data = &rk3399_ops }, + { .compatible = "rockchip,rk3568-gmac", .data = &rk3568_ops }, + { .compatible = "rockchip,rk3588-gmac", .data = &rk3588_ops }, { .compatible = "rockchip,rv1108-gmac", .data = &rv1108_ops }, + { .compatible = "rockchip,rv1126-gmac", .data = &rv1126_ops }, { } }; MODULE_DEVICE_TABLE(of, rk_gmac_dwmac_match); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c index e0212d2fc2a1..6b447d8f0bd8 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c @@ -18,9 +18,6 @@ #include "altr_tse_pcs.h" -#define SGMII_ADAPTER_CTRL_REG 0x00 -#define SGMII_ADAPTER_DISABLE 0x0001 - #define SYSMGR_EMACGRP_CTRL_PHYSEL_ENUM_GMII_MII 0x0 #define SYSMGR_EMACGRP_CTRL_PHYSEL_ENUM_RGMII 0x1 #define SYSMGR_EMACGRP_CTRL_PHYSEL_ENUM_RMII 0x2 @@ -62,14 +59,13 @@ static void socfpga_dwmac_fix_mac_speed(void *priv, unsigned int speed) { struct socfpga_dwmac *dwmac = (struct socfpga_dwmac *)priv; void __iomem *splitter_base = dwmac->splitter_base; - void __iomem *tse_pcs_base = dwmac->pcs.tse_pcs_base; void __iomem *sgmii_adapter_base = dwmac->pcs.sgmii_adapter_base; struct device *dev = dwmac->dev; struct net_device *ndev = dev_get_drvdata(dev); struct phy_device *phy_dev = ndev->phydev; u32 val; - if ((tse_pcs_base) && (sgmii_adapter_base)) + if (sgmii_adapter_base) writew(SGMII_ADAPTER_DISABLE, sgmii_adapter_base + SGMII_ADAPTER_CTRL_REG); @@ -93,8 +89,11 @@ static void socfpga_dwmac_fix_mac_speed(void *priv, unsigned int speed) writel(val, splitter_base + EMAC_SPLITTER_CTRL_REG); } - if (tse_pcs_base && sgmii_adapter_base) + if (phy_dev && sgmii_adapter_base) { + writew(SGMII_ADAPTER_ENABLE, + sgmii_adapter_base + SGMII_ADAPTER_CTRL_REG); tse_pcs_fix_mac_speed(&dwmac->pcs, phy_dev, speed); + } } static int socfpga_dwmac_parse_data(struct socfpga_dwmac *dwmac, struct device *dev) @@ -241,6 +240,8 @@ static int socfpga_set_phy_mode_common(int phymode, u32 *val) switch (phymode) { case PHY_INTERFACE_MODE_RGMII: case PHY_INTERFACE_MODE_RGMII_ID: + case PHY_INTERFACE_MODE_RGMII_RXID: + case PHY_INTERFACE_MODE_RGMII_TXID: *val = SYSMGR_EMACGRP_CTRL_PHYSEL_ENUM_RGMII; break; case PHY_INTERFACE_MODE_MII: @@ -289,16 +290,19 @@ static int socfpga_gen5_set_phy_mode(struct socfpga_dwmac *dwmac) phymode == PHY_INTERFACE_MODE_MII || phymode == PHY_INTERFACE_MODE_GMII || phymode == PHY_INTERFACE_MODE_SGMII) { - ctrl |= SYSMGR_EMACGRP_CTRL_PTP_REF_CLK_MASK << (reg_shift / 2); regmap_read(sys_mgr_base_addr, SYSMGR_FPGAGRP_MODULE_REG, &module); module |= (SYSMGR_FPGAGRP_MODULE_EMAC << (reg_shift / 2)); regmap_write(sys_mgr_base_addr, SYSMGR_FPGAGRP_MODULE_REG, module); - } else { - ctrl &= ~(SYSMGR_EMACGRP_CTRL_PTP_REF_CLK_MASK << (reg_shift / 2)); } + if (dwmac->f2h_ptp_ref_clk) + ctrl |= SYSMGR_EMACGRP_CTRL_PTP_REF_CLK_MASK << (reg_shift / 2); + else + ctrl &= ~(SYSMGR_EMACGRP_CTRL_PTP_REF_CLK_MASK << + (reg_shift / 2)); + regmap_write(sys_mgr_base_addr, reg_offset, ctrl); /* Deassert reset for the phy configuration to be sampled by @@ -393,7 +397,7 @@ static int socfpga_dwmac_probe(struct platform_device *pdev) if (ret) return ret; - plat_dat = stmmac_probe_config_dt(pdev, &stmmac_res.mac); + plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac); if (IS_ERR(plat_dat)) return PTR_ERR(plat_dat); @@ -480,8 +484,28 @@ static int socfpga_dwmac_resume(struct device *dev) } #endif /* CONFIG_PM_SLEEP */ -static SIMPLE_DEV_PM_OPS(socfpga_dwmac_pm_ops, stmmac_suspend, - socfpga_dwmac_resume); +static int __maybe_unused socfpga_dwmac_runtime_suspend(struct device *dev) +{ + struct net_device *ndev = dev_get_drvdata(dev); + struct stmmac_priv *priv = netdev_priv(ndev); + + stmmac_bus_clks_config(priv, false); + + return 0; +} + +static int __maybe_unused socfpga_dwmac_runtime_resume(struct device *dev) +{ + struct net_device *ndev = dev_get_drvdata(dev); + struct stmmac_priv *priv = netdev_priv(ndev); + + return stmmac_bus_clks_config(priv, true); +} + +static const struct dev_pm_ops socfpga_dwmac_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(stmmac_suspend, socfpga_dwmac_resume) + SET_RUNTIME_PM_OPS(socfpga_dwmac_runtime_suspend, socfpga_dwmac_runtime_resume, NULL) +}; static const struct socfpga_dwmac_ops socfpga_gen5_ops = { .set_phy_mode = socfpga_gen5_set_phy_mode, diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sti.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sti.c index e1b63df6f96f..710d7435733e 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sti.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sti.c @@ -325,7 +325,7 @@ static int sti_dwmac_probe(struct platform_device *pdev) if (ret) return ret; - plat_dat = stmmac_probe_config_dt(pdev, &stmmac_res.mac); + plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac); if (IS_ERR(plat_dat)) return PTR_ERR(plat_dat); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-stm32.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-stm32.c index 9b7be996d07b..2b38a499a404 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-stm32.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-stm32.c @@ -29,6 +29,11 @@ #define SYSCFG_PMCR_ETH_CLK_SEL BIT(16) #define SYSCFG_PMCR_ETH_REF_CLK_SEL BIT(17) +/* CLOCK feed to PHY*/ +#define ETH_CK_F_25M 25000000 +#define ETH_CK_F_50M 50000000 +#define ETH_CK_F_125M 125000000 + /* Ethernet PHY interface selection in register SYSCFG Configuration *------------------------------------------ * src |BIT(23)| BIT(22)| BIT(21)|BIT(20)| @@ -58,33 +63,20 @@ *| | | 25MHz | 50MHz | | * --------------------------------------------------------------------------- *| MII | - | eth-ck | n/a | n/a | - *| | | | | | + *| | | st,ext-phyclk | | | * --------------------------------------------------------------------------- *| GMII | - | eth-ck | n/a | n/a | - *| | | | | | + *| | | st,ext-phyclk | | | * --------------------------------------------------------------------------- - *| RGMII | - | eth-ck | n/a | eth-ck (no pin) | - *| | | | | st,eth-clk-sel | + *| RGMII | - | eth-ck | n/a | eth-ck | + *| | | st,ext-phyclk | | st,eth-clk-sel or| + *| | | | | st,ext-phyclk | * --------------------------------------------------------------------------- *| RMII | - | eth-ck | eth-ck | n/a | - *| | | | st,eth-ref-clk-sel | | + *| | | st,ext-phyclk | st,eth-ref-clk-sel | | + *| | | | or st,ext-phyclk | | * --------------------------------------------------------------------------- * - * BIT(17) : set this bit in RMII mode when you have PHY without crystal 50MHz - * BIT(16) : set this bit in GMII/RGMII PHY when you do not want use 125Mhz - * from PHY - *----------------------------------------------------- - * src | BIT(17) | BIT(16) | - *----------------------------------------------------- - * MII | n/a | n/a | - *----------------------------------------------------- - * GMII | n/a | st,eth-clk-sel | - *----------------------------------------------------- - * RGMII | n/a | st,eth-clk-sel | - *----------------------------------------------------- - * RMII | st,eth-ref-clk-sel | n/a | - *----------------------------------------------------- - * */ struct stm32_dwmac { @@ -93,6 +85,8 @@ struct stm32_dwmac { struct clk *clk_eth_ck; struct clk *clk_ethstp; struct clk *syscfg_clk; + int ext_phyclk; + int enable_eth_ck; int eth_clk_sel_reg; int eth_ref_clk_sel_reg; int irq_pwr_wakeup; @@ -155,14 +149,17 @@ static int stm32mp1_clk_prepare(struct stm32_dwmac *dwmac, bool prepare) ret = clk_prepare_enable(dwmac->syscfg_clk); if (ret) return ret; - ret = clk_prepare_enable(dwmac->clk_eth_ck); - if (ret) { - clk_disable_unprepare(dwmac->syscfg_clk); - return ret; + if (dwmac->enable_eth_ck) { + ret = clk_prepare_enable(dwmac->clk_eth_ck); + if (ret) { + clk_disable_unprepare(dwmac->syscfg_clk); + return ret; + } } } else { clk_disable_unprepare(dwmac->syscfg_clk); - clk_disable_unprepare(dwmac->clk_eth_ck); + if (dwmac->enable_eth_ck) + clk_disable_unprepare(dwmac->clk_eth_ck); } return ret; } @@ -170,24 +167,34 @@ static int stm32mp1_clk_prepare(struct stm32_dwmac *dwmac, bool prepare) static int stm32mp1_set_mode(struct plat_stmmacenet_data *plat_dat) { struct stm32_dwmac *dwmac = plat_dat->bsp_priv; - u32 reg = dwmac->mode_reg; + u32 reg = dwmac->mode_reg, clk_rate; int val; + clk_rate = clk_get_rate(dwmac->clk_eth_ck); + dwmac->enable_eth_ck = false; switch (plat_dat->interface) { case PHY_INTERFACE_MODE_MII: + if (clk_rate == ETH_CK_F_25M && dwmac->ext_phyclk) + dwmac->enable_eth_ck = true; val = SYSCFG_PMCR_ETH_SEL_MII; pr_debug("SYSCFG init : PHY_INTERFACE_MODE_MII\n"); break; case PHY_INTERFACE_MODE_GMII: val = SYSCFG_PMCR_ETH_SEL_GMII; - if (dwmac->eth_clk_sel_reg) + if (clk_rate == ETH_CK_F_25M && + (dwmac->eth_clk_sel_reg || dwmac->ext_phyclk)) { + dwmac->enable_eth_ck = true; val |= SYSCFG_PMCR_ETH_CLK_SEL; + } pr_debug("SYSCFG init : PHY_INTERFACE_MODE_GMII\n"); break; case PHY_INTERFACE_MODE_RMII: val = SYSCFG_PMCR_ETH_SEL_RMII; - if (dwmac->eth_ref_clk_sel_reg) + if ((clk_rate == ETH_CK_F_25M || clk_rate == ETH_CK_F_50M) && + (dwmac->eth_ref_clk_sel_reg || dwmac->ext_phyclk)) { + dwmac->enable_eth_ck = true; val |= SYSCFG_PMCR_ETH_REF_CLK_SEL; + } pr_debug("SYSCFG init : PHY_INTERFACE_MODE_RMII\n"); break; case PHY_INTERFACE_MODE_RGMII: @@ -195,8 +202,11 @@ static int stm32mp1_set_mode(struct plat_stmmacenet_data *plat_dat) case PHY_INTERFACE_MODE_RGMII_RXID: case PHY_INTERFACE_MODE_RGMII_TXID: val = SYSCFG_PMCR_ETH_SEL_RGMII; - if (dwmac->eth_clk_sel_reg) + if ((clk_rate == ETH_CK_F_25M || clk_rate == ETH_CK_F_125M) && + (dwmac->eth_clk_sel_reg || dwmac->ext_phyclk)) { + dwmac->enable_eth_ck = true; val |= SYSCFG_PMCR_ETH_CLK_SEL; + } pr_debug("SYSCFG init : PHY_INTERFACE_MODE_RGMII\n"); break; default: @@ -294,6 +304,9 @@ static int stm32mp1_parse_data(struct stm32_dwmac *dwmac, struct device_node *np = dev->of_node; int err = 0; + /* Ethernet PHY have no crystal */ + dwmac->ext_phyclk = of_property_read_bool(np, "st,ext-phyclk"); + /* Gigabit Ethernet 125MHz clock selection. */ dwmac->eth_clk_sel_reg = of_property_read_bool(np, "st,eth-clk-sel"); @@ -304,7 +317,7 @@ static int stm32mp1_parse_data(struct stm32_dwmac *dwmac, /* Get ETH_CLK clocks */ dwmac->clk_eth_ck = devm_clk_get(dev, "eth-ck"); if (IS_ERR(dwmac->clk_eth_ck)) { - dev_warn(dev, "No phy clock provided...\n"); + dev_info(dev, "No phy clock provided...\n"); dwmac->clk_eth_ck = NULL; } @@ -324,7 +337,7 @@ static int stm32mp1_parse_data(struct stm32_dwmac *dwmac, /* Get IRQ information early to have an ability to ask for deferred * probe if needed before we went too far with resource allocation. */ - dwmac->irq_pwr_wakeup = platform_get_irq_byname(pdev, + dwmac->irq_pwr_wakeup = platform_get_irq_byname_optional(pdev, "stm32_pwr_wakeup"); if (dwmac->irq_pwr_wakeup == -EPROBE_DEFER) return -EPROBE_DEFER; @@ -358,7 +371,7 @@ static int stm32_dwmac_probe(struct platform_device *pdev) if (ret) return ret; - plat_dat = stmmac_probe_config_dt(pdev, &stmmac_res.mac); + plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac); if (IS_ERR(plat_dat)) return PTR_ERR(plat_dat); @@ -431,7 +444,8 @@ static int stm32mp1_suspend(struct stm32_dwmac *dwmac) clk_disable_unprepare(dwmac->clk_tx); clk_disable_unprepare(dwmac->syscfg_clk); - clk_disable_unprepare(dwmac->clk_eth_ck); + if (dwmac->enable_eth_ck) + clk_disable_unprepare(dwmac->clk_eth_ck); return ret; } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c index 58e0511badba..f834472599f7 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c @@ -16,6 +16,7 @@ #include <linux/of_net.h> #include <linux/phy.h> #include <linux/platform_device.h> +#include <linux/pm_runtime.h> #include <linux/regulator/consumer.h> #include <linux/regmap.h> #include <linux/stmmac.h> @@ -57,23 +58,23 @@ struct emac_variant { }; /* struct sunxi_priv_data - hold all sunxi private data - * @tx_clk: reference to MAC TX clock * @ephy_clk: reference to the optional EPHY clock for the internal PHY * @regulator: reference to the optional regulator * @rst_ephy: reference to the optional EPHY reset for the internal PHY * @variant: reference to the current board variant * @regmap: regmap for using the syscon * @internal_phy_powered: Does the internal PHY is enabled + * @use_internal_phy: Is the internal PHY selected for use * @mux_handle: Internal pointer used by mdio-mux lib */ struct sunxi_priv_data { - struct clk *tx_clk; struct clk *ephy_clk; struct regulator *regulator; struct reset_control *rst_ephy; const struct emac_variant *variant; struct regmap_field *regmap_field; bool internal_phy_powered; + bool use_internal_phy; void *mux_handle; }; @@ -237,6 +238,22 @@ static const struct emac_variant emac_variant_h6 = { #define EMAC_RX_EARLY_INT BIT(13) #define EMAC_RGMII_STA_INT BIT(16) +#define EMAC_INT_MSK_COMMON EMAC_RGMII_STA_INT +#define EMAC_INT_MSK_TX (EMAC_TX_INT | \ + EMAC_TX_DMA_STOP_INT | \ + EMAC_TX_BUF_UA_INT | \ + EMAC_TX_TIMEOUT_INT | \ + EMAC_TX_UNDERFLOW_INT | \ + EMAC_TX_EARLY_INT |\ + EMAC_INT_MSK_COMMON) +#define EMAC_INT_MSK_RX (EMAC_RX_INT | \ + EMAC_RX_BUF_UA_INT | \ + EMAC_RX_DMA_STOP_INT | \ + EMAC_RX_TIMEOUT_INT | \ + EMAC_RX_OVERFLOW_INT | \ + EMAC_RX_EARLY_INT | \ + EMAC_INT_MSK_COMMON) + #define MAC_ADDR_TYPE_DST BIT(31) /* H3 specific bits for EPHY */ @@ -410,13 +427,19 @@ static void sun8i_dwmac_dma_stop_rx(void __iomem *ioaddr, u32 chan) } static int sun8i_dwmac_dma_interrupt(void __iomem *ioaddr, - struct stmmac_extra_stats *x, u32 chan) + struct stmmac_extra_stats *x, u32 chan, + u32 dir) { u32 v; int ret = 0; v = readl(ioaddr + EMAC_INT_STA); + if (dir == DMA_DIR_RX) + v &= EMAC_INT_MSK_RX; + else if (dir == DMA_DIR_TX) + v &= EMAC_INT_MSK_TX; + if (v & EMAC_TX_INT) { ret |= handle_tx; x->tx_normal_irq_n++; @@ -539,8 +562,11 @@ static const struct stmmac_dma_ops sun8i_dwmac_dma_ops = { .dma_interrupt = sun8i_dwmac_dma_interrupt, }; +static int sun8i_dwmac_power_internal_phy(struct stmmac_priv *priv); + static int sun8i_dwmac_init(struct platform_device *pdev, void *priv) { + struct net_device *ndev = platform_get_drvdata(pdev); struct sunxi_priv_data *gmac = priv; int ret; @@ -552,15 +578,19 @@ static int sun8i_dwmac_init(struct platform_device *pdev, void *priv) } } - ret = clk_prepare_enable(gmac->tx_clk); - if (ret) { - if (gmac->regulator) - regulator_disable(gmac->regulator); - dev_err(&pdev->dev, "Could not enable AHB clock\n"); - return ret; + if (gmac->use_internal_phy) { + ret = sun8i_dwmac_power_internal_phy(netdev_priv(ndev)); + if (ret) + goto err_disable_regulator; } return 0; + +err_disable_regulator: + if (gmac->regulator) + regulator_disable(gmac->regulator); + + return ret; } static void sun8i_dwmac_core_init(struct mac_device_info *hw, @@ -595,7 +625,7 @@ static void sun8i_dwmac_set_mac(void __iomem *ioaddr, bool enable) * If addr is NULL, clear the slot */ static void sun8i_dwmac_set_umac_addr(struct mac_device_info *hw, - unsigned char *addr, + const unsigned char *addr, unsigned int reg_n) { void __iomem *ioaddr = hw->pcsr; @@ -717,7 +747,7 @@ static int sun8i_dwmac_reset(struct stmmac_priv *priv) if (err) { dev_err(priv->device, "EMAC reset timeout\n"); - return -EFAULT; + return err; } return 0; } @@ -788,12 +818,12 @@ static int sun8i_dwmac_power_internal_phy(struct stmmac_priv *priv) /* Make sure the EPHY is properly reseted, as U-Boot may leave * it at deasserted state, and thus it may fail to reset EMAC. + * + * This assumes the driver has exclusive access to the EPHY reset. */ - reset_control_assert(gmac->rst_ephy); - - ret = reset_control_deassert(gmac->rst_ephy); + ret = reset_control_reset(gmac->rst_ephy); if (ret) { - dev_err(priv->device, "Cannot deassert internal phy\n"); + dev_err(priv->device, "Cannot reset internal PHY\n"); clk_disable_unprepare(gmac->ephy_clk); return ret; } @@ -803,15 +833,14 @@ static int sun8i_dwmac_power_internal_phy(struct stmmac_priv *priv) return 0; } -static int sun8i_dwmac_unpower_internal_phy(struct sunxi_priv_data *gmac) +static void sun8i_dwmac_unpower_internal_phy(struct sunxi_priv_data *gmac) { if (!gmac->internal_phy_powered) - return 0; + return; clk_disable_unprepare(gmac->ephy_clk); reset_control_assert(gmac->rst_ephy); gmac->internal_phy_powered = false; - return 0; } /* MDIO multiplexing switch function @@ -831,7 +860,6 @@ static int mdio_mux_syscon_switch_fn(int current_child, int desired_child, struct sunxi_priv_data *gmac = priv->plat->bsp_priv; u32 reg, val; int ret = 0; - bool need_power_ephy = false; if (current_child ^ desired_child) { regmap_field_read(gmac->regmap_field, ®); @@ -839,13 +867,12 @@ static int mdio_mux_syscon_switch_fn(int current_child, int desired_child, case DWMAC_SUN8I_MDIO_MUX_INTERNAL_ID: dev_info(priv->device, "Switch mux to internal PHY"); val = (reg & ~H3_EPHY_MUX_MASK) | H3_EPHY_SELECT; - - need_power_ephy = true; + gmac->use_internal_phy = true; break; case DWMAC_SUN8I_MDIO_MUX_EXTERNAL_ID: dev_info(priv->device, "Switch mux to external PHY"); val = (reg & ~H3_EPHY_MUX_MASK) | H3_EPHY_SHUTDOWN; - need_power_ephy = false; + gmac->use_internal_phy = false; break; default: dev_err(priv->device, "Invalid child ID %x\n", @@ -853,7 +880,7 @@ static int mdio_mux_syscon_switch_fn(int current_child, int desired_child, return -EINVAL; } regmap_field_write(gmac->regmap_field, val); - if (need_power_ephy) { + if (gmac->use_internal_phy) { ret = sun8i_dwmac_power_internal_phy(priv); if (ret) return ret; @@ -880,25 +907,27 @@ static int sun8i_dwmac_register_mdio_mux(struct stmmac_priv *priv) ret = mdio_mux_init(priv->device, mdio_mux, mdio_mux_syscon_switch_fn, &gmac->mux_handle, priv, priv->mii); + of_node_put(mdio_mux); return ret; } -static int sun8i_dwmac_set_syscon(struct stmmac_priv *priv) +static int sun8i_dwmac_set_syscon(struct device *dev, + struct plat_stmmacenet_data *plat) { - struct sunxi_priv_data *gmac = priv->plat->bsp_priv; - struct device_node *node = priv->device->of_node; + struct sunxi_priv_data *gmac = plat->bsp_priv; + struct device_node *node = dev->of_node; int ret; u32 reg, val; ret = regmap_field_read(gmac->regmap_field, &val); if (ret) { - dev_err(priv->device, "Fail to read from regmap field.\n"); + dev_err(dev, "Fail to read from regmap field.\n"); return ret; } reg = gmac->variant->default_syscon_value; if (reg != val) - dev_warn(priv->device, + dev_warn(dev, "Current syscon value is not the default %x (expect %x)\n", val, reg); @@ -911,9 +940,9 @@ static int sun8i_dwmac_set_syscon(struct stmmac_priv *priv) /* Force EPHY xtal frequency to 24MHz. */ reg |= H3_EPHY_CLK_SEL; - ret = of_mdio_parse_addr(priv->device, priv->plat->phy_node); + ret = of_mdio_parse_addr(dev, plat->phy_node); if (ret < 0) { - dev_err(priv->device, "Could not parse MDIO addr\n"); + dev_err(dev, "Could not parse MDIO addr\n"); return ret; } /* of_mdio_parse_addr returns a valid (0 ~ 31) PHY @@ -929,17 +958,17 @@ static int sun8i_dwmac_set_syscon(struct stmmac_priv *priv) if (!of_property_read_u32(node, "allwinner,tx-delay-ps", &val)) { if (val % 100) { - dev_err(priv->device, "tx-delay must be a multiple of 100\n"); + dev_err(dev, "tx-delay must be a multiple of 100\n"); return -EINVAL; } val /= 100; - dev_dbg(priv->device, "set tx-delay to %x\n", val); + dev_dbg(dev, "set tx-delay to %x\n", val); if (val <= gmac->variant->tx_delay_max) { reg &= ~(gmac->variant->tx_delay_max << SYSCON_ETXDC_SHIFT); reg |= (val << SYSCON_ETXDC_SHIFT); } else { - dev_err(priv->device, "Invalid TX clock delay: %d\n", + dev_err(dev, "Invalid TX clock delay: %d\n", val); return -EINVAL; } @@ -947,17 +976,17 @@ static int sun8i_dwmac_set_syscon(struct stmmac_priv *priv) if (!of_property_read_u32(node, "allwinner,rx-delay-ps", &val)) { if (val % 100) { - dev_err(priv->device, "rx-delay must be a multiple of 100\n"); + dev_err(dev, "rx-delay must be a multiple of 100\n"); return -EINVAL; } val /= 100; - dev_dbg(priv->device, "set rx-delay to %x\n", val); + dev_dbg(dev, "set rx-delay to %x\n", val); if (val <= gmac->variant->rx_delay_max) { reg &= ~(gmac->variant->rx_delay_max << SYSCON_ERXDC_SHIFT); reg |= (val << SYSCON_ERXDC_SHIFT); } else { - dev_err(priv->device, "Invalid RX clock delay: %d\n", + dev_err(dev, "Invalid RX clock delay: %d\n", val); return -EINVAL; } @@ -968,7 +997,7 @@ static int sun8i_dwmac_set_syscon(struct stmmac_priv *priv) if (gmac->variant->support_rmii) reg &= ~SYSCON_RMII_EN; - switch (priv->plat->interface) { + switch (plat->interface) { case PHY_INTERFACE_MODE_MII: /* default */ break; @@ -982,8 +1011,8 @@ static int sun8i_dwmac_set_syscon(struct stmmac_priv *priv) reg |= SYSCON_RMII_EN | SYSCON_ETCS_EXT_GMII; break; default: - dev_err(priv->device, "Unsupported interface mode: %s", - phy_modes(priv->plat->interface)); + dev_err(dev, "Unsupported interface mode: %s", + phy_modes(plat->interface)); return -EINVAL; } @@ -1003,19 +1032,8 @@ static void sun8i_dwmac_exit(struct platform_device *pdev, void *priv) { struct sunxi_priv_data *gmac = priv; - if (gmac->variant->soc_has_internal_phy) { - /* sun8i_dwmac_exit could be called with mdiomux uninit */ - if (gmac->mux_handle) - mdio_mux_uninit(gmac->mux_handle); - if (gmac->internal_phy_powered) - sun8i_dwmac_unpower_internal_phy(gmac); - } - - sun8i_dwmac_unset_syscon(gmac); - - reset_control_put(gmac->rst_ephy); - - clk_disable_unprepare(gmac->tx_clk); + if (gmac->variant->soc_has_internal_phy) + sun8i_dwmac_unpower_internal_phy(gmac); if (gmac->regulator) regulator_disable(gmac->regulator); @@ -1049,16 +1067,11 @@ static struct mac_device_info *sun8i_dwmac_setup(void *ppriv) { struct mac_device_info *mac; struct stmmac_priv *priv = ppriv; - int ret; mac = devm_kzalloc(priv->device, sizeof(*mac), GFP_KERNEL); if (!mac) return NULL; - ret = sun8i_dwmac_set_syscon(priv); - if (ret) - return NULL; - mac->pcsr = priv->ioaddr; mac->mac = &sun8i_dwmac_ops; mac->dma = &sun8i_dwmac_dma_ops; @@ -1134,10 +1147,6 @@ static int sun8i_dwmac_probe(struct platform_device *pdev) if (ret) return ret; - plat_dat = stmmac_probe_config_dt(pdev, &stmmac_res.mac); - if (IS_ERR(plat_dat)) - return PTR_ERR(plat_dat); - gmac = devm_kzalloc(dev, sizeof(*gmac), GFP_KERNEL); if (!gmac) return -ENOMEM; @@ -1148,12 +1157,6 @@ static int sun8i_dwmac_probe(struct platform_device *pdev) return -EINVAL; } - gmac->tx_clk = devm_clk_get(dev, "stmmaceth"); - if (IS_ERR(gmac->tx_clk)) { - dev_err(dev, "Could not get TX clock\n"); - return PTR_ERR(gmac->tx_clk); - } - /* Optional regulator for PHY */ gmac->regulator = devm_regulator_get_optional(dev, "phy"); if (IS_ERR(gmac->regulator)) { @@ -1201,11 +1204,15 @@ static int sun8i_dwmac_probe(struct platform_device *pdev) ret = of_get_phy_mode(dev->of_node, &interface); if (ret) return -EINVAL; - plat_dat->interface = interface; + + plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac); + if (IS_ERR(plat_dat)) + return PTR_ERR(plat_dat); /* platform data specifying hardware features and callbacks. * hardware features were copied from Allwinner drivers. */ + plat_dat->interface = interface; plat_dat->rx_coe = STMMAC_RX_COE_TYPE2; plat_dat->tx_coe = 1; plat_dat->has_sun8i = true; @@ -1213,10 +1220,16 @@ static int sun8i_dwmac_probe(struct platform_device *pdev) plat_dat->init = sun8i_dwmac_init; plat_dat->exit = sun8i_dwmac_exit; plat_dat->setup = sun8i_dwmac_setup; + plat_dat->tx_fifo_size = 4096; + plat_dat->rx_fifo_size = 16384; + + ret = sun8i_dwmac_set_syscon(&pdev->dev, plat_dat); + if (ret) + goto dwmac_deconfig; ret = sun8i_dwmac_init(pdev, plat_dat->bsp_priv); if (ret) - return ret; + goto dwmac_syscon; ret = stmmac_dvr_probe(&pdev->dev, plat_dat, &stmmac_res); if (ret) @@ -1224,13 +1237,20 @@ static int sun8i_dwmac_probe(struct platform_device *pdev) ndev = dev_get_drvdata(&pdev->dev); priv = netdev_priv(ndev); + + /* the MAC is runtime suspended after stmmac_dvr_probe(), so we + * need to ensure the MAC resume back before other operations such + * as reset. + */ + pm_runtime_get_sync(&pdev->dev); + /* The mux must be registered after parent MDIO * so after stmmac_dvr_probe() */ if (gmac->variant->soc_has_internal_phy) { ret = get_ephy_nodes(priv); if (ret) - goto dwmac_exit; + goto dwmac_remove; ret = sun8i_dwmac_register_mdio_mux(priv); if (ret) { dev_err(&pdev->dev, "Failed to register mux\n"); @@ -1239,15 +1259,55 @@ static int sun8i_dwmac_probe(struct platform_device *pdev) } else { ret = sun8i_dwmac_reset(priv); if (ret) - goto dwmac_exit; + goto dwmac_remove; } - return ret; + pm_runtime_put(&pdev->dev); + + return 0; + dwmac_mux: - sun8i_dwmac_unset_syscon(gmac); + reset_control_put(gmac->rst_ephy); + clk_put(gmac->ephy_clk); +dwmac_remove: + pm_runtime_put_noidle(&pdev->dev); + stmmac_dvr_remove(&pdev->dev); dwmac_exit: + sun8i_dwmac_exit(pdev, gmac); +dwmac_syscon: + sun8i_dwmac_unset_syscon(gmac); +dwmac_deconfig: + stmmac_remove_config_dt(pdev, plat_dat); + + return ret; +} + +static int sun8i_dwmac_remove(struct platform_device *pdev) +{ + struct net_device *ndev = platform_get_drvdata(pdev); + struct stmmac_priv *priv = netdev_priv(ndev); + struct sunxi_priv_data *gmac = priv->plat->bsp_priv; + + if (gmac->variant->soc_has_internal_phy) { + mdio_mux_uninit(gmac->mux_handle); + sun8i_dwmac_unpower_internal_phy(gmac); + reset_control_put(gmac->rst_ephy); + clk_put(gmac->ephy_clk); + } + stmmac_pltfr_remove(pdev); -return ret; + sun8i_dwmac_unset_syscon(gmac); + + return 0; +} + +static void sun8i_dwmac_shutdown(struct platform_device *pdev) +{ + struct net_device *ndev = platform_get_drvdata(pdev); + struct stmmac_priv *priv = netdev_priv(ndev); + struct sunxi_priv_data *gmac = priv->plat->bsp_priv; + + sun8i_dwmac_exit(pdev, gmac); } static const struct of_device_id sun8i_dwmac_match[] = { @@ -1269,7 +1329,8 @@ MODULE_DEVICE_TABLE(of, sun8i_dwmac_match); static struct platform_driver sun8i_dwmac_driver = { .probe = sun8i_dwmac_probe, - .remove = stmmac_pltfr_remove, + .remove = sun8i_dwmac_remove, + .shutdown = sun8i_dwmac_shutdown, .driver = { .name = "dwmac-sun8i", .pm = &stmmac_pltfr_pm_ops, diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c index 7d40760e9ba8..fc3b0acc8f99 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c @@ -30,7 +30,7 @@ struct sunxi_priv_data { static int sun7i_gmac_init(struct platform_device *pdev, void *priv) { struct sunxi_priv_data *gmac = priv; - int ret; + int ret = 0; if (gmac->regulator) { ret = regulator_enable(gmac->regulator); @@ -51,11 +51,11 @@ static int sun7i_gmac_init(struct platform_device *pdev, void *priv) } else { clk_set_rate(gmac->tx_clk, SUN7I_GMAC_MII_RATE); ret = clk_prepare(gmac->tx_clk); - if (ret) - return ret; + if (ret && gmac->regulator) + regulator_disable(gmac->regulator); } - return 0; + return ret; } static void sun7i_gmac_exit(struct platform_device *pdev, void *priv) @@ -108,7 +108,7 @@ static int sun7i_gmac_probe(struct platform_device *pdev) if (ret) return ret; - plat_dat = stmmac_probe_config_dt(pdev, &stmmac_res.mac); + plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac); if (IS_ERR(plat_dat)) return PTR_ERR(plat_dat); @@ -150,6 +150,8 @@ static int sun7i_gmac_probe(struct platform_device *pdev) plat_dat->init = sun7i_gmac_init; plat_dat->exit = sun7i_gmac_exit; plat_dat->fix_mac_speed = sun7i_fix_speed; + plat_dat->tx_fifo_size = 4096; + plat_dat->rx_fifo_size = 16384; ret = sun7i_gmac_init(pdev, plat_dat->bsp_priv); if (ret) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-visconti.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-visconti.c new file mode 100644 index 000000000000..c3f10a92b62b --- /dev/null +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-visconti.c @@ -0,0 +1,300 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Toshiba Visconti Ethernet Support + * + * (C) Copyright 2020 TOSHIBA CORPORATION + * (C) Copyright 2020 Toshiba Electronic Devices & Storage Corporation + */ + +#include <linux/module.h> +#include <linux/of_device.h> +#include <linux/of_net.h> +#include <linux/stmmac.h> + +#include "stmmac_platform.h" +#include "dwmac4.h" + +#define REG_ETHER_CONTROL 0x52D4 +#define ETHER_ETH_CONTROL_RESET BIT(17) + +#define REG_ETHER_CLOCK_SEL 0x52D0 +#define ETHER_CLK_SEL_TX_CLK_EN BIT(0) +#define ETHER_CLK_SEL_RX_CLK_EN BIT(1) +#define ETHER_CLK_SEL_RMII_CLK_EN BIT(2) +#define ETHER_CLK_SEL_RMII_CLK_RST BIT(3) +#define ETHER_CLK_SEL_DIV_SEL_2 BIT(4) +#define ETHER_CLK_SEL_DIV_SEL_20 0 +#define ETHER_CLK_SEL_FREQ_SEL_125M (BIT(9) | BIT(8)) +#define ETHER_CLK_SEL_FREQ_SEL_50M BIT(9) +#define ETHER_CLK_SEL_FREQ_SEL_25M BIT(8) +#define ETHER_CLK_SEL_FREQ_SEL_2P5M 0 +#define ETHER_CLK_SEL_TX_CLK_EXT_SEL_IN 0 +#define ETHER_CLK_SEL_TX_CLK_EXT_SEL_TXC BIT(10) +#define ETHER_CLK_SEL_TX_CLK_EXT_SEL_DIV BIT(11) +#define ETHER_CLK_SEL_RX_CLK_EXT_SEL_IN 0 +#define ETHER_CLK_SEL_RX_CLK_EXT_SEL_RXC BIT(12) +#define ETHER_CLK_SEL_RX_CLK_EXT_SEL_DIV BIT(13) +#define ETHER_CLK_SEL_TX_CLK_O_TX_I 0 +#define ETHER_CLK_SEL_TX_CLK_O_RMII_I BIT(14) +#define ETHER_CLK_SEL_TX_O_E_N_IN BIT(15) +#define ETHER_CLK_SEL_RMII_CLK_SEL_IN 0 +#define ETHER_CLK_SEL_RMII_CLK_SEL_RX_C BIT(16) + +#define ETHER_CLK_SEL_RX_TX_CLK_EN (ETHER_CLK_SEL_RX_CLK_EN | ETHER_CLK_SEL_TX_CLK_EN) + +#define ETHER_CONFIG_INTF_MII 0 +#define ETHER_CONFIG_INTF_RGMII BIT(0) +#define ETHER_CONFIG_INTF_RMII BIT(2) + +struct visconti_eth { + void __iomem *reg; + u32 phy_intf_sel; + struct clk *phy_ref_clk; + struct device *dev; + spinlock_t lock; /* lock to protect register update */ +}; + +static void visconti_eth_fix_mac_speed(void *priv, unsigned int speed) +{ + struct visconti_eth *dwmac = priv; + struct net_device *netdev = dev_get_drvdata(dwmac->dev); + unsigned int val, clk_sel_val = 0; + unsigned long flags; + + spin_lock_irqsave(&dwmac->lock, flags); + + /* adjust link */ + val = readl(dwmac->reg + MAC_CTRL_REG); + val &= ~(GMAC_CONFIG_PS | GMAC_CONFIG_FES); + + switch (speed) { + case SPEED_1000: + if (dwmac->phy_intf_sel == ETHER_CONFIG_INTF_RGMII) + clk_sel_val = ETHER_CLK_SEL_FREQ_SEL_125M; + break; + case SPEED_100: + if (dwmac->phy_intf_sel == ETHER_CONFIG_INTF_RGMII) + clk_sel_val = ETHER_CLK_SEL_FREQ_SEL_25M; + if (dwmac->phy_intf_sel == ETHER_CONFIG_INTF_RMII) + clk_sel_val = ETHER_CLK_SEL_DIV_SEL_2; + val |= GMAC_CONFIG_PS | GMAC_CONFIG_FES; + break; + case SPEED_10: + if (dwmac->phy_intf_sel == ETHER_CONFIG_INTF_RGMII) + clk_sel_val = ETHER_CLK_SEL_FREQ_SEL_2P5M; + if (dwmac->phy_intf_sel == ETHER_CONFIG_INTF_RMII) + clk_sel_val = ETHER_CLK_SEL_DIV_SEL_20; + val |= GMAC_CONFIG_PS; + break; + default: + /* No bit control */ + netdev_err(netdev, "Unsupported speed request (%d)", speed); + spin_unlock_irqrestore(&dwmac->lock, flags); + return; + } + + writel(val, dwmac->reg + MAC_CTRL_REG); + + /* Stop internal clock */ + val = readl(dwmac->reg + REG_ETHER_CLOCK_SEL); + val &= ~(ETHER_CLK_SEL_RMII_CLK_EN | ETHER_CLK_SEL_RX_TX_CLK_EN); + val |= ETHER_CLK_SEL_TX_O_E_N_IN; + writel(val, dwmac->reg + REG_ETHER_CLOCK_SEL); + + /* Set Clock-Mux, Start clock, Set TX_O direction */ + switch (dwmac->phy_intf_sel) { + case ETHER_CONFIG_INTF_RGMII: + val = clk_sel_val | ETHER_CLK_SEL_RX_CLK_EXT_SEL_RXC; + writel(val, dwmac->reg + REG_ETHER_CLOCK_SEL); + + val |= ETHER_CLK_SEL_RX_TX_CLK_EN; + writel(val, dwmac->reg + REG_ETHER_CLOCK_SEL); + + val &= ~ETHER_CLK_SEL_TX_O_E_N_IN; + writel(val, dwmac->reg + REG_ETHER_CLOCK_SEL); + break; + case ETHER_CONFIG_INTF_RMII: + val = clk_sel_val | ETHER_CLK_SEL_RX_CLK_EXT_SEL_DIV | + ETHER_CLK_SEL_TX_CLK_EXT_SEL_DIV | ETHER_CLK_SEL_TX_O_E_N_IN | + ETHER_CLK_SEL_RMII_CLK_SEL_RX_C; + writel(val, dwmac->reg + REG_ETHER_CLOCK_SEL); + + val |= ETHER_CLK_SEL_RMII_CLK_RST; + writel(val, dwmac->reg + REG_ETHER_CLOCK_SEL); + + val |= ETHER_CLK_SEL_RMII_CLK_EN | ETHER_CLK_SEL_RX_TX_CLK_EN; + writel(val, dwmac->reg + REG_ETHER_CLOCK_SEL); + break; + case ETHER_CONFIG_INTF_MII: + default: + val = clk_sel_val | ETHER_CLK_SEL_RX_CLK_EXT_SEL_RXC | + ETHER_CLK_SEL_TX_CLK_EXT_SEL_TXC | ETHER_CLK_SEL_TX_O_E_N_IN; + writel(val, dwmac->reg + REG_ETHER_CLOCK_SEL); + + val |= ETHER_CLK_SEL_RX_TX_CLK_EN; + writel(val, dwmac->reg + REG_ETHER_CLOCK_SEL); + break; + } + + spin_unlock_irqrestore(&dwmac->lock, flags); +} + +static int visconti_eth_init_hw(struct platform_device *pdev, struct plat_stmmacenet_data *plat_dat) +{ + struct visconti_eth *dwmac = plat_dat->bsp_priv; + unsigned int reg_val, clk_sel_val; + + switch (plat_dat->phy_interface) { + case PHY_INTERFACE_MODE_RGMII: + case PHY_INTERFACE_MODE_RGMII_ID: + case PHY_INTERFACE_MODE_RGMII_RXID: + case PHY_INTERFACE_MODE_RGMII_TXID: + dwmac->phy_intf_sel = ETHER_CONFIG_INTF_RGMII; + break; + case PHY_INTERFACE_MODE_MII: + dwmac->phy_intf_sel = ETHER_CONFIG_INTF_MII; + break; + case PHY_INTERFACE_MODE_RMII: + dwmac->phy_intf_sel = ETHER_CONFIG_INTF_RMII; + break; + default: + dev_err(&pdev->dev, "Unsupported phy-mode (%d)\n", plat_dat->phy_interface); + return -EOPNOTSUPP; + } + + reg_val = dwmac->phy_intf_sel; + writel(reg_val, dwmac->reg + REG_ETHER_CONTROL); + + /* Enable TX/RX clock */ + clk_sel_val = ETHER_CLK_SEL_FREQ_SEL_125M; + writel(clk_sel_val, dwmac->reg + REG_ETHER_CLOCK_SEL); + + writel((clk_sel_val | ETHER_CLK_SEL_RMII_CLK_EN | ETHER_CLK_SEL_RX_TX_CLK_EN), + dwmac->reg + REG_ETHER_CLOCK_SEL); + + /* release internal-reset */ + reg_val |= ETHER_ETH_CONTROL_RESET; + writel(reg_val, dwmac->reg + REG_ETHER_CONTROL); + + return 0; +} + +static int visconti_eth_clock_probe(struct platform_device *pdev, + struct plat_stmmacenet_data *plat_dat) +{ + struct visconti_eth *dwmac = plat_dat->bsp_priv; + int err; + + dwmac->phy_ref_clk = devm_clk_get(&pdev->dev, "phy_ref_clk"); + if (IS_ERR(dwmac->phy_ref_clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(dwmac->phy_ref_clk), + "phy_ref_clk clock not found.\n"); + + err = clk_prepare_enable(dwmac->phy_ref_clk); + if (err < 0) { + dev_err(&pdev->dev, "failed to enable phy_ref clock: %d\n", err); + return err; + } + + return 0; +} + +static int visconti_eth_clock_remove(struct platform_device *pdev) +{ + struct visconti_eth *dwmac = get_stmmac_bsp_priv(&pdev->dev); + struct net_device *ndev = platform_get_drvdata(pdev); + struct stmmac_priv *priv = netdev_priv(ndev); + + clk_disable_unprepare(dwmac->phy_ref_clk); + clk_disable_unprepare(priv->plat->stmmac_clk); + + return 0; +} + +static int visconti_eth_dwmac_probe(struct platform_device *pdev) +{ + struct plat_stmmacenet_data *plat_dat; + struct stmmac_resources stmmac_res; + struct visconti_eth *dwmac; + int ret; + + ret = stmmac_get_platform_resources(pdev, &stmmac_res); + if (ret) + return ret; + + plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac); + if (IS_ERR(plat_dat)) + return PTR_ERR(plat_dat); + + dwmac = devm_kzalloc(&pdev->dev, sizeof(*dwmac), GFP_KERNEL); + if (!dwmac) { + ret = -ENOMEM; + goto remove_config; + } + + spin_lock_init(&dwmac->lock); + dwmac->reg = stmmac_res.addr; + dwmac->dev = &pdev->dev; + plat_dat->bsp_priv = dwmac; + plat_dat->fix_mac_speed = visconti_eth_fix_mac_speed; + + ret = visconti_eth_clock_probe(pdev, plat_dat); + if (ret) + goto remove_config; + + visconti_eth_init_hw(pdev, plat_dat); + + plat_dat->dma_cfg->aal = 1; + + ret = stmmac_dvr_probe(&pdev->dev, plat_dat, &stmmac_res); + if (ret) + goto remove; + + return ret; + +remove: + visconti_eth_clock_remove(pdev); +remove_config: + stmmac_remove_config_dt(pdev, plat_dat); + + return ret; +} + +static int visconti_eth_dwmac_remove(struct platform_device *pdev) +{ + struct net_device *ndev = platform_get_drvdata(pdev); + struct stmmac_priv *priv = netdev_priv(ndev); + int err; + + err = stmmac_pltfr_remove(pdev); + if (err < 0) + dev_err(&pdev->dev, "failed to remove platform: %d\n", err); + + err = visconti_eth_clock_remove(pdev); + if (err < 0) + dev_err(&pdev->dev, "failed to remove clock: %d\n", err); + + stmmac_remove_config_dt(pdev, priv->plat); + + return err; +} + +static const struct of_device_id visconti_eth_dwmac_match[] = { + { .compatible = "toshiba,visconti-dwmac" }, + { } +}; +MODULE_DEVICE_TABLE(of, visconti_eth_dwmac_match); + +static struct platform_driver visconti_eth_dwmac_driver = { + .probe = visconti_eth_dwmac_probe, + .remove = visconti_eth_dwmac_remove, + .driver = { + .name = "visconti-eth-dwmac", + .of_match_table = visconti_eth_dwmac_match, + }, +}; +module_platform_driver(visconti_eth_dwmac_driver); + +MODULE_AUTHOR("Toshiba"); +MODULE_DESCRIPTION("Toshiba Visconti Ethernet DWMAC glue driver"); +MODULE_AUTHOR("Nobuhiro Iwamatsu <nobuhiro1.iwamatsu@toshiba.co.jp"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac100.h b/drivers/net/ethernet/stmicro/stmmac/dwmac100.h index 35ab8d0bdce7..7ab791c8d355 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac100.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac100.h @@ -56,7 +56,7 @@ #define MAC_CONTROL_TE 0x00000008 /* Transmitter Enable */ #define MAC_CONTROL_RE 0x00000004 /* Receiver Enable */ -#define MAC_CORE_INIT (MAC_CONTROL_HBD | MAC_CONTROL_ASTP) +#define MAC_CORE_INIT (MAC_CONTROL_HBD) /* MAC FLOW CTRL defines */ #define MAC_FLOW_CTRL_PT_MASK 0xffff0000 /* Pause Time Mask */ diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h b/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h index b70d44ac0990..4296ddda8aaa 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h @@ -76,10 +76,10 @@ enum power_event { #define LPI_CTRL_STATUS_TLPIEN 0x00000001 /* Transmit LPI Entry */ /* GMAC HW ADDR regs */ -#define GMAC_ADDR_HIGH(reg) (((reg > 15) ? 0x00000800 : 0x00000040) + \ - (reg * 8)) -#define GMAC_ADDR_LOW(reg) (((reg > 15) ? 0x00000804 : 0x00000044) + \ - (reg * 8)) +#define GMAC_ADDR_HIGH(reg) ((reg > 15) ? 0x00000800 + (reg - 16) * 8 : \ + 0x00000040 + (reg * 8)) +#define GMAC_ADDR_LOW(reg) ((reg > 15) ? 0x00000804 + (reg - 16) * 8 : \ + 0x00000044 + (reg * 8)) #define GMAC_MAX_PERFECT_ADDRESSES 1 #define GMAC_PCS_BASE 0x000000c0 /* PCS register base */ @@ -126,7 +126,7 @@ enum inter_frame_gap { #define GMAC_CONTROL_TE 0x00000008 /* Transmitter Enable */ #define GMAC_CONTROL_RE 0x00000004 /* Receiver Enable */ -#define GMAC_CORE_INIT (GMAC_CONTROL_JD | GMAC_CONTROL_PS | GMAC_CONTROL_ACS | \ +#define GMAC_CORE_INIT (GMAC_CONTROL_JD | GMAC_CONTROL_PS | \ GMAC_CONTROL_BE | GMAC_CONTROL_DCRS) /* GMAC Frame Filter defines */ diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c index 542784300620..0e00dd83d027 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c @@ -15,7 +15,6 @@ #include <linux/crc32.h> #include <linux/slab.h> #include <linux/ethtool.h> -#include <net/dsa.h> #include <asm/io.h> #include "stmmac.h" #include "stmmac_pcs.h" @@ -24,7 +23,6 @@ static void dwmac1000_core_init(struct mac_device_info *hw, struct net_device *dev) { - struct stmmac_priv *priv = netdev_priv(dev); void __iomem *ioaddr = hw->pcsr; u32 value = readl(ioaddr + GMAC_CONTROL); int mtu = dev->mtu; @@ -32,13 +30,6 @@ static void dwmac1000_core_init(struct mac_device_info *hw, /* Configure GMAC core */ value |= GMAC_CORE_INIT; - /* Clear ACS bit because Ethernet switch tagging formats such as - * Broadcom tags can look like invalid LLC/SNAP packets and cause the - * hardware to truncate packets on reception. - */ - if (netdev_uses_dsa(dev) || !priv->plat->enh_desc) - value &= ~GMAC_CONTROL_ACS; - if (mtu > 1500) value |= GMAC_CONTROL_2K; if (mtu > 2000) @@ -104,7 +95,7 @@ static void dwmac1000_dump_regs(struct mac_device_info *hw, u32 *reg_space) } static void dwmac1000_set_umac_addr(struct mac_device_info *hw, - unsigned char *addr, + const unsigned char *addr, unsigned int reg_n) { void __iomem *ioaddr = hw->pcsr; @@ -164,6 +155,9 @@ static void dwmac1000_set_filter(struct mac_device_info *hw, value = GMAC_FRAME_FILTER_PR | GMAC_FRAME_FILTER_PCF; } else if (dev->flags & IFF_ALLMULTI) { value = GMAC_FRAME_FILTER_PM; /* pass all multi */ + } else if (!netdev_mc_empty(dev) && (mcbitslog2 == 0)) { + /* Fall back to all multicast if we've no filter */ + value = GMAC_FRAME_FILTER_PM; } else if (!netdev_mc_empty(dev)) { struct netdev_hw_addr *ha; @@ -207,7 +201,7 @@ static void dwmac1000_set_filter(struct mac_device_info *hw, reg++; } - while (reg <= perfect_addr_number) { + while (reg < perfect_addr_number) { writel(0, ioaddr + GMAC_ADDR_HIGH(reg)); writel(0, ioaddr + GMAC_ADDR_LOW(reg)); reg++; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_dma.c index 2bac49b49f73..f5581db0ba9b 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_dma.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_dma.c @@ -218,11 +218,18 @@ static void dwmac1000_dump_dma_regs(void __iomem *ioaddr, u32 *reg_space) readl(ioaddr + DMA_BUS_MODE + i * 4); } -static void dwmac1000_get_hw_feature(void __iomem *ioaddr, - struct dma_features *dma_cap) +static int dwmac1000_get_hw_feature(void __iomem *ioaddr, + struct dma_features *dma_cap) { u32 hw_cap = readl(ioaddr + DMA_HW_FEATURE); + if (!hw_cap) { + /* 0x00000000 is the value read on old hardware that does not + * implement this register + */ + return -EOPNOTSUPP; + } + dma_cap->mbps_10_100 = (hw_cap & DMA_HW_FEAT_MIISEL); dma_cap->mbps_1000 = (hw_cap & DMA_HW_FEAT_GMIISEL) >> 1; dma_cap->half_duplex = (hw_cap & DMA_HW_FEAT_HDSEL) >> 2; @@ -252,10 +259,12 @@ static void dwmac1000_get_hw_feature(void __iomem *ioaddr, dma_cap->number_tx_channel = (hw_cap & DMA_HW_FEAT_TXCHCNT) >> 22; /* Alternate (enhanced) DESC mode */ dma_cap->enh_desc = (hw_cap & DMA_HW_FEAT_ENHDESSEL) >> 24; + + return 0; } static void dwmac1000_rx_watchdog(void __iomem *ioaddr, u32 riwt, - u32 number_chan) + u32 queue) { writel(riwt, ioaddr + DMA_RX_WATCHDOG); } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c index ebcad8dd99db..a6e8d7bd9588 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c @@ -15,7 +15,6 @@ *******************************************************************************/ #include <linux/crc32.h> -#include <net/dsa.h> #include <asm/io.h> #include "stmmac.h" #include "dwmac100.h" @@ -28,13 +27,6 @@ static void dwmac100_core_init(struct mac_device_info *hw, value |= MAC_CORE_INIT; - /* Clear ASTP bit because Ethernet switch tagging formats such as - * Broadcom tags can look like invalid LLC/SNAP packets and cause the - * hardware to truncate packets on reception. - */ - if (netdev_uses_dsa(dev)) - value &= ~MAC_CONTROL_ASTP; - writel(value, ioaddr + MAC_CONTROL); #ifdef STMMAC_VLAN_TAG_USED @@ -68,7 +60,7 @@ static int dwmac100_irq_status(struct mac_device_info *hw, } static void dwmac100_set_umac_addr(struct mac_device_info *hw, - unsigned char *addr, + const unsigned char *addr, unsigned int reg_n) { void __iomem *ioaddr = hw->pcsr; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4.h b/drivers/net/ethernet/stmicro/stmmac/dwmac4.h index af50af27550b..71dad409f78b 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4.h @@ -18,6 +18,7 @@ #define GMAC_PACKET_FILTER 0x00000008 #define GMAC_HASH_TAB(x) (0x10 + (x) * 4) #define GMAC_VLAN_TAG 0x00000050 +#define GMAC_VLAN_TAG_DATA 0x00000054 #define GMAC_VLAN_HASH_TABLE 0x00000058 #define GMAC_RX_FLOW_CTRL 0x00000090 #define GMAC_VLAN_INCL 0x00000060 @@ -41,6 +42,7 @@ #define GMAC_HW_FEATURE3 0x00000128 #define GMAC_MDIO_ADDR 0x00000200 #define GMAC_MDIO_DATA 0x00000204 +#define GMAC_GPIO_STATUS 0x0000020C #define GMAC_ARP_ADDR 0x00000210 #define GMAC_ADDR_HIGH(reg) (0x300 + reg * 8) #define GMAC_ADDR_LOW(reg) (0x304 + reg * 8) @@ -48,6 +50,7 @@ #define GMAC_L4_ADDR(reg) (0x904 + (reg) * 0x30) #define GMAC_L3_ADDR0(reg) (0x910 + (reg) * 0x30) #define GMAC_L3_ADDR1(reg) (0x914 + (reg) * 0x30) +#define GMAC_TIMESTAMP_STATUS 0x00000b20 /* RX Queues Routing */ #define GMAC_RXQCTRL_AVCPQ_MASK GENMASK(2, 0) @@ -75,6 +78,7 @@ #define GMAC_PACKET_FILTER_HPF BIT(10) #define GMAC_PACKET_FILTER_VTFE BIT(16) #define GMAC_PACKET_FILTER_IPFE BIT(20) +#define GMAC_PACKET_FILTER_RA BIT(31) #define GMAC_MAX_PERFECT_ADDRESSES 128 @@ -89,6 +93,30 @@ #define GMAC_VLAN_CSVL BIT(19) #define GMAC_VLAN_VLC GENMASK(17, 16) #define GMAC_VLAN_VLC_SHIFT 16 +#define GMAC_VLAN_VLHT GENMASK(15, 0) + +/* MAC VLAN Tag */ +#define GMAC_VLAN_TAG_VID GENMASK(15, 0) +#define GMAC_VLAN_TAG_ETV BIT(16) + +/* MAC VLAN Tag Control */ +#define GMAC_VLAN_TAG_CTRL_OB BIT(0) +#define GMAC_VLAN_TAG_CTRL_CT BIT(1) +#define GMAC_VLAN_TAG_CTRL_OFS_MASK GENMASK(6, 2) +#define GMAC_VLAN_TAG_CTRL_OFS_SHIFT 2 +#define GMAC_VLAN_TAG_CTRL_EVLS_MASK GENMASK(22, 21) +#define GMAC_VLAN_TAG_CTRL_EVLS_SHIFT 21 +#define GMAC_VLAN_TAG_CTRL_EVLRXS BIT(24) + +#define GMAC_VLAN_TAG_STRIP_NONE (0x0 << GMAC_VLAN_TAG_CTRL_EVLS_SHIFT) +#define GMAC_VLAN_TAG_STRIP_PASS (0x1 << GMAC_VLAN_TAG_CTRL_EVLS_SHIFT) +#define GMAC_VLAN_TAG_STRIP_FAIL (0x2 << GMAC_VLAN_TAG_CTRL_EVLS_SHIFT) +#define GMAC_VLAN_TAG_STRIP_ALL (0x3 << GMAC_VLAN_TAG_CTRL_EVLS_SHIFT) + +/* MAC VLAN Tag Data/Filter */ +#define GMAC_VLAN_TAG_DATA_VID GENMASK(15, 0) +#define GMAC_VLAN_TAG_DATA_VEN BIT(16) +#define GMAC_VLAN_TAG_DATA_ETV BIT(17) /* MAC RX Queue Enable */ #define GMAC_RX_QUEUE_CLEAR(queue) ~(GENMASK(1, 0) << ((queue) * 2)) @@ -117,11 +145,13 @@ #define GMAC_INT_PCS_PHYIS BIT(3) #define GMAC_INT_PMT_EN BIT(4) #define GMAC_INT_LPI_EN BIT(5) +#define GMAC_INT_TSIE BIT(12) #define GMAC_PCS_IRQ_DEFAULT (GMAC_INT_RGSMIIS | GMAC_INT_PCS_LINK | \ GMAC_INT_PCS_ANE) -#define GMAC_INT_DEFAULT_ENABLE (GMAC_INT_PMT_EN | GMAC_INT_LPI_EN) +#define GMAC_INT_DEFAULT_ENABLE (GMAC_INT_PMT_EN | GMAC_INT_LPI_EN | \ + GMAC_INT_TSIE) enum dwmac4_irq_status { time_stamp_irq = 0x00001000, @@ -150,9 +180,11 @@ enum power_event { */ #define GMAC4_LPI_CTRL_STATUS 0xd0 #define GMAC4_LPI_TIMER_CTRL 0xd4 +#define GMAC4_LPI_ENTRY_TIMER 0xd8 /* LPI control and status defines */ #define GMAC4_LPI_CTRL_STATUS_LPITCSE BIT(21) /* LPI Tx Clock Stop Enable */ +#define GMAC4_LPI_CTRL_STATUS_LPIATE BIT(20) /* LPI Timer Enable */ #define GMAC4_LPI_CTRL_STATUS_LPITXA BIT(19) /* Enable LPI TX Automate */ #define GMAC4_LPI_CTRL_STATUS_PLS BIT(17) /* PHY Link Status */ #define GMAC4_LPI_CTRL_STATUS_LPIEN BIT(16) /* LPI Enable */ @@ -231,6 +263,7 @@ enum power_event { #define GMAC_HW_RXFIFOSIZE GENMASK(4, 0) /* MAC HW features2 bitmap */ +#define GMAC_HW_FEAT_AUXSNAPNUM GENMASK(30, 28) #define GMAC_HW_FEAT_PPSOUTNUM GENMASK(26, 24) #define GMAC_HW_FEAT_TXCHCNT GENMASK(21, 18) #define GMAC_HW_FEAT_RXCHCNT GENMASK(15, 12) @@ -248,6 +281,13 @@ enum power_event { #define GMAC_HW_FEAT_FRPBS GENMASK(12, 11) #define GMAC_HW_FEAT_FRPSEL BIT(10) #define GMAC_HW_FEAT_DVLAN BIT(5) +#define GMAC_HW_FEAT_NRVF GENMASK(2, 0) + +/* GMAC GPIO Status reg */ +#define GMAC_GPO0 BIT(16) +#define GMAC_GPO1 BIT(17) +#define GMAC_GPO2 BIT(18) +#define GMAC_GPO3 BIT(19) /* MAC HW ADDR regs */ #define GMAC_HI_DCS GENMASK(18, 16) @@ -269,6 +309,11 @@ enum power_event { #define GMAC_L4DP0_SHIFT 16 #define GMAC_L4SP0 GENMASK(15, 0) +/* MAC Timestamp Status */ +#define GMAC_TIMESTAMP_AUXTSTRIG BIT(2) +#define GMAC_TIMESTAMP_ATSNS_MASK GENMASK(29, 25) +#define GMAC_TIMESTAMP_ATSNS_SHIFT 25 + /* MTL registers */ #define MTL_OPERATION_MODE 0x00000c00 #define MTL_FRPE BIT(15) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c index dc09d2131e40..c25bfecb4a2d 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c @@ -14,7 +14,6 @@ #include <linux/slab.h> #include <linux/ethtool.h> #include <linux/io.h> -#include <net/dsa.h> #include "stmmac.h" #include "stmmac_pcs.h" #include "dwmac4.h" @@ -23,6 +22,7 @@ static void dwmac4_core_init(struct mac_device_info *hw, struct net_device *dev) { + struct stmmac_priv *priv = netdev_priv(dev); void __iomem *ioaddr = hw->pcsr; u32 value = readl(ioaddr + GMAC_CONFIG); @@ -53,7 +53,14 @@ static void dwmac4_core_init(struct mac_device_info *hw, if (hw->pcs) value |= GMAC_PCS_IRQ_DEFAULT; + /* Enable FPE interrupt */ + if ((GMAC_HW_FEAT_FPESEL & readl(ioaddr + GMAC_HW_FEATURE3)) >> 26) + value |= GMAC_INT_FPE_EN; + writel(value, ioaddr + GMAC_INT_EN); + + if (GMAC_INT_DEFAULT_ENABLE & GMAC_INT_TSIE) + init_waitqueue_head(&priv->tstamp_busy_wait); } static void dwmac4_rx_queue_enable(struct mac_device_info *hw, @@ -215,6 +222,9 @@ static void dwmac4_map_mtl_dma(struct mac_device_info *hw, u32 queue, u32 chan) if (queue == 0 || queue == 4) { value &= ~MTL_RXQ_DMA_Q04MDMACH_MASK; value |= MTL_RXQ_DMA_Q04MDMACH(chan); + } else if (queue > 4) { + value &= ~MTL_RXQ_DMA_QXMDMACH_MASK(queue - 4); + value |= MTL_RXQ_DMA_QXMDMACH(chan, queue - 4); } else { value &= ~MTL_RXQ_DMA_QXMDMACH_MASK(queue); value |= MTL_RXQ_DMA_QXMDMACH(chan, queue); @@ -318,7 +328,7 @@ static void dwmac4_pmt(struct mac_device_info *hw, unsigned long mode) } static void dwmac4_set_umac_addr(struct mac_device_info *hw, - unsigned char *addr, unsigned int reg_n) + const unsigned char *addr, unsigned int reg_n) { void __iomem *ioaddr = hw->pcsr; @@ -379,6 +389,27 @@ static void dwmac4_set_eee_pls(struct mac_device_info *hw, int link) writel(value, ioaddr + GMAC4_LPI_CTRL_STATUS); } +static void dwmac4_set_eee_lpi_entry_timer(struct mac_device_info *hw, int et) +{ + void __iomem *ioaddr = hw->pcsr; + int value = et & STMMAC_ET_MAX; + int regval; + + /* Program LPI entry timer value into register */ + writel(value, ioaddr + GMAC4_LPI_ENTRY_TIMER); + + /* Enable/disable LPI entry timer */ + regval = readl(ioaddr + GMAC4_LPI_CTRL_STATUS); + regval |= GMAC4_LPI_CTRL_STATUS_LPIEN | GMAC4_LPI_CTRL_STATUS_LPITXA; + + if (et) + regval |= GMAC4_LPI_CTRL_STATUS_LPIATE; + else + regval &= ~GMAC4_LPI_CTRL_STATUS_LPIATE; + + writel(regval, ioaddr + GMAC4_LPI_CTRL_STATUS); +} + static void dwmac4_set_eee_timer(struct mac_device_info *hw, int ls, int tw) { void __iomem *ioaddr = hw->pcsr; @@ -394,6 +425,211 @@ static void dwmac4_set_eee_timer(struct mac_device_info *hw, int ls, int tw) writel(value, ioaddr + GMAC4_LPI_TIMER_CTRL); } +static void dwmac4_write_single_vlan(struct net_device *dev, u16 vid) +{ + void __iomem *ioaddr = (void __iomem *)dev->base_addr; + u32 val; + + val = readl(ioaddr + GMAC_VLAN_TAG); + val &= ~GMAC_VLAN_TAG_VID; + val |= GMAC_VLAN_TAG_ETV | vid; + + writel(val, ioaddr + GMAC_VLAN_TAG); +} + +static int dwmac4_write_vlan_filter(struct net_device *dev, + struct mac_device_info *hw, + u8 index, u32 data) +{ + void __iomem *ioaddr = (void __iomem *)dev->base_addr; + int i, timeout = 10; + u32 val; + + if (index >= hw->num_vlan) + return -EINVAL; + + writel(data, ioaddr + GMAC_VLAN_TAG_DATA); + + val = readl(ioaddr + GMAC_VLAN_TAG); + val &= ~(GMAC_VLAN_TAG_CTRL_OFS_MASK | + GMAC_VLAN_TAG_CTRL_CT | + GMAC_VLAN_TAG_CTRL_OB); + val |= (index << GMAC_VLAN_TAG_CTRL_OFS_SHIFT) | GMAC_VLAN_TAG_CTRL_OB; + + writel(val, ioaddr + GMAC_VLAN_TAG); + + for (i = 0; i < timeout; i++) { + val = readl(ioaddr + GMAC_VLAN_TAG); + if (!(val & GMAC_VLAN_TAG_CTRL_OB)) + return 0; + udelay(1); + } + + netdev_err(dev, "Timeout accessing MAC_VLAN_Tag_Filter\n"); + + return -EBUSY; +} + +static int dwmac4_add_hw_vlan_rx_fltr(struct net_device *dev, + struct mac_device_info *hw, + __be16 proto, u16 vid) +{ + int index = -1; + u32 val = 0; + int i, ret; + + if (vid > 4095) + return -EINVAL; + + if (hw->promisc) { + netdev_err(dev, + "Adding VLAN in promisc mode not supported\n"); + return -EPERM; + } + + /* Single Rx VLAN Filter */ + if (hw->num_vlan == 1) { + /* For single VLAN filter, VID 0 means VLAN promiscuous */ + if (vid == 0) { + netdev_warn(dev, "Adding VLAN ID 0 is not supported\n"); + return -EPERM; + } + + if (hw->vlan_filter[0] & GMAC_VLAN_TAG_VID) { + netdev_err(dev, "Only single VLAN ID supported\n"); + return -EPERM; + } + + hw->vlan_filter[0] = vid; + dwmac4_write_single_vlan(dev, vid); + + return 0; + } + + /* Extended Rx VLAN Filter Enable */ + val |= GMAC_VLAN_TAG_DATA_ETV | GMAC_VLAN_TAG_DATA_VEN | vid; + + for (i = 0; i < hw->num_vlan; i++) { + if (hw->vlan_filter[i] == val) + return 0; + else if (!(hw->vlan_filter[i] & GMAC_VLAN_TAG_DATA_VEN)) + index = i; + } + + if (index == -1) { + netdev_err(dev, "MAC_VLAN_Tag_Filter full (size: %0u)\n", + hw->num_vlan); + return -EPERM; + } + + ret = dwmac4_write_vlan_filter(dev, hw, index, val); + + if (!ret) + hw->vlan_filter[index] = val; + + return ret; +} + +static int dwmac4_del_hw_vlan_rx_fltr(struct net_device *dev, + struct mac_device_info *hw, + __be16 proto, u16 vid) +{ + int i, ret = 0; + + if (hw->promisc) { + netdev_err(dev, + "Deleting VLAN in promisc mode not supported\n"); + return -EPERM; + } + + /* Single Rx VLAN Filter */ + if (hw->num_vlan == 1) { + if ((hw->vlan_filter[0] & GMAC_VLAN_TAG_VID) == vid) { + hw->vlan_filter[0] = 0; + dwmac4_write_single_vlan(dev, 0); + } + return 0; + } + + /* Extended Rx VLAN Filter Enable */ + for (i = 0; i < hw->num_vlan; i++) { + if ((hw->vlan_filter[i] & GMAC_VLAN_TAG_DATA_VID) == vid) { + ret = dwmac4_write_vlan_filter(dev, hw, i, 0); + + if (!ret) + hw->vlan_filter[i] = 0; + else + return ret; + } + } + + return ret; +} + +static void dwmac4_vlan_promisc_enable(struct net_device *dev, + struct mac_device_info *hw) +{ + void __iomem *ioaddr = hw->pcsr; + u32 value; + u32 hash; + u32 val; + int i; + + /* Single Rx VLAN Filter */ + if (hw->num_vlan == 1) { + dwmac4_write_single_vlan(dev, 0); + return; + } + + /* Extended Rx VLAN Filter Enable */ + for (i = 0; i < hw->num_vlan; i++) { + if (hw->vlan_filter[i] & GMAC_VLAN_TAG_DATA_VEN) { + val = hw->vlan_filter[i] & ~GMAC_VLAN_TAG_DATA_VEN; + dwmac4_write_vlan_filter(dev, hw, i, val); + } + } + + hash = readl(ioaddr + GMAC_VLAN_HASH_TABLE); + if (hash & GMAC_VLAN_VLHT) { + value = readl(ioaddr + GMAC_VLAN_TAG); + if (value & GMAC_VLAN_VTHM) { + value &= ~GMAC_VLAN_VTHM; + writel(value, ioaddr + GMAC_VLAN_TAG); + } + } +} + +static void dwmac4_restore_hw_vlan_rx_fltr(struct net_device *dev, + struct mac_device_info *hw) +{ + void __iomem *ioaddr = hw->pcsr; + u32 value; + u32 hash; + u32 val; + int i; + + /* Single Rx VLAN Filter */ + if (hw->num_vlan == 1) { + dwmac4_write_single_vlan(dev, hw->vlan_filter[0]); + return; + } + + /* Extended Rx VLAN Filter Enable */ + for (i = 0; i < hw->num_vlan; i++) { + if (hw->vlan_filter[i] & GMAC_VLAN_TAG_DATA_VEN) { + val = hw->vlan_filter[i]; + dwmac4_write_vlan_filter(dev, hw, i, val); + } + } + + hash = readl(ioaddr + GMAC_VLAN_HASH_TABLE); + if (hash & GMAC_VLAN_VLHT) { + value = readl(ioaddr + GMAC_VLAN_TAG); + value |= GMAC_VLAN_VTHM; + writel(value, ioaddr + GMAC_VLAN_TAG); + } +} + static void dwmac4_set_filter(struct mac_device_info *hw, struct net_device *dev) { @@ -412,8 +648,20 @@ static void dwmac4_set_filter(struct mac_device_info *hw, value &= ~GMAC_PACKET_FILTER_PCF; value &= ~GMAC_PACKET_FILTER_PM; value &= ~GMAC_PACKET_FILTER_PR; + value &= ~GMAC_PACKET_FILTER_RA; if (dev->flags & IFF_PROMISC) { - value = GMAC_PACKET_FILTER_PR | GMAC_PACKET_FILTER_PCF; + /* VLAN Tag Filter Fail Packets Queuing */ + if (hw->vlan_fail_q_en) { + value = readl(ioaddr + GMAC_RXQ_CTRL4); + value &= ~GMAC_RXQCTRL_VFFQ_MASK; + value |= GMAC_RXQCTRL_VFFQE | + (hw->vlan_fail_q << GMAC_RXQCTRL_VFFQ_SHIFT); + writel(value, ioaddr + GMAC_RXQ_CTRL4); + value = GMAC_PACKET_FILTER_PR | GMAC_PACKET_FILTER_RA; + } else { + value = GMAC_PACKET_FILTER_PR | GMAC_PACKET_FILTER_PCF; + } + } else if ((dev->flags & IFF_ALLMULTI) || (netdev_mc_count(dev) > hw->multicast_filter_bins)) { /* Pass all multi */ @@ -469,7 +717,23 @@ static void dwmac4_set_filter(struct mac_device_info *hw, } } + /* VLAN filtering */ + if (dev->features & NETIF_F_HW_VLAN_CTAG_FILTER) + value |= GMAC_PACKET_FILTER_VTFE; + writel(value, ioaddr + GMAC_PACKET_FILTER); + + if (dev->flags & IFF_PROMISC && !hw->vlan_fail_q_en) { + if (!hw->promisc) { + hw->promisc = 1; + dwmac4_vlan_promisc_enable(dev, hw); + } + } else { + if (hw->promisc) { + hw->promisc = 0; + dwmac4_restore_hw_vlan_rx_fltr(dev, hw); + } + } } static void dwmac4_flow_ctrl(struct mac_device_info *hw, unsigned int duplex, @@ -932,6 +1196,7 @@ const struct stmmac_ops dwmac4_ops = { .get_umac_addr = dwmac4_get_umac_addr, .set_eee_mode = dwmac4_set_eee_mode, .reset_eee_mode = dwmac4_reset_eee_mode, + .set_eee_lpi_entry_timer = dwmac4_set_eee_lpi_entry_timer, .set_eee_timer = dwmac4_set_eee_timer, .set_eee_pls = dwmac4_set_eee_pls, .pcs_ctrl_ane = dwmac4_ctrl_ane, @@ -939,7 +1204,6 @@ const struct stmmac_ops dwmac4_ops = { .pcs_get_adv_lp = dwmac4_get_adv_lp, .debug = dwmac4_debug, .set_filter = dwmac4_set_filter, - .flex_pps_config = dwmac5_flex_pps_config, .set_mac_loopback = dwmac4_set_mac_loopback, .update_vlan_hash = dwmac4_update_vlan_hash, .sarc_configure = dwmac4_sarc_configure, @@ -947,6 +1211,9 @@ const struct stmmac_ops dwmac4_ops = { .set_arp_offload = dwmac4_set_arp_offload, .config_l3_filter = dwmac4_config_l3_filter, .config_l4_filter = dwmac4_config_l4_filter, + .add_hw_vlan_rx_fltr = dwmac4_add_hw_vlan_rx_fltr, + .del_hw_vlan_rx_fltr = dwmac4_del_hw_vlan_rx_fltr, + .restore_hw_vlan_rx_fltr = dwmac4_restore_hw_vlan_rx_fltr, }; const struct stmmac_ops dwmac410_ops = { @@ -971,6 +1238,7 @@ const struct stmmac_ops dwmac410_ops = { .get_umac_addr = dwmac4_get_umac_addr, .set_eee_mode = dwmac4_set_eee_mode, .reset_eee_mode = dwmac4_reset_eee_mode, + .set_eee_lpi_entry_timer = dwmac4_set_eee_lpi_entry_timer, .set_eee_timer = dwmac4_set_eee_timer, .set_eee_pls = dwmac4_set_eee_pls, .pcs_ctrl_ane = dwmac4_ctrl_ane, @@ -978,6 +1246,7 @@ const struct stmmac_ops dwmac410_ops = { .pcs_get_adv_lp = dwmac4_get_adv_lp, .debug = dwmac4_debug, .set_filter = dwmac4_set_filter, + .flex_pps_config = dwmac5_flex_pps_config, .set_mac_loopback = dwmac4_set_mac_loopback, .update_vlan_hash = dwmac4_update_vlan_hash, .sarc_configure = dwmac4_sarc_configure, @@ -986,7 +1255,13 @@ const struct stmmac_ops dwmac410_ops = { .config_l3_filter = dwmac4_config_l3_filter, .config_l4_filter = dwmac4_config_l4_filter, .est_configure = dwmac5_est_configure, + .est_irq_status = dwmac5_est_irq_status, .fpe_configure = dwmac5_fpe_configure, + .fpe_send_mpacket = dwmac5_fpe_send_mpacket, + .fpe_irq_status = dwmac5_fpe_irq_status, + .add_hw_vlan_rx_fltr = dwmac4_add_hw_vlan_rx_fltr, + .del_hw_vlan_rx_fltr = dwmac4_del_hw_vlan_rx_fltr, + .restore_hw_vlan_rx_fltr = dwmac4_restore_hw_vlan_rx_fltr, }; const struct stmmac_ops dwmac510_ops = { @@ -1011,6 +1286,7 @@ const struct stmmac_ops dwmac510_ops = { .get_umac_addr = dwmac4_get_umac_addr, .set_eee_mode = dwmac4_set_eee_mode, .reset_eee_mode = dwmac4_reset_eee_mode, + .set_eee_lpi_entry_timer = dwmac4_set_eee_lpi_entry_timer, .set_eee_timer = dwmac4_set_eee_timer, .set_eee_pls = dwmac4_set_eee_pls, .pcs_ctrl_ane = dwmac4_ctrl_ane, @@ -1031,9 +1307,46 @@ const struct stmmac_ops dwmac510_ops = { .config_l3_filter = dwmac4_config_l3_filter, .config_l4_filter = dwmac4_config_l4_filter, .est_configure = dwmac5_est_configure, + .est_irq_status = dwmac5_est_irq_status, .fpe_configure = dwmac5_fpe_configure, + .fpe_send_mpacket = dwmac5_fpe_send_mpacket, + .fpe_irq_status = dwmac5_fpe_irq_status, + .add_hw_vlan_rx_fltr = dwmac4_add_hw_vlan_rx_fltr, + .del_hw_vlan_rx_fltr = dwmac4_del_hw_vlan_rx_fltr, + .restore_hw_vlan_rx_fltr = dwmac4_restore_hw_vlan_rx_fltr, }; +static u32 dwmac4_get_num_vlan(void __iomem *ioaddr) +{ + u32 val, num_vlan; + + val = readl(ioaddr + GMAC_HW_FEATURE3); + switch (val & GMAC_HW_FEAT_NRVF) { + case 0: + num_vlan = 1; + break; + case 1: + num_vlan = 4; + break; + case 2: + num_vlan = 8; + break; + case 3: + num_vlan = 16; + break; + case 4: + num_vlan = 24; + break; + case 5: + num_vlan = 32; + break; + default: + num_vlan = 1; + } + + return num_vlan; +} + int dwmac4_setup(struct stmmac_priv *priv) { struct mac_device_info *mac = priv->hw; @@ -1053,6 +1366,7 @@ int dwmac4_setup(struct stmmac_priv *priv) mac->link.speed10 = GMAC_CONFIG_PS; mac->link.speed100 = GMAC_CONFIG_FES | GMAC_CONFIG_PS; mac->link.speed1000 = 0; + mac->link.speed2500 = GMAC_CONFIG_FES; mac->link.speed_mask = GMAC_CONFIG_FES | GMAC_CONFIG_PS; mac->mii.addr = GMAC_MDIO_ADDR; mac->mii.data = GMAC_MDIO_DATA; @@ -1062,6 +1376,7 @@ int dwmac4_setup(struct stmmac_priv *priv) mac->mii.reg_mask = GENMASK(20, 16); mac->mii.clk_csr_shift = 8; mac->mii.clk_csr_mask = GENMASK(11, 8); + mac->num_vlan = dwmac4_get_num_vlan(priv->ioaddr); return 0; } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c index eff82065a501..8cc80b1db4cb 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c @@ -32,6 +32,8 @@ static int dwmac4_wrback_get_tx_status(void *data, struct stmmac_extra_stats *x, return tx_not_ls; if (unlikely(tdes3 & TDES3_ERROR_SUMMARY)) { + ret = tx_err; + if (unlikely(tdes3 & TDES3_JABBER_TIMEOUT)) x->tx_jabber++; if (unlikely(tdes3 & TDES3_PACKET_FLUSHED)) @@ -53,16 +55,16 @@ static int dwmac4_wrback_get_tx_status(void *data, struct stmmac_extra_stats *x, if (unlikely(tdes3 & TDES3_EXCESSIVE_DEFERRAL)) x->tx_deferred++; - if (unlikely(tdes3 & TDES3_UNDERFLOW_ERROR)) + if (unlikely(tdes3 & TDES3_UNDERFLOW_ERROR)) { x->tx_underflow++; + ret |= tx_err_bump_tc; + } if (unlikely(tdes3 & TDES3_IP_HDR_ERROR)) x->tx_ip_header_error++; if (unlikely(tdes3 & TDES3_PAYLOAD_ERROR)) x->tx_payload_error++; - - ret = tx_err; } if (unlikely(tdes3 & TDES3_DEFERRED)) @@ -402,19 +404,53 @@ static void dwmac4_rd_set_tx_ic(struct dma_desc *p) p->des2 |= cpu_to_le32(TDES2_INTERRUPT_ON_COMPLETION); } -static void dwmac4_display_ring(void *head, unsigned int size, bool rx) +static void dwmac4_display_ring(void *head, unsigned int size, bool rx, + dma_addr_t dma_rx_phy, unsigned int desc_size) { - struct dma_desc *p = (struct dma_desc *)head; + dma_addr_t dma_addr; int i; pr_info("%s descriptor ring:\n", rx ? "RX" : "TX"); - for (i = 0; i < size; i++) { - pr_info("%03d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n", - i, (unsigned int)virt_to_phys(p), - le32_to_cpu(p->des0), le32_to_cpu(p->des1), - le32_to_cpu(p->des2), le32_to_cpu(p->des3)); - p++; + if (desc_size == sizeof(struct dma_desc)) { + struct dma_desc *p = (struct dma_desc *)head; + + for (i = 0; i < size; i++) { + dma_addr = dma_rx_phy + i * sizeof(*p); + pr_info("%03d [%pad]: 0x%x 0x%x 0x%x 0x%x\n", + i, &dma_addr, + le32_to_cpu(p->des0), le32_to_cpu(p->des1), + le32_to_cpu(p->des2), le32_to_cpu(p->des3)); + p++; + } + } else if (desc_size == sizeof(struct dma_extended_desc)) { + struct dma_extended_desc *extp = (struct dma_extended_desc *)head; + + for (i = 0; i < size; i++) { + dma_addr = dma_rx_phy + i * sizeof(*extp); + pr_info("%03d [%pad]: 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n", + i, &dma_addr, + le32_to_cpu(extp->basic.des0), le32_to_cpu(extp->basic.des1), + le32_to_cpu(extp->basic.des2), le32_to_cpu(extp->basic.des3), + le32_to_cpu(extp->des4), le32_to_cpu(extp->des5), + le32_to_cpu(extp->des6), le32_to_cpu(extp->des7)); + extp++; + } + } else if (desc_size == sizeof(struct dma_edesc)) { + struct dma_edesc *ep = (struct dma_edesc *)head; + + for (i = 0; i < size; i++) { + dma_addr = dma_rx_phy + i * sizeof(*ep); + pr_info("%03d [%pad]: 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n", + i, &dma_addr, + le32_to_cpu(ep->des4), le32_to_cpu(ep->des5), + le32_to_cpu(ep->des6), le32_to_cpu(ep->des7), + le32_to_cpu(ep->basic.des0), le32_to_cpu(ep->basic.des1), + le32_to_cpu(ep->basic.des2), le32_to_cpu(ep->basic.des3)); + ep++; + } + } else { + pr_err("unsupported descriptor!"); } } @@ -426,11 +462,6 @@ static void dwmac4_set_mss_ctxt(struct dma_desc *p, unsigned int mss) p->des3 = cpu_to_le32(TDES3_CONTEXT_TYPE | TDES3_CTXT_TCMSSV); } -static void dwmac4_get_addr(struct dma_desc *p, unsigned int *addr) -{ - *addr = le32_to_cpu(p->des0); -} - static void dwmac4_set_addr(struct dma_desc *p, dma_addr_t addr) { p->des0 = cpu_to_le32(lower_32_bits(addr)); @@ -494,16 +525,20 @@ static void dwmac4_set_vlan(struct dma_desc *p, u32 type) p->des2 |= cpu_to_le32(type & TDES2_VLAN_TAG_MASK); } -static int dwmac4_get_rx_header_len(struct dma_desc *p, unsigned int *len) +static void dwmac4_get_rx_header_len(struct dma_desc *p, unsigned int *len) { *len = le32_to_cpu(p->des2) & RDES2_HL; - return 0; } -static void dwmac4_set_sec_addr(struct dma_desc *p, dma_addr_t addr) +static void dwmac4_set_sec_addr(struct dma_desc *p, dma_addr_t addr, bool buf2_valid) { p->des2 = cpu_to_le32(lower_32_bits(addr)); - p->des3 = cpu_to_le32(upper_32_bits(addr) | RDES3_BUFFER2_VALID_ADDR); + p->des3 = cpu_to_le32(upper_32_bits(addr)); + + if (buf2_valid) + p->des3 |= cpu_to_le32(RDES3_BUFFER2_VALID_ADDR); + else + p->des3 &= cpu_to_le32(~RDES3_BUFFER2_VALID_ADDR); } static void dwmac4_set_tbs(struct dma_edesc *p, u32 sec, u32 nsec) @@ -535,7 +570,6 @@ const struct stmmac_desc_ops dwmac4_desc_ops = { .init_tx_desc = dwmac4_rd_init_tx_desc, .display_ring = dwmac4_display_ring, .set_mss = dwmac4_set_mss_ctxt, - .get_addr = dwmac4_get_addr, .set_addr = dwmac4_set_addr, .clear = dwmac4_clear, .set_sarc = dwmac4_set_sarc, diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c index bb29bfcd62c3..d99fa028c646 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c @@ -124,6 +124,23 @@ static void dwmac4_dma_init_channel(void __iomem *ioaddr, ioaddr + DMA_CHAN_INTR_ENA(chan)); } +static void dwmac410_dma_init_channel(void __iomem *ioaddr, + struct stmmac_dma_cfg *dma_cfg, u32 chan) +{ + u32 value; + + /* common channel control register config */ + value = readl(ioaddr + DMA_CHAN_CONTROL(chan)); + if (dma_cfg->pblx8) + value = value | DMA_BUS_MODE_PBL; + + writel(value, ioaddr + DMA_CHAN_CONTROL(chan)); + + /* Mask interrupts by writing to CSR7 */ + writel(DMA_CHAN_INTR_DEFAULT_MASK_4_10, + ioaddr + DMA_CHAN_INTR_ENA(chan)); +} + static void dwmac4_dma_init(void __iomem *ioaddr, struct stmmac_dma_cfg *dma_cfg, int atds) { @@ -144,6 +161,19 @@ static void dwmac4_dma_init(void __iomem *ioaddr, value |= DMA_SYS_BUS_EAME; writel(value, ioaddr + DMA_SYS_BUS_MODE); + + value = readl(ioaddr + DMA_BUS_MODE); + + if (dma_cfg->multi_msi_en) { + value &= ~DMA_BUS_MODE_INTM_MASK; + value |= (DMA_BUS_MODE_INTM_MODE1 << DMA_BUS_MODE_INTM_SHIFT); + } + + if (dma_cfg->dche) + value |= DMA_BUS_MODE_DCHE; + + writel(value, ioaddr + DMA_BUS_MODE); + } static void _dwmac4_dump_dma_regs(void __iomem *ioaddr, u32 channel, @@ -193,19 +223,16 @@ static void dwmac4_dump_dma_regs(void __iomem *ioaddr, u32 *reg_space) _dwmac4_dump_dma_regs(ioaddr, i, reg_space); } -static void dwmac4_rx_watchdog(void __iomem *ioaddr, u32 riwt, u32 number_chan) +static void dwmac4_rx_watchdog(void __iomem *ioaddr, u32 riwt, u32 queue) { - u32 chan; - - for (chan = 0; chan < number_chan; chan++) - writel(riwt, ioaddr + DMA_CHAN_RX_WATCHDOG(chan)); + writel(riwt, ioaddr + DMA_CHAN_RX_WATCHDOG(queue)); } static void dwmac4_dma_rx_chan_op_mode(void __iomem *ioaddr, int mode, u32 channel, int fifosz, u8 qmode) { unsigned int rqs = fifosz / 256 - 1; - u32 mtl_rx_op, mtl_rx_int; + u32 mtl_rx_op; mtl_rx_op = readl(ioaddr + MTL_CHAN_RX_OP_MODE(channel)); @@ -266,11 +293,6 @@ static void dwmac4_dma_rx_chan_op_mode(void __iomem *ioaddr, int mode, } writel(mtl_rx_op, ioaddr + MTL_CHAN_RX_OP_MODE(channel)); - - /* Enable MTL RX overflow */ - mtl_rx_int = readl(ioaddr + MTL_CHAN_INT_CTRL(channel)); - writel(mtl_rx_int | MTL_RX_OVERFLOW_INT_EN, - ioaddr + MTL_CHAN_INT_CTRL(channel)); } static void dwmac4_dma_tx_chan_op_mode(void __iomem *ioaddr, int mode, @@ -325,8 +347,8 @@ static void dwmac4_dma_tx_chan_op_mode(void __iomem *ioaddr, int mode, writel(mtl_tx_op, ioaddr + MTL_CHAN_TX_OP_MODE(channel)); } -static void dwmac4_get_hw_feature(void __iomem *ioaddr, - struct dma_features *dma_cap) +static int dwmac4_get_hw_feature(void __iomem *ioaddr, + struct dma_features *dma_cap) { u32 hw_cap = readl(ioaddr + GMAC_HW_FEATURE0); @@ -398,6 +420,8 @@ static void dwmac4_get_hw_feature(void __iomem *ioaddr, /* IEEE 1588-2002 */ dma_cap->time_stamp = 0; + /* Number of Auxiliary Snapshot Inputs */ + dma_cap->aux_snapshot_n = (hw_cap & GMAC_HW_FEAT_AUXSNAPNUM) >> 28; /* MAC HW feature3 */ hw_cap = readl(ioaddr + GMAC_HW_FEATURE3); @@ -413,6 +437,8 @@ static void dwmac4_get_hw_feature(void __iomem *ioaddr, dma_cap->frpbs = (hw_cap & GMAC_HW_FEAT_FRPBS) >> 11; dma_cap->frpsel = (hw_cap & GMAC_HW_FEAT_FRPSEL) >> 10; dma_cap->dvlan = (hw_cap & GMAC_HW_FEAT_DVLAN) >> 5; + + return 0; } /* Enable/disable TSO feature and set MSS */ @@ -523,7 +549,7 @@ const struct stmmac_dma_ops dwmac4_dma_ops = { const struct stmmac_dma_ops dwmac410_dma_ops = { .reset = dwmac4_dma_reset, .init = dwmac4_dma_init, - .init_chan = dwmac4_dma_init_channel, + .init_chan = dwmac410_dma_init_channel, .init_rx_chan = dwmac4_dma_init_rx_chan, .init_tx_chan = dwmac4_dma_init_tx_chan, .axi = dwmac4_dma_axi, diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h index 8391ca63d943..9321879b599c 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h @@ -25,6 +25,10 @@ #define DMA_TBS_CTRL 0x00001050 /* DMA Bus Mode bitmap */ +#define DMA_BUS_MODE_DCHE BIT(19) +#define DMA_BUS_MODE_INTM_MASK GENMASK(17, 16) +#define DMA_BUS_MODE_INTM_SHIFT 16 +#define DMA_BUS_MODE_INTM_MODE1 0x1 #define DMA_BUS_MODE_SFT_RESET BIT(0) /* DMA SYS Bus Mode bitmap */ @@ -149,6 +153,25 @@ #define DMA_CHAN_STATUS_TPS BIT(1) #define DMA_CHAN_STATUS_TI BIT(0) +#define DMA_CHAN_STATUS_MSK_COMMON (DMA_CHAN_STATUS_NIS | \ + DMA_CHAN_STATUS_AIS | \ + DMA_CHAN_STATUS_CDE | \ + DMA_CHAN_STATUS_FBE) + +#define DMA_CHAN_STATUS_MSK_RX (DMA_CHAN_STATUS_REB | \ + DMA_CHAN_STATUS_ERI | \ + DMA_CHAN_STATUS_RWT | \ + DMA_CHAN_STATUS_RPS | \ + DMA_CHAN_STATUS_RBU | \ + DMA_CHAN_STATUS_RI | \ + DMA_CHAN_STATUS_MSK_COMMON) + +#define DMA_CHAN_STATUS_MSK_TX (DMA_CHAN_STATUS_ETI | \ + DMA_CHAN_STATUS_TBU | \ + DMA_CHAN_STATUS_TPS | \ + DMA_CHAN_STATUS_TI | \ + DMA_CHAN_STATUS_MSK_COMMON) + /* Interrupt enable bits per channel */ #define DMA_CHAN_INTR_ENA_NIE BIT(16) #define DMA_CHAN_INTR_ENA_AIE BIT(15) @@ -206,7 +229,7 @@ void dwmac4_dma_stop_tx(void __iomem *ioaddr, u32 chan); void dwmac4_dma_start_rx(void __iomem *ioaddr, u32 chan); void dwmac4_dma_stop_rx(void __iomem *ioaddr, u32 chan); int dwmac4_dma_interrupt(void __iomem *ioaddr, - struct stmmac_extra_stats *x, u32 chan); + struct stmmac_extra_stats *x, u32 chan, u32 dir); void dwmac4_set_rx_ring_len(void __iomem *ioaddr, u32 len, u32 chan); void dwmac4_set_tx_ring_len(void __iomem *ioaddr, u32 len, u32 chan); void dwmac4_set_rx_tail_ptr(void __iomem *ioaddr, u32 tail_ptr, u32 chan); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c index 9becca280074..d1c605777985 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c @@ -6,6 +6,7 @@ */ #include <linux/io.h> +#include <linux/iopoll.h> #include <linux/delay.h> #include "common.h" #include "dwmac4_dma.h" @@ -14,22 +15,14 @@ int dwmac4_dma_reset(void __iomem *ioaddr) { u32 value = readl(ioaddr + DMA_BUS_MODE); - int limit; /* DMA SW reset */ value |= DMA_BUS_MODE_SFT_RESET; writel(value, ioaddr + DMA_BUS_MODE); - limit = 10; - while (limit--) { - if (!(readl(ioaddr + DMA_BUS_MODE) & DMA_BUS_MODE_SFT_RESET)) - break; - mdelay(10); - } - - if (limit < 0) - return -EBUSY; - return 0; + return readl_poll_timeout(ioaddr + DMA_BUS_MODE, value, + !(value & DMA_BUS_MODE_SFT_RESET), + 10000, 1000000); } void dwmac4_set_rx_tail_ptr(void __iomem *ioaddr, u32 tail_ptr, u32 chan) @@ -60,10 +53,6 @@ void dwmac4_dma_stop_tx(void __iomem *ioaddr, u32 chan) value &= ~DMA_CONTROL_ST; writel(value, ioaddr + DMA_CHAN_TX_CONTROL(chan)); - - value = readl(ioaddr + GMAC_CONFIG); - value &= ~GMAC_CONFIG_TE; - writel(value, ioaddr + GMAC_CONFIG); } void dwmac4_dma_start_rx(void __iomem *ioaddr, u32 chan) @@ -146,12 +135,17 @@ void dwmac410_disable_dma_irq(void __iomem *ioaddr, u32 chan, bool rx, bool tx) } int dwmac4_dma_interrupt(void __iomem *ioaddr, - struct stmmac_extra_stats *x, u32 chan) + struct stmmac_extra_stats *x, u32 chan, u32 dir) { u32 intr_status = readl(ioaddr + DMA_CHAN_STATUS(chan)); u32 intr_en = readl(ioaddr + DMA_CHAN_INTR_ENA(chan)); int ret = 0; + if (dir == DMA_DIR_RX) + intr_status &= DMA_CHAN_STATUS_MSK_RX; + else if (dir == DMA_DIR_TX) + intr_status &= DMA_CHAN_STATUS_MSK_TX; + /* ABNORMAL interrupts */ if (unlikely(intr_status & DMA_CHAN_STATUS_AIS)) { if (unlikely(intr_status & DMA_CHAN_STATUS_RBU)) @@ -172,26 +166,28 @@ int dwmac4_dma_interrupt(void __iomem *ioaddr, } } /* TX/RX NORMAL interrupts */ - if (likely(intr_status & DMA_CHAN_STATUS_NIS)) { + if (likely(intr_status & DMA_CHAN_STATUS_NIS)) x->normal_irq_n++; - if (likely(intr_status & DMA_CHAN_STATUS_RI)) { - x->rx_normal_irq_n++; - ret |= handle_rx; - } - if (likely(intr_status & (DMA_CHAN_STATUS_TI | - DMA_CHAN_STATUS_TBU))) { - x->tx_normal_irq_n++; - ret |= handle_tx; - } - if (unlikely(intr_status & DMA_CHAN_STATUS_ERI)) - x->rx_early_irq++; + if (likely(intr_status & DMA_CHAN_STATUS_RI)) { + x->rx_normal_irq_n++; + x->rxq_stats[chan].rx_normal_irq_n++; + ret |= handle_rx; + } + if (likely(intr_status & DMA_CHAN_STATUS_TI)) { + x->tx_normal_irq_n++; + x->txq_stats[chan].tx_normal_irq_n++; + ret |= handle_tx; } + if (unlikely(intr_status & DMA_CHAN_STATUS_TBU)) + ret |= handle_tx; + if (unlikely(intr_status & DMA_CHAN_STATUS_ERI)) + x->rx_early_irq++; writel(intr_status & intr_en, ioaddr + DMA_CHAN_STATUS(chan)); return ret; } -void stmmac_dwmac4_set_mac_addr(void __iomem *ioaddr, u8 addr[6], +void stmmac_dwmac4_set_mac_addr(void __iomem *ioaddr, const u8 addr[6], unsigned int high, unsigned int low) { unsigned long data; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac5.c b/drivers/net/ethernet/stmicro/stmmac/dwmac5.c index 494c859b4ade..9c2d40f853ed 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac5.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac5.c @@ -183,7 +183,8 @@ static void dwmac5_handle_dma_err(struct net_device *ndev, STAT_OFF(dma_errors), stats); } -int dwmac5_safety_feat_config(void __iomem *ioaddr, unsigned int asp) +int dwmac5_safety_feat_config(void __iomem *ioaddr, unsigned int asp, + struct stmmac_safety_feature_cfg *safety_feat_cfg) { u32 value; @@ -192,11 +193,17 @@ int dwmac5_safety_feat_config(void __iomem *ioaddr, unsigned int asp) /* 1. Enable Safety Features */ value = readl(ioaddr + MTL_ECC_CONTROL); - value |= TSOEE; /* TSO ECC */ - value |= MRXPEE; /* MTL RX Parser ECC */ - value |= MESTEE; /* MTL EST ECC */ - value |= MRXEE; /* MTL RX FIFO ECC */ - value |= MTXEE; /* MTL TX FIFO ECC */ + value |= MEEAO; /* MTL ECC Error Addr Status Override */ + if (safety_feat_cfg->tsoee) + value |= TSOEE; /* TSO ECC */ + if (safety_feat_cfg->mrxpee) + value |= MRXPEE; /* MTL RX Parser ECC */ + if (safety_feat_cfg->mestee) + value |= MESTEE; /* MTL EST ECC */ + if (safety_feat_cfg->mrxee) + value |= MRXEE; /* MTL RX FIFO ECC */ + if (safety_feat_cfg->mtxee) + value |= MTXEE; /* MTL TX FIFO ECC */ writel(value, ioaddr + MTL_ECC_CONTROL); /* 2. Enable MTL Safety Interrupts */ @@ -218,13 +225,16 @@ int dwmac5_safety_feat_config(void __iomem *ioaddr, unsigned int asp) /* 5. Enable Parity and Timeout for FSM */ value = readl(ioaddr + MAC_FSM_CONTROL); - value |= PRTYEN; /* FSM Parity Feature */ - value |= TMOUTEN; /* FSM Timeout Feature */ + if (safety_feat_cfg->prtyen) + value |= PRTYEN; /* FSM Parity Feature */ + if (safety_feat_cfg->tmouten) + value |= TMOUTEN; /* FSM Timeout Feature */ writel(value, ioaddr + MAC_FSM_CONTROL); /* 4. Enable Data Parity Protection */ value = readl(ioaddr + MTL_DPP_CONTROL); - value |= EDPP; + if (safety_feat_cfg->edpp) + value |= EDPP; writel(value, ioaddr + MTL_DPP_CONTROL); /* @@ -234,7 +244,8 @@ int dwmac5_safety_feat_config(void __iomem *ioaddr, unsigned int asp) if (asp <= 0x2) return 0; - value |= EPSI; + if (safety_feat_cfg->epsi) + value |= EPSI; writel(value, ioaddr + MTL_DPP_CONTROL); return 0; } @@ -305,17 +316,13 @@ int dwmac5_safety_feat_dump(struct stmmac_safety_stats *stats, static int dwmac5_rxp_disable(void __iomem *ioaddr) { u32 val; - int ret; val = readl(ioaddr + MTL_OPERATION_MODE); val &= ~MTL_FRPE; writel(val, ioaddr + MTL_OPERATION_MODE); - ret = readl_poll_timeout(ioaddr + MTL_RXP_CONTROL_STATUS, val, + return readl_poll_timeout(ioaddr + MTL_RXP_CONTROL_STATUS, val, val & RXPI, 1, 10000); - if (ret) - return ret; - return 0; } static void dwmac5_rxp_enable(void __iomem *ioaddr) @@ -572,68 +579,24 @@ static int dwmac5_est_write(void __iomem *ioaddr, u32 reg, u32 val, bool gcl) int dwmac5_est_configure(void __iomem *ioaddr, struct stmmac_est *cfg, unsigned int ptp_rate) { - u32 speed, total_offset, offset, ctrl, ctr_low; - u32 extcfg = readl(ioaddr + GMAC_EXT_CONFIG); - u32 mac_cfg = readl(ioaddr + GMAC_CONFIG); int i, ret = 0x0; - u64 total_ctr; - - if (extcfg & GMAC_CONFIG_EIPG_EN) { - offset = (extcfg & GMAC_CONFIG_EIPG) >> GMAC_CONFIG_EIPG_SHIFT; - offset = 104 + (offset * 8); - } else { - offset = (mac_cfg & GMAC_CONFIG_IPG) >> GMAC_CONFIG_IPG_SHIFT; - offset = 96 - (offset * 8); - } - - speed = mac_cfg & (GMAC_CONFIG_PS | GMAC_CONFIG_FES); - speed = speed >> GMAC_CONFIG_FES_SHIFT; - - switch (speed) { - case 0x0: - offset = offset * 1000; /* 1G */ - break; - case 0x1: - offset = offset * 400; /* 2.5G */ - break; - case 0x2: - offset = offset * 100000; /* 10M */ - break; - case 0x3: - offset = offset * 10000; /* 100M */ - break; - default: - return -EINVAL; - } - - offset = offset / 1000; + u32 ctrl; ret |= dwmac5_est_write(ioaddr, BTR_LOW, cfg->btr[0], false); ret |= dwmac5_est_write(ioaddr, BTR_HIGH, cfg->btr[1], false); ret |= dwmac5_est_write(ioaddr, TER, cfg->ter, false); ret |= dwmac5_est_write(ioaddr, LLR, cfg->gcl_size, false); + ret |= dwmac5_est_write(ioaddr, CTR_LOW, cfg->ctr[0], false); + ret |= dwmac5_est_write(ioaddr, CTR_HIGH, cfg->ctr[1], false); if (ret) return ret; - total_offset = 0; for (i = 0; i < cfg->gcl_size; i++) { - ret = dwmac5_est_write(ioaddr, i, cfg->gcl[i] + offset, true); + ret = dwmac5_est_write(ioaddr, i, cfg->gcl[i], true); if (ret) return ret; - - total_offset += offset; } - total_ctr = cfg->ctr[0] + cfg->ctr[1] * 1000000000; - total_ctr += total_offset; - - ctr_low = do_div(total_ctr, 1000000000); - - ret |= dwmac5_est_write(ioaddr, CTR_LOW, ctr_low, false); - ret |= dwmac5_est_write(ioaddr, CTR_HIGH, total_ctr, false); - if (ret) - return ret; - ctrl = readl(ioaddr + MTL_EST_CONTROL); ctrl &= ~PTOV; ctrl |= ((1000000000 / ptp_rate) * 6) << PTOV_SHIFT; @@ -643,9 +606,95 @@ int dwmac5_est_configure(void __iomem *ioaddr, struct stmmac_est *cfg, ctrl &= ~EEST; writel(ctrl, ioaddr + MTL_EST_CONTROL); + + /* Configure EST interrupt */ + if (cfg->enable) + ctrl = (IECGCE | IEHS | IEHF | IEBE | IECC); + else + ctrl = 0; + + writel(ctrl, ioaddr + MTL_EST_INT_EN); + return 0; } +void dwmac5_est_irq_status(void __iomem *ioaddr, struct net_device *dev, + struct stmmac_extra_stats *x, u32 txqcnt) +{ + u32 status, value, feqn, hbfq, hbfs, btrl; + u32 txqcnt_mask = (1 << txqcnt) - 1; + + status = readl(ioaddr + MTL_EST_STATUS); + + value = (CGCE | HLBS | HLBF | BTRE | SWLC); + + /* Return if there is no error */ + if (!(status & value)) + return; + + if (status & CGCE) { + /* Clear Interrupt */ + writel(CGCE, ioaddr + MTL_EST_STATUS); + + x->mtl_est_cgce++; + } + + if (status & HLBS) { + value = readl(ioaddr + MTL_EST_SCH_ERR); + value &= txqcnt_mask; + + x->mtl_est_hlbs++; + + /* Clear Interrupt */ + writel(value, ioaddr + MTL_EST_SCH_ERR); + + /* Collecting info to shows all the queues that has HLBS + * issue. The only way to clear this is to clear the + * statistic + */ + if (net_ratelimit()) + netdev_err(dev, "EST: HLB(sched) Queue 0x%x\n", value); + } + + if (status & HLBF) { + value = readl(ioaddr + MTL_EST_FRM_SZ_ERR); + feqn = value & txqcnt_mask; + + value = readl(ioaddr + MTL_EST_FRM_SZ_CAP); + hbfq = (value & SZ_CAP_HBFQ_MASK(txqcnt)) >> SZ_CAP_HBFQ_SHIFT; + hbfs = value & SZ_CAP_HBFS_MASK; + + x->mtl_est_hlbf++; + + /* Clear Interrupt */ + writel(feqn, ioaddr + MTL_EST_FRM_SZ_ERR); + + if (net_ratelimit()) + netdev_err(dev, "EST: HLB(size) Queue %u Size %u\n", + hbfq, hbfs); + } + + if (status & BTRE) { + if ((status & BTRL) == BTRL_MAX) + x->mtl_est_btrlm++; + else + x->mtl_est_btre++; + + btrl = (status & BTRL) >> BTRL_SHIFT; + + if (net_ratelimit()) + netdev_info(dev, "EST: BTR Error Loop Count %u\n", + btrl); + + writel(BTRE, ioaddr + MTL_EST_STATUS); + } + + if (status & SWLC) { + writel(SWLC, ioaddr + MTL_EST_STATUS); + netdev_info(dev, "EST: SWOL has been switched\n"); + } +} + void dwmac5_fpe_configure(void __iomem *ioaddr, u32 num_txq, u32 num_rxq, bool enable) { @@ -669,3 +718,52 @@ void dwmac5_fpe_configure(void __iomem *ioaddr, u32 num_txq, u32 num_rxq, value |= EFPE; writel(value, ioaddr + MAC_FPE_CTRL_STS); } + +int dwmac5_fpe_irq_status(void __iomem *ioaddr, struct net_device *dev) +{ + u32 value; + int status; + + status = FPE_EVENT_UNKNOWN; + + value = readl(ioaddr + MAC_FPE_CTRL_STS); + + if (value & TRSP) { + status |= FPE_EVENT_TRSP; + netdev_info(dev, "FPE: Respond mPacket is transmitted\n"); + } + + if (value & TVER) { + status |= FPE_EVENT_TVER; + netdev_info(dev, "FPE: Verify mPacket is transmitted\n"); + } + + if (value & RRSP) { + status |= FPE_EVENT_RRSP; + netdev_info(dev, "FPE: Respond mPacket is received\n"); + } + + if (value & RVER) { + status |= FPE_EVENT_RVER; + netdev_info(dev, "FPE: Verify mPacket is received\n"); + } + + return status; +} + +void dwmac5_fpe_send_mpacket(void __iomem *ioaddr, enum stmmac_mpacket_type type) +{ + u32 value; + + value = readl(ioaddr + MAC_FPE_CTRL_STS); + + if (type == MPACKET_VERIFY) { + value &= ~SRSP; + value |= SVER; + } else { + value &= ~SVER; + value |= SRSP; + } + + writel(value, ioaddr + MAC_FPE_CTRL_STS); +} diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac5.h b/drivers/net/ethernet/stmicro/stmmac/dwmac5.h index 3e8faa96b4d4..53c138d0ff48 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac5.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac5.h @@ -12,6 +12,12 @@ #define TMOUTEN BIT(0) #define MAC_FPE_CTRL_STS 0x00000234 +#define TRSP BIT(19) +#define TVER BIT(18) +#define RRSP BIT(17) +#define RVER BIT(16) +#define SRSP BIT(2) +#define SVER BIT(1) #define EFPE BIT(0) #define MAC_PPS_CONTROL 0x00000b70 @@ -38,6 +44,36 @@ #define PTOV_SHIFT 24 #define SSWL BIT(1) #define EEST BIT(0) + +#define MTL_EST_STATUS 0x00000c58 +#define BTRL GENMASK(11, 8) +#define BTRL_SHIFT 8 +#define BTRL_MAX (0xF << BTRL_SHIFT) +#define SWOL BIT(7) +#define SWOL_SHIFT 7 +#define CGCE BIT(4) +#define HLBS BIT(3) +#define HLBF BIT(2) +#define BTRE BIT(1) +#define SWLC BIT(0) + +#define MTL_EST_SCH_ERR 0x00000c60 +#define MTL_EST_FRM_SZ_ERR 0x00000c64 +#define MTL_EST_FRM_SZ_CAP 0x00000c68 +#define SZ_CAP_HBFS_MASK GENMASK(14, 0) +#define SZ_CAP_HBFQ_SHIFT 16 +#define SZ_CAP_HBFQ_MASK(_val) ({ typeof(_val) (val) = (_val); \ + ((val) > 4 ? GENMASK(18, 16) : \ + (val) > 2 ? GENMASK(17, 16) : \ + BIT(16)); }) + +#define MTL_EST_INT_EN 0x00000c70 +#define IECGCE CGCE +#define IEHS HLBS +#define IEHF HLBF +#define IEBE BTRE +#define IECC SWLC + #define MTL_EST_GCL_CONTROL 0x00000c80 #define BTR_LOW 0x0 #define BTR_HIGH 0x1 @@ -62,6 +98,7 @@ #define ADDR GENMASK(15, 0) #define MTL_RXP_IACC_DATA 0x00000cb4 #define MTL_ECC_CONTROL 0x00000cc0 +#define MEEAO BIT(8) #define TSOEE BIT(4) #define MRXPEE BIT(3) #define MESTEE BIT(2) @@ -92,7 +129,16 @@ #define TCEIE BIT(0) #define DMA_ECC_INT_STATUS 0x00001088 -int dwmac5_safety_feat_config(void __iomem *ioaddr, unsigned int asp); +/* EQoS version 5.xx VLAN Tag Filter Fail Packets Queuing */ +#define GMAC_RXQ_CTRL4 0x00000094 +#define GMAC_RXQCTRL_VFFQ_MASK GENMASK(19, 17) +#define GMAC_RXQCTRL_VFFQ_SHIFT 17 +#define GMAC_RXQCTRL_VFFQE BIT(16) + +#define GMAC_INT_FPE_EN BIT(17) + +int dwmac5_safety_feat_config(void __iomem *ioaddr, unsigned int asp, + struct stmmac_safety_feature_cfg *safety_cfg); int dwmac5_safety_feat_irq_status(struct net_device *ndev, void __iomem *ioaddr, unsigned int asp, struct stmmac_safety_stats *stats); @@ -105,7 +151,12 @@ int dwmac5_flex_pps_config(void __iomem *ioaddr, int index, u32 sub_second_inc, u32 systime_flags); int dwmac5_est_configure(void __iomem *ioaddr, struct stmmac_est *cfg, unsigned int ptp_rate); +void dwmac5_est_irq_status(void __iomem *ioaddr, struct net_device *dev, + struct stmmac_extra_stats *x, u32 txqcnt); void dwmac5_fpe_configure(void __iomem *ioaddr, u32 num_txq, u32 num_rxq, bool enable); +void dwmac5_fpe_send_mpacket(void __iomem *ioaddr, + enum stmmac_mpacket_type type); +int dwmac5_fpe_irq_status(void __iomem *ioaddr, struct net_device *dev); #endif /* __DWMAC5_H__ */ diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac_dma.h b/drivers/net/ethernet/stmicro/stmmac/dwmac_dma.h index e5dbd0bc257e..acd70b9a3173 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac_dma.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac_dma.h @@ -128,8 +128,29 @@ #define DMA_STATUS_TI 0x00000001 /* Transmit Interrupt */ #define DMA_CONTROL_FTF 0x00100000 /* Flush transmit FIFO */ +#define DMA_STATUS_MSK_COMMON (DMA_STATUS_NIS | \ + DMA_STATUS_AIS | \ + DMA_STATUS_FBI) + +#define DMA_STATUS_MSK_RX (DMA_STATUS_ERI | \ + DMA_STATUS_RWT | \ + DMA_STATUS_RPS | \ + DMA_STATUS_RU | \ + DMA_STATUS_RI | \ + DMA_STATUS_OVF | \ + DMA_STATUS_MSK_COMMON) + +#define DMA_STATUS_MSK_TX (DMA_STATUS_ETI | \ + DMA_STATUS_UNF | \ + DMA_STATUS_TJT | \ + DMA_STATUS_TU | \ + DMA_STATUS_TPS | \ + DMA_STATUS_TI | \ + DMA_STATUS_MSK_COMMON) + #define NUM_DWMAC100_DMA_REGS 9 #define NUM_DWMAC1000_DMA_REGS 23 +#define NUM_DWMAC4_DMA_REGS 27 void dwmac_enable_dma_transmission(void __iomem *ioaddr); void dwmac_enable_dma_irq(void __iomem *ioaddr, u32 chan, bool rx, bool tx); @@ -139,7 +160,7 @@ void dwmac_dma_stop_tx(void __iomem *ioaddr, u32 chan); void dwmac_dma_start_rx(void __iomem *ioaddr, u32 chan); void dwmac_dma_stop_rx(void __iomem *ioaddr, u32 chan); int dwmac_dma_interrupt(void __iomem *ioaddr, struct stmmac_extra_stats *x, - u32 chan); + u32 chan, u32 dir); int dwmac_dma_reset(void __iomem *ioaddr); #endif /* __DWMAC_DMA_H__ */ diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac_lib.c b/drivers/net/ethernet/stmicro/stmmac/dwmac_lib.c index 688d36095333..9b6138b11776 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac_lib.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac_lib.c @@ -16,19 +16,14 @@ int dwmac_dma_reset(void __iomem *ioaddr) { u32 value = readl(ioaddr + DMA_BUS_MODE); - int err; /* DMA SW reset */ value |= DMA_BUS_MODE_SFT_RESET; writel(value, ioaddr + DMA_BUS_MODE); - err = readl_poll_timeout(ioaddr + DMA_BUS_MODE, value, + return readl_poll_timeout(ioaddr + DMA_BUS_MODE, value, !(value & DMA_BUS_MODE_SFT_RESET), - 10000, 100000); - if (err) - return -EBUSY; - - return 0; + 10000, 200000); } /* CSR1 enables the transmit DMA to check for new descriptor */ @@ -160,7 +155,7 @@ static void show_rx_process_state(unsigned int status) #endif int dwmac_dma_interrupt(void __iomem *ioaddr, - struct stmmac_extra_stats *x, u32 chan) + struct stmmac_extra_stats *x, u32 chan, u32 dir) { int ret = 0; /* read the status register (CSR5) */ @@ -172,6 +167,12 @@ int dwmac_dma_interrupt(void __iomem *ioaddr, show_tx_process_state(intr_status); show_rx_process_state(intr_status); #endif + + if (dir == DMA_DIR_RX) + intr_status &= DMA_STATUS_MSK_RX; + else if (dir == DMA_DIR_TX) + intr_status &= DMA_STATUS_MSK_TX; + /* ABNORMAL interrupts */ if (unlikely(intr_status & DMA_STATUS_AIS)) { if (unlikely(intr_status & DMA_STATUS_UNF)) { @@ -238,7 +239,7 @@ void dwmac_dma_flush_tx_fifo(void __iomem *ioaddr) do {} while ((readl(ioaddr + DMA_CONTROL) & DMA_CONTROL_FTF)); } -void stmmac_set_mac_addr(void __iomem *ioaddr, u8 addr[6], +void stmmac_set_mac_addr(void __iomem *ioaddr, const u8 addr[6], unsigned int high, unsigned int low) { unsigned long data; @@ -257,14 +258,18 @@ EXPORT_SYMBOL_GPL(stmmac_set_mac_addr); /* Enable disable MAC RX/TX */ void stmmac_set_mac(void __iomem *ioaddr, bool enable) { - u32 value = readl(ioaddr + MAC_CTRL_REG); + u32 old_val, value; + + old_val = readl(ioaddr + MAC_CTRL_REG); + value = old_val; if (enable) value |= MAC_ENABLE_RX | MAC_ENABLE_TX; else value &= ~(MAC_ENABLE_TX | MAC_ENABLE_RX); - writel(value, ioaddr + MAC_CTRL_REG); + if (value != old_val) + writel(value, ioaddr + MAC_CTRL_REG); } void stmmac_get_mac_addr(void __iomem *ioaddr, unsigned char *addr, diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2.h b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2.h index 6c3b8a950f58..1913385df685 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2.h @@ -412,6 +412,12 @@ #define XGMAC_TI BIT(0) #define XGMAC_REGSIZE ((0x0000317c + (0x80 * 15)) / 4) +#define XGMAC_DMA_STATUS_MSK_COMMON (XGMAC_NIS | XGMAC_AIS | XGMAC_FBE) +#define XGMAC_DMA_STATUS_MSK_RX (XGMAC_RBU | XGMAC_RI | \ + XGMAC_DMA_STATUS_MSK_COMMON) +#define XGMAC_DMA_STATUS_MSK_TX (XGMAC_TBU | XGMAC_TPS | XGMAC_TI | \ + XGMAC_DMA_STATUS_MSK_COMMON) + /* Descriptors */ #define XGMAC_TDES0_LTV BIT(31) #define XGMAC_TDES0_LT GENMASK(7, 0) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c index 67b754a56288..c6c4d7948fe5 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c @@ -9,6 +9,7 @@ #include <linux/iopoll.h> #include "stmmac.h" #include "stmmac_ptp.h" +#include "dwxlgmac2.h" #include "dwxgmac2.h" static void dwxgmac2_core_init(struct mac_device_info *hw, @@ -334,7 +335,8 @@ static void dwxgmac2_pmt(struct mac_device_info *hw, unsigned long mode) } static void dwxgmac2_set_umac_addr(struct mac_device_info *hw, - unsigned char *addr, unsigned int reg_n) + const unsigned char *addr, + unsigned int reg_n) { void __iomem *ioaddr = hw->pcsr; u32 value; @@ -576,8 +578,13 @@ static void dwxgmac2_update_vlan_hash(struct mac_device_info *hw, u32 hash, value |= XGMAC_VLAN_EDVLP; value |= XGMAC_VLAN_ESVL; value |= XGMAC_VLAN_DOVLTC; + } else { + value &= ~XGMAC_VLAN_EDVLP; + value &= ~XGMAC_VLAN_ESVL; + value &= ~XGMAC_VLAN_DOVLTC; } + value &= ~XGMAC_VLAN_VID; writel(value, ioaddr + XGMAC_VLAN_TAG); } else if (perfect_match) { u32 value = readl(ioaddr + XGMAC_PACKET_FILTER); @@ -588,13 +595,19 @@ static void dwxgmac2_update_vlan_hash(struct mac_device_info *hw, u32 hash, value = readl(ioaddr + XGMAC_VLAN_TAG); + value &= ~XGMAC_VLAN_VTHM; value |= XGMAC_VLAN_ETV; if (is_double) { value |= XGMAC_VLAN_EDVLP; value |= XGMAC_VLAN_ESVL; value |= XGMAC_VLAN_DOVLTC; + } else { + value &= ~XGMAC_VLAN_EDVLP; + value &= ~XGMAC_VLAN_ESVL; + value &= ~XGMAC_VLAN_DOVLTC; } + value &= ~XGMAC_VLAN_VID; writel(value | perfect_match, ioaddr + XGMAC_VLAN_TAG); } else { u32 value = readl(ioaddr + XGMAC_PACKET_FILTER); @@ -789,7 +802,9 @@ static void dwxgmac3_handle_dma_err(struct net_device *ndev, dwxgmac3_dma_errors, STAT_OFF(dma_errors), stats); } -static int dwxgmac3_safety_feat_config(void __iomem *ioaddr, unsigned int asp) +static int +dwxgmac3_safety_feat_config(void __iomem *ioaddr, unsigned int asp, + struct stmmac_safety_feature_cfg *safety_cfg) { u32 value; @@ -1485,6 +1500,67 @@ const struct stmmac_ops dwxgmac210_ops = { .fpe_configure = dwxgmac3_fpe_configure, }; +static void dwxlgmac2_rx_queue_enable(struct mac_device_info *hw, u8 mode, + u32 queue) +{ + void __iomem *ioaddr = hw->pcsr; + u32 value; + + value = readl(ioaddr + XLGMAC_RXQ_ENABLE_CTRL0) & ~XGMAC_RXQEN(queue); + if (mode == MTL_QUEUE_AVB) + value |= 0x1 << XGMAC_RXQEN_SHIFT(queue); + else if (mode == MTL_QUEUE_DCB) + value |= 0x2 << XGMAC_RXQEN_SHIFT(queue); + writel(value, ioaddr + XLGMAC_RXQ_ENABLE_CTRL0); +} + +const struct stmmac_ops dwxlgmac2_ops = { + .core_init = dwxgmac2_core_init, + .set_mac = dwxgmac2_set_mac, + .rx_ipc = dwxgmac2_rx_ipc, + .rx_queue_enable = dwxlgmac2_rx_queue_enable, + .rx_queue_prio = dwxgmac2_rx_queue_prio, + .tx_queue_prio = dwxgmac2_tx_queue_prio, + .rx_queue_routing = NULL, + .prog_mtl_rx_algorithms = dwxgmac2_prog_mtl_rx_algorithms, + .prog_mtl_tx_algorithms = dwxgmac2_prog_mtl_tx_algorithms, + .set_mtl_tx_queue_weight = dwxgmac2_set_mtl_tx_queue_weight, + .map_mtl_to_dma = dwxgmac2_map_mtl_to_dma, + .config_cbs = dwxgmac2_config_cbs, + .dump_regs = dwxgmac2_dump_regs, + .host_irq_status = dwxgmac2_host_irq_status, + .host_mtl_irq_status = dwxgmac2_host_mtl_irq_status, + .flow_ctrl = dwxgmac2_flow_ctrl, + .pmt = dwxgmac2_pmt, + .set_umac_addr = dwxgmac2_set_umac_addr, + .get_umac_addr = dwxgmac2_get_umac_addr, + .set_eee_mode = dwxgmac2_set_eee_mode, + .reset_eee_mode = dwxgmac2_reset_eee_mode, + .set_eee_timer = dwxgmac2_set_eee_timer, + .set_eee_pls = dwxgmac2_set_eee_pls, + .pcs_ctrl_ane = NULL, + .pcs_rane = NULL, + .pcs_get_adv_lp = NULL, + .debug = NULL, + .set_filter = dwxgmac2_set_filter, + .safety_feat_config = dwxgmac3_safety_feat_config, + .safety_feat_irq_status = dwxgmac3_safety_feat_irq_status, + .safety_feat_dump = dwxgmac3_safety_feat_dump, + .set_mac_loopback = dwxgmac2_set_mac_loopback, + .rss_configure = dwxgmac2_rss_configure, + .update_vlan_hash = dwxgmac2_update_vlan_hash, + .rxp_config = dwxgmac3_rxp_config, + .get_mac_tx_timestamp = dwxgmac2_get_mac_tx_timestamp, + .flex_pps_config = dwxgmac2_flex_pps_config, + .sarc_configure = dwxgmac2_sarc_configure, + .enable_vlan = dwxgmac2_enable_vlan, + .config_l3_filter = dwxgmac2_config_l3_filter, + .config_l4_filter = dwxgmac2_config_l4_filter, + .set_arp_offload = dwxgmac2_set_arp_offload, + .est_configure = dwxgmac3_est_configure, + .fpe_configure = dwxgmac3_fpe_configure, +}; + int dwxgmac2_setup(struct stmmac_priv *priv) { struct mac_device_info *mac = priv->hw; @@ -1521,3 +1597,40 @@ int dwxgmac2_setup(struct stmmac_priv *priv) return 0; } + +int dwxlgmac2_setup(struct stmmac_priv *priv) +{ + struct mac_device_info *mac = priv->hw; + + dev_info(priv->device, "\tXLGMAC\n"); + + priv->dev->priv_flags |= IFF_UNICAST_FLT; + mac->pcsr = priv->ioaddr; + mac->multicast_filter_bins = priv->plat->multicast_filter_bins; + mac->unicast_filter_entries = priv->plat->unicast_filter_entries; + mac->mcast_bits_log2 = 0; + + if (mac->multicast_filter_bins) + mac->mcast_bits_log2 = ilog2(mac->multicast_filter_bins); + + mac->link.duplex = 0; + mac->link.speed1000 = XLGMAC_CONFIG_SS_1000; + mac->link.speed2500 = XLGMAC_CONFIG_SS_2500; + mac->link.xgmii.speed10000 = XLGMAC_CONFIG_SS_10G; + mac->link.xlgmii.speed25000 = XLGMAC_CONFIG_SS_25G; + mac->link.xlgmii.speed40000 = XLGMAC_CONFIG_SS_40G; + mac->link.xlgmii.speed50000 = XLGMAC_CONFIG_SS_50G; + mac->link.xlgmii.speed100000 = XLGMAC_CONFIG_SS_100G; + mac->link.speed_mask = XLGMAC_CONFIG_SS; + + mac->mii.addr = XGMAC_MDIO_ADDR; + mac->mii.data = XGMAC_MDIO_DATA; + mac->mii.addr_shift = 16; + mac->mii.addr_mask = GENMASK(20, 16); + mac->mii.reg_shift = 0; + mac->mii.reg_mask = GENMASK(15, 0); + mac->mii.clk_csr_shift = 19; + mac->mii.clk_csr_mask = GENMASK(21, 19); + + return 0; +} diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c index c3d654cfa9ef..b1f0c3984a09 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c @@ -239,11 +239,6 @@ static void dwxgmac2_set_mss(struct dma_desc *p, unsigned int mss) p->des3 = cpu_to_le32(XGMAC_TDES3_CTXT | XGMAC_TDES3_TCMSSV); } -static void dwxgmac2_get_addr(struct dma_desc *p, unsigned int *addr) -{ - *addr = le32_to_cpu(p->des0); -} - static void dwxgmac2_set_addr(struct dma_desc *p, dma_addr_t addr) { p->des0 = cpu_to_le32(lower_32_bits(addr)); @@ -286,14 +281,13 @@ static int dwxgmac2_get_rx_hash(struct dma_desc *p, u32 *hash, return -EINVAL; } -static int dwxgmac2_get_rx_header_len(struct dma_desc *p, unsigned int *len) +static void dwxgmac2_get_rx_header_len(struct dma_desc *p, unsigned int *len) { if (le32_to_cpu(p->des3) & XGMAC_RDES3_L34T) *len = le32_to_cpu(p->des2) & XGMAC_RDES2_HL; - return 0; } -static void dwxgmac2_set_sec_addr(struct dma_desc *p, dma_addr_t addr) +static void dwxgmac2_set_sec_addr(struct dma_desc *p, dma_addr_t addr, bool is_valid) { p->des2 = cpu_to_le32(lower_32_bits(addr)); p->des3 = cpu_to_le32(upper_32_bits(addr)); @@ -367,7 +361,6 @@ const struct stmmac_desc_ops dwxgmac210_desc_ops = { .init_rx_desc = dwxgmac2_init_rx_desc, .init_tx_desc = dwxgmac2_init_tx_desc, .set_mss = dwxgmac2_set_mss, - .get_addr = dwxgmac2_get_addr, .set_addr = dwxgmac2_set_addr, .clear = dwxgmac2_clear, .get_rx_hash = dwxgmac2_get_rx_hash, diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c index 77308c5c5d29..5e98355f422b 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c @@ -323,12 +323,18 @@ static void dwxgmac2_dma_stop_rx(void __iomem *ioaddr, u32 chan) } static int dwxgmac2_dma_interrupt(void __iomem *ioaddr, - struct stmmac_extra_stats *x, u32 chan) + struct stmmac_extra_stats *x, u32 chan, + u32 dir) { u32 intr_status = readl(ioaddr + XGMAC_DMA_CH_STATUS(chan)); u32 intr_en = readl(ioaddr + XGMAC_DMA_CH_INT_EN(chan)); int ret = 0; + if (dir == DMA_DIR_RX) + intr_status &= XGMAC_DMA_STATUS_MSK_RX; + else if (dir == DMA_DIR_TX) + intr_status &= XGMAC_DMA_STATUS_MSK_TX; + /* ABNORMAL interrupts */ if (unlikely(intr_status & XGMAC_AIS)) { if (unlikely(intr_status & XGMAC_RBU)) { @@ -365,8 +371,8 @@ static int dwxgmac2_dma_interrupt(void __iomem *ioaddr, return ret; } -static void dwxgmac2_get_hw_feature(void __iomem *ioaddr, - struct dma_features *dma_cap) +static int dwxgmac2_get_hw_feature(void __iomem *ioaddr, + struct dma_features *dma_cap) { u32 hw_cap; @@ -439,14 +445,13 @@ static void dwxgmac2_get_hw_feature(void __iomem *ioaddr, dma_cap->frpes = (hw_cap & XGMAC_HWFEAT_FRPES) >> 11; dma_cap->frpbs = (hw_cap & XGMAC_HWFEAT_FRPPB) >> 9; dma_cap->frpsel = (hw_cap & XGMAC_HWFEAT_FRPSEL) >> 3; + + return 0; } -static void dwxgmac2_rx_watchdog(void __iomem *ioaddr, u32 riwt, u32 nchan) +static void dwxgmac2_rx_watchdog(void __iomem *ioaddr, u32 riwt, u32 queue) { - u32 i; - - for (i = 0; i < nchan; i++) - writel(riwt & XGMAC_RWT, ioaddr + XGMAC_DMA_CH_Rx_WATCHDOG(i)); + writel(riwt & XGMAC_RWT, ioaddr + XGMAC_DMA_CH_Rx_WATCHDOG(queue)); } static void dwxgmac2_set_rx_ring_len(void __iomem *ioaddr, u32 len, u32 chan) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxlgmac2.h b/drivers/net/ethernet/stmicro/stmmac/dwxlgmac2.h new file mode 100644 index 000000000000..726090d49221 --- /dev/null +++ b/drivers/net/ethernet/stmicro/stmmac/dwxlgmac2.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2020 Synopsys, Inc. and/or its affiliates. + * Synopsys DesignWare XLGMAC definitions. + */ + +#ifndef __STMMAC_DWXLGMAC2_H__ +#define __STMMAC_DWXLGMAC2_H__ + +/* MAC Registers */ +#define XLGMAC_CONFIG_SS GENMASK(30, 28) +#define XLGMAC_CONFIG_SS_SHIFT 28 +#define XLGMAC_CONFIG_SS_40G (0x0 << XLGMAC_CONFIG_SS_SHIFT) +#define XLGMAC_CONFIG_SS_25G (0x1 << XLGMAC_CONFIG_SS_SHIFT) +#define XLGMAC_CONFIG_SS_50G (0x2 << XLGMAC_CONFIG_SS_SHIFT) +#define XLGMAC_CONFIG_SS_100G (0x3 << XLGMAC_CONFIG_SS_SHIFT) +#define XLGMAC_CONFIG_SS_10G (0x4 << XLGMAC_CONFIG_SS_SHIFT) +#define XLGMAC_CONFIG_SS_2500 (0x6 << XLGMAC_CONFIG_SS_SHIFT) +#define XLGMAC_CONFIG_SS_1000 (0x7 << XLGMAC_CONFIG_SS_SHIFT) +#define XLGMAC_RXQ_ENABLE_CTRL0 0x00000140 + +#endif /* __STMMAC_DWXLGMAC2_H__ */ diff --git a/drivers/net/ethernet/stmicro/stmmac/enh_desc.c b/drivers/net/ethernet/stmicro/stmmac/enh_desc.c index d02cec296f51..1bcbbd724fb5 100644 --- a/drivers/net/ethernet/stmicro/stmmac/enh_desc.c +++ b/drivers/net/ethernet/stmicro/stmmac/enh_desc.c @@ -417,19 +417,22 @@ static int enh_desc_get_rx_timestamp_status(void *desc, void *next_desc, } } -static void enh_desc_display_ring(void *head, unsigned int size, bool rx) +static void enh_desc_display_ring(void *head, unsigned int size, bool rx, + dma_addr_t dma_rx_phy, unsigned int desc_size) { struct dma_extended_desc *ep = (struct dma_extended_desc *)head; + dma_addr_t dma_addr; int i; pr_info("Extended %s descriptor ring:\n", rx ? "RX" : "TX"); for (i = 0; i < size; i++) { u64 x; + dma_addr = dma_rx_phy + i * sizeof(*ep); x = *(u64 *)ep; - pr_info("%03d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n", - i, (unsigned int)virt_to_phys(ep), + pr_info("%03d [%pad]: 0x%x 0x%x 0x%x 0x%x\n", + i, &dma_addr, (unsigned int)x, (unsigned int)(x >> 32), ep->basic.des2, ep->basic.des3); ep++; @@ -437,11 +440,6 @@ static void enh_desc_display_ring(void *head, unsigned int size, bool rx) pr_info("\n"); } -static void enh_desc_get_addr(struct dma_desc *p, unsigned int *addr) -{ - *addr = le32_to_cpu(p->des2); -} - static void enh_desc_set_addr(struct dma_desc *p, dma_addr_t addr) { p->des2 = cpu_to_le32(addr); @@ -472,7 +470,6 @@ const struct stmmac_desc_ops enh_desc_ops = { .get_timestamp = enh_desc_get_timestamp, .get_rx_timestamp_status = enh_desc_get_rx_timestamp_status, .display_ring = enh_desc_display_ring, - .get_addr = enh_desc_get_addr, .set_addr = enh_desc_set_addr, .clear = enh_desc_clear, }; diff --git a/drivers/net/ethernet/stmicro/stmmac/hwif.c b/drivers/net/ethernet/stmicro/stmmac/hwif.c index 3af2e5015245..bb7114f970f8 100644 --- a/drivers/net/ethernet/stmicro/stmmac/hwif.c +++ b/drivers/net/ethernet/stmicro/stmmac/hwif.c @@ -23,6 +23,18 @@ static u32 stmmac_get_id(struct stmmac_priv *priv, u32 id_reg) return reg & GENMASK(7, 0); } +static u32 stmmac_get_dev_id(struct stmmac_priv *priv, u32 id_reg) +{ + u32 reg = readl(priv->ioaddr + id_reg); + + if (!reg) { + dev_info(priv->device, "Version ID not available\n"); + return 0x0; + } + + return (reg & GENMASK(15, 8)) >> 8; +} + static void stmmac_dwmac_mode_quirk(struct stmmac_priv *priv) { struct mac_device_info *mac = priv->hw; @@ -69,11 +81,18 @@ static int stmmac_dwmac4_quirks(struct stmmac_priv *priv) return 0; } +static int stmmac_dwxlgmac_quirks(struct stmmac_priv *priv) +{ + priv->hw->xlgmac = true; + return 0; +} + static const struct stmmac_hwif_entry { bool gmac; bool gmac4; bool xgmac; u32 min_id; + u32 dev_id; const struct stmmac_regs_off regs; const void *desc; const void *dma; @@ -199,6 +218,7 @@ static const struct stmmac_hwif_entry { .gmac4 = false, .xgmac = true, .min_id = DWXGMAC_CORE_2_10, + .dev_id = DWXGMAC_ID, .regs = { .ptp_off = PTP_XGMAC_OFFSET, .mmc_off = MMC_XGMAC_OFFSET, @@ -212,6 +232,25 @@ static const struct stmmac_hwif_entry { .mmc = &dwxgmac_mmc_ops, .setup = dwxgmac2_setup, .quirks = NULL, + }, { + .gmac = false, + .gmac4 = false, + .xgmac = true, + .min_id = DWXLGMAC_CORE_2_00, + .dev_id = DWXLGMAC_ID, + .regs = { + .ptp_off = PTP_XGMAC_OFFSET, + .mmc_off = MMC_XGMAC_OFFSET, + }, + .desc = &dwxgmac210_desc_ops, + .dma = &dwxgmac210_dma_ops, + .mac = &dwxlgmac2_ops, + .hwtimestamp = &stmmac_ptp, + .mode = NULL, + .tc = &dwmac510_tc_ops, + .mmc = &dwxgmac_mmc_ops, + .setup = dwxlgmac2_setup, + .quirks = stmmac_dwxlgmac_quirks, }, }; @@ -223,13 +262,15 @@ int stmmac_hwif_init(struct stmmac_priv *priv) const struct stmmac_hwif_entry *entry; struct mac_device_info *mac; bool needs_setup = true; + u32 id, dev_id = 0; int i, ret; - u32 id; if (needs_gmac) { id = stmmac_get_id(priv, GMAC_VERSION); } else if (needs_gmac4 || needs_xgmac) { id = stmmac_get_id(priv, GMAC4_VERSION); + if (needs_xgmac) + dev_id = stmmac_get_dev_id(priv, GMAC4_VERSION); } else { id = 0; } @@ -267,6 +308,8 @@ int stmmac_hwif_init(struct stmmac_priv *priv) /* Use synopsys_id var because some setups can override this */ if (priv->synopsys_id < entry->min_id) continue; + if (needs_xgmac && (dev_id ^ entry->dev_id)) + continue; /* Only use generic HW helpers if needed */ mac->desc = mac->desc ? : entry->desc; diff --git a/drivers/net/ethernet/stmicro/stmmac/hwif.h b/drivers/net/ethernet/stmicro/stmmac/hwif.h index df63b0367aff..592b4067f9b8 100644 --- a/drivers/net/ethernet/stmicro/stmmac/hwif.h +++ b/drivers/net/ethernet/stmicro/stmmac/hwif.h @@ -78,11 +78,10 @@ struct stmmac_desc_ops { /* get rx timestamp status */ int (*get_rx_timestamp_status)(void *desc, void *next_desc, u32 ats); /* Display ring */ - void (*display_ring)(void *head, unsigned int size, bool rx); + void (*display_ring)(void *head, unsigned int size, bool rx, + dma_addr_t dma_rx_phy, unsigned int desc_size); /* set MSS via context descriptor */ void (*set_mss)(struct dma_desc *p, unsigned int mss); - /* get descriptor skbuff address */ - void (*get_addr)(struct dma_desc *p, unsigned int *addr); /* set descriptor skbuff address */ void (*set_addr)(struct dma_desc *p, dma_addr_t addr); /* clear descriptor */ @@ -90,8 +89,8 @@ struct stmmac_desc_ops { /* RSS */ int (*get_rx_hash)(struct dma_desc *p, u32 *hash, enum pkt_hash_types *type); - int (*get_rx_header_len)(struct dma_desc *p, unsigned int *len); - void (*set_sec_addr)(struct dma_desc *p, dma_addr_t addr); + void (*get_rx_header_len)(struct dma_desc *p, unsigned int *len); + void (*set_sec_addr)(struct dma_desc *p, dma_addr_t addr, bool buf2_valid); void (*set_sarc)(struct dma_desc *p, u32 sarc_type); void (*set_vlan_tag)(struct dma_desc *p, u16 tag, u16 inner_tag, u32 inner_type); @@ -141,8 +140,6 @@ struct stmmac_desc_ops { stmmac_do_void_callback(__priv, desc, display_ring, __args) #define stmmac_set_mss(__priv, __args...) \ stmmac_do_void_callback(__priv, desc, set_mss, __args) -#define stmmac_get_desc_addr(__priv, __args...) \ - stmmac_do_void_callback(__priv, desc, get_addr, __args) #define stmmac_set_desc_addr(__priv, __args...) \ stmmac_do_void_callback(__priv, desc, set_addr, __args) #define stmmac_clear_desc(__priv, __args...) \ @@ -150,7 +147,7 @@ struct stmmac_desc_ops { #define stmmac_get_rx_hash(__priv, __args...) \ stmmac_do_callback(__priv, desc, get_rx_hash, __args) #define stmmac_get_rx_header_len(__priv, __args...) \ - stmmac_do_callback(__priv, desc, get_rx_header_len, __args) + stmmac_do_void_callback(__priv, desc, get_rx_header_len, __args) #define stmmac_set_desc_sec_addr(__priv, __args...) \ stmmac_do_void_callback(__priv, desc, set_sec_addr, __args) #define stmmac_set_desc_sarc(__priv, __args...) \ @@ -200,12 +197,12 @@ struct stmmac_dma_ops { void (*start_rx)(void __iomem *ioaddr, u32 chan); void (*stop_rx)(void __iomem *ioaddr, u32 chan); int (*dma_interrupt) (void __iomem *ioaddr, - struct stmmac_extra_stats *x, u32 chan); + struct stmmac_extra_stats *x, u32 chan, u32 dir); /* If supported then get the optional core features */ - void (*get_hw_feature)(void __iomem *ioaddr, - struct dma_features *dma_cap); + int (*get_hw_feature)(void __iomem *ioaddr, + struct dma_features *dma_cap); /* Program the HW RX Watchdog */ - void (*rx_watchdog)(void __iomem *ioaddr, u32 riwt, u32 number_chan); + void (*rx_watchdog)(void __iomem *ioaddr, u32 riwt, u32 queue); void (*set_tx_ring_len)(void __iomem *ioaddr, u32 len, u32 chan); void (*set_rx_ring_len)(void __iomem *ioaddr, u32 len, u32 chan); void (*set_rx_tail_ptr)(void __iomem *ioaddr, u32 tail_ptr, u32 chan); @@ -254,7 +251,7 @@ struct stmmac_dma_ops { #define stmmac_dma_interrupt_status(__priv, __args...) \ stmmac_do_callback(__priv, dma, dma_interrupt, __args) #define stmmac_get_hw_feature(__priv, __args...) \ - stmmac_do_void_callback(__priv, dma, get_hw_feature, __args) + stmmac_do_callback(__priv, dma, get_hw_feature, __args) #define stmmac_rx_watchdog(__priv, __args...) \ stmmac_do_void_callback(__priv, dma, rx_watchdog, __args) #define stmmac_set_tx_ring_len(__priv, __args...) \ @@ -279,7 +276,6 @@ struct stmmac_dma_ops { struct mac_device_info; struct net_device; struct rgmii_adv; -struct stmmac_safety_stats; struct stmmac_tc_entry; struct stmmac_pps_cfg; struct stmmac_rss; @@ -330,13 +326,15 @@ struct stmmac_ops { /* Set power management mode (e.g. magic frame) */ void (*pmt)(struct mac_device_info *hw, unsigned long mode); /* Set/Get Unicast MAC addresses */ - void (*set_umac_addr)(struct mac_device_info *hw, unsigned char *addr, + void (*set_umac_addr)(struct mac_device_info *hw, + const unsigned char *addr, unsigned int reg_n); void (*get_umac_addr)(struct mac_device_info *hw, unsigned char *addr, unsigned int reg_n); void (*set_eee_mode)(struct mac_device_info *hw, bool en_tx_lpi_clockgating); void (*reset_eee_mode)(struct mac_device_info *hw); + void (*set_eee_lpi_entry_timer)(struct mac_device_info *hw, int et); void (*set_eee_timer)(struct mac_device_info *hw, int ls, int tw); void (*set_eee_pls)(struct mac_device_info *hw, int link); void (*debug)(void __iomem *ioaddr, struct stmmac_extra_stats *x, @@ -347,7 +345,8 @@ struct stmmac_ops { void (*pcs_rane)(void __iomem *ioaddr, bool restart); void (*pcs_get_adv_lp)(void __iomem *ioaddr, struct rgmii_adv *adv); /* Safety Features */ - int (*safety_feat_config)(void __iomem *ioaddr, unsigned int asp); + int (*safety_feat_config)(void __iomem *ioaddr, unsigned int asp, + struct stmmac_safety_feature_cfg *safety_cfg); int (*safety_feat_irq_status)(struct net_device *ndev, void __iomem *ioaddr, unsigned int asp, struct stmmac_safety_stats *stats); @@ -369,6 +368,14 @@ struct stmmac_ops { void (*update_vlan_hash)(struct mac_device_info *hw, u32 hash, __le16 perfect_match, bool is_double); void (*enable_vlan)(struct mac_device_info *hw, u32 type); + int (*add_hw_vlan_rx_fltr)(struct net_device *dev, + struct mac_device_info *hw, + __be16 proto, u16 vid); + int (*del_hw_vlan_rx_fltr)(struct net_device *dev, + struct mac_device_info *hw, + __be16 proto, u16 vid); + void (*restore_hw_vlan_rx_fltr)(struct net_device *dev, + struct mac_device_info *hw); /* TX Timestamp */ int (*get_mac_tx_timestamp)(struct mac_device_info *hw, u64 *ts); /* Source Address Insertion / Replacement */ @@ -383,8 +390,13 @@ struct stmmac_ops { void (*set_arp_offload)(struct mac_device_info *hw, bool en, u32 addr); int (*est_configure)(void __iomem *ioaddr, struct stmmac_est *cfg, unsigned int ptp_rate); + void (*est_irq_status)(void __iomem *ioaddr, struct net_device *dev, + struct stmmac_extra_stats *x, u32 txqcnt); void (*fpe_configure)(void __iomem *ioaddr, u32 num_txq, u32 num_rxq, bool enable); + void (*fpe_send_mpacket)(void __iomem *ioaddr, + enum stmmac_mpacket_type type); + int (*fpe_irq_status)(void __iomem *ioaddr, struct net_device *dev); }; #define stmmac_core_init(__priv, __args...) \ @@ -431,6 +443,8 @@ struct stmmac_ops { stmmac_do_void_callback(__priv, mac, set_eee_mode, __args) #define stmmac_reset_eee_mode(__priv, __args...) \ stmmac_do_void_callback(__priv, mac, reset_eee_mode, __args) +#define stmmac_set_eee_lpi_timer(__priv, __args...) \ + stmmac_do_void_callback(__priv, mac, set_eee_lpi_entry_timer, __args) #define stmmac_set_eee_timer(__priv, __args...) \ stmmac_do_void_callback(__priv, mac, set_eee_timer, __args) #define stmmac_set_eee_pls(__priv, __args...) \ @@ -461,6 +475,12 @@ struct stmmac_ops { stmmac_do_void_callback(__priv, mac, update_vlan_hash, __args) #define stmmac_enable_vlan(__priv, __args...) \ stmmac_do_void_callback(__priv, mac, enable_vlan, __args) +#define stmmac_add_hw_vlan_rx_fltr(__priv, __args...) \ + stmmac_do_callback(__priv, mac, add_hw_vlan_rx_fltr, __args) +#define stmmac_del_hw_vlan_rx_fltr(__priv, __args...) \ + stmmac_do_callback(__priv, mac, del_hw_vlan_rx_fltr, __args) +#define stmmac_restore_hw_vlan_rx_fltr(__priv, __args...) \ + stmmac_do_void_callback(__priv, mac, restore_hw_vlan_rx_fltr, __args) #define stmmac_get_mac_tx_timestamp(__priv, __args...) \ stmmac_do_callback(__priv, mac, get_mac_tx_timestamp, __args) #define stmmac_sarc_configure(__priv, __args...) \ @@ -473,8 +493,16 @@ struct stmmac_ops { stmmac_do_void_callback(__priv, mac, set_arp_offload, __args) #define stmmac_est_configure(__priv, __args...) \ stmmac_do_callback(__priv, mac, est_configure, __args) +#define stmmac_est_irq_status(__priv, __args...) \ + stmmac_do_void_callback(__priv, mac, est_irq_status, __args) #define stmmac_fpe_configure(__priv, __args...) \ stmmac_do_void_callback(__priv, mac, fpe_configure, __args) +#define stmmac_fpe_send_mpacket(__priv, __args...) \ + stmmac_do_void_callback(__priv, mac, fpe_send_mpacket, __args) +#define stmmac_fpe_irq_status(__priv, __args...) \ + stmmac_do_callback(__priv, mac, fpe_irq_status, __args) + +struct stmmac_priv; /* PTP and HW Timer helpers */ struct stmmac_hwtimestamp { @@ -486,6 +514,8 @@ struct stmmac_hwtimestamp { int (*adjust_systime) (void __iomem *ioaddr, u32 sec, u32 nsec, int add_sub, int gmac4); void (*get_systime) (void __iomem *ioaddr, u64 *systime); + void (*get_ptptime)(void __iomem *ioaddr, u64 *ptp_time); + void (*timestamp_interrupt)(struct stmmac_priv *priv); }; #define stmmac_config_hw_tstamping(__priv, __args...) \ @@ -500,6 +530,10 @@ struct stmmac_hwtimestamp { stmmac_do_callback(__priv, ptp, adjust_systime, __args) #define stmmac_get_systime(__priv, __args...) \ stmmac_do_void_callback(__priv, ptp, get_systime, __args) +#define stmmac_get_ptptime(__priv, __args...) \ + stmmac_do_void_callback(__priv, ptp, get_ptptime, __args) +#define stmmac_timestamp_interrupt(__priv, __args...) \ + stmmac_do_void_callback(__priv, ptp, timestamp_interrupt, __args) /* Helpers to manage the descriptors for chain and ring modes */ struct stmmac_mode_ops { @@ -528,7 +562,6 @@ struct stmmac_mode_ops { #define stmmac_clean_desc3(__priv, __args...) \ stmmac_do_void_callback(__priv, mode, clean_desc3, __args) -struct stmmac_priv; struct tc_cls_u32_offload; struct tc_cbs_qopt_offload; struct flow_cls_offload; @@ -593,6 +626,7 @@ extern const struct stmmac_dma_ops dwmac410_dma_ops; extern const struct stmmac_ops dwmac510_ops; extern const struct stmmac_tc_ops dwmac510_tc_ops; extern const struct stmmac_ops dwxgmac210_ops; +extern const struct stmmac_ops dwxlgmac2_ops; extern const struct stmmac_dma_ops dwxgmac210_dma_ops; extern const struct stmmac_desc_ops dwxgmac210_desc_ops; extern const struct stmmac_mmc_ops dwmac_mmc_ops; diff --git a/drivers/net/ethernet/stmicro/stmmac/mmc_core.c b/drivers/net/ethernet/stmicro/stmmac/mmc_core.c index a57b0fa815ab..ea4910ae0921 100644 --- a/drivers/net/ethernet/stmicro/stmmac/mmc_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/mmc_core.c @@ -197,7 +197,7 @@ static void dwmac_mmc_ctrl(void __iomem *mmcaddr, unsigned int mode) MMC_CNTRL, value); } -/* To mask all all interrupts.*/ +/* To mask all interrupts.*/ static void dwmac_mmc_intr_all_mask(void __iomem *mmcaddr) { writel(MMC_DEFAULT_MASK, mmcaddr + MMC_RX_INTR_MASK); diff --git a/drivers/net/ethernet/stmicro/stmmac/norm_desc.c b/drivers/net/ethernet/stmicro/stmmac/norm_desc.c index f083360e4ba6..e3da4da242ee 100644 --- a/drivers/net/ethernet/stmicro/stmmac/norm_desc.c +++ b/drivers/net/ethernet/stmicro/stmmac/norm_desc.c @@ -269,19 +269,22 @@ static int ndesc_get_rx_timestamp_status(void *desc, void *next_desc, u32 ats) return 1; } -static void ndesc_display_ring(void *head, unsigned int size, bool rx) +static void ndesc_display_ring(void *head, unsigned int size, bool rx, + dma_addr_t dma_rx_phy, unsigned int desc_size) { struct dma_desc *p = (struct dma_desc *)head; + dma_addr_t dma_addr; int i; pr_info("%s descriptor ring:\n", rx ? "RX" : "TX"); for (i = 0; i < size; i++) { u64 x; + dma_addr = dma_rx_phy + i * sizeof(*p); x = *(u64 *)p; - pr_info("%03d [0x%x]: 0x%x 0x%x 0x%x 0x%x", - i, (unsigned int)virt_to_phys(p), + pr_info("%03d [%pad]: 0x%x 0x%x 0x%x 0x%x", + i, &dma_addr, (unsigned int)x, (unsigned int)(x >> 32), p->des2, p->des3); p++; @@ -289,11 +292,6 @@ static void ndesc_display_ring(void *head, unsigned int size, bool rx) pr_info("\n"); } -static void ndesc_get_addr(struct dma_desc *p, unsigned int *addr) -{ - *addr = le32_to_cpu(p->des2); -} - static void ndesc_set_addr(struct dma_desc *p, dma_addr_t addr) { p->des2 = cpu_to_le32(addr); @@ -323,7 +321,6 @@ const struct stmmac_desc_ops ndesc_ops = { .get_timestamp = ndesc_get_timestamp, .get_rx_timestamp_status = ndesc_get_rx_timestamp_status, .display_ring = ndesc_display_ring, - .get_addr = ndesc_get_addr, .set_addr = ndesc_set_addr, .clear = ndesc_clear, }; diff --git a/drivers/net/ethernet/stmicro/stmmac/ring_mode.c b/drivers/net/ethernet/stmicro/stmmac/ring_mode.c index 14bd5e7b9875..2b5b17d8b8a0 100644 --- a/drivers/net/ethernet/stmicro/stmmac/ring_mode.c +++ b/drivers/net/ethernet/stmicro/stmmac/ring_mode.c @@ -51,7 +51,7 @@ static int jumbo_frm(void *p, struct sk_buff *skb, int csum) stmmac_prepare_tx_desc(priv, desc, 1, bmax, csum, STMMAC_RING_MODE, 0, false, skb->len); tx_q->tx_skbuff[entry] = NULL; - entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE); + entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); if (priv->extend_desc) desc = (struct dma_desc *)(tx_q->dma_etx + entry); @@ -107,7 +107,7 @@ static void refill_desc3(void *priv_ptr, struct dma_desc *p) struct stmmac_priv *priv = rx_q->priv_data; /* Fill DES3 in case of RING mode */ - if (priv->dma_buf_sz == BUF_SIZE_16KiB) + if (priv->dma_conf.dma_buf_sz == BUF_SIZE_16KiB) p->des3 = cpu_to_le32(le32_to_cpu(p->des2) + BUF_SIZE_8KiB); } diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac.h b/drivers/net/ethernet/stmicro/stmmac/stmmac.h index 9c02fc754bf1..bdbf86cb102a 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac.h +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac.h @@ -10,9 +10,9 @@ #define __STMMAC_H__ #define STMMAC_RESOURCE_NAME "stmmaceth" -#define DRV_MODULE_VERSION "Jan_2016" #include <linux/clk.h> +#include <linux/hrtimer.h> #include <linux/if_vlan.h> #include <linux/stmmac.h> #include <linux/phylink.h> @@ -22,13 +22,25 @@ #include <linux/net_tstamp.h> #include <linux/reset.h> #include <net/page_pool.h> +#include <uapi/linux/bpf.h> struct stmmac_resources { void __iomem *addr; - const char *mac; + u8 mac[ETH_ALEN]; int wol_irq; int lpi_irq; int irq; + int sfty_ce_irq; + int sfty_ue_irq; + int rx_irq[MTL_MAX_RX_QUEUES]; + int tx_irq[MTL_MAX_TX_QUEUES]; +}; + +enum stmmac_txbuf_type { + STMMAC_TXBUF_T_SKB, + STMMAC_TXBUF_T_XDP_TX, + STMMAC_TXBUF_T_XDP_NDO, + STMMAC_TXBUF_T_XSK_TX, }; struct stmmac_tx_info { @@ -37,6 +49,7 @@ struct stmmac_tx_info { unsigned len; bool last_segment; bool is_jumbo; + enum stmmac_txbuf_type buf_type; }; #define STMMAC_TBS_AVAIL BIT(0) @@ -46,31 +59,44 @@ struct stmmac_tx_info { struct stmmac_tx_queue { u32 tx_count_frames; int tbs; - struct timer_list txtimer; + struct hrtimer txtimer; u32 queue_index; struct stmmac_priv *priv_data; struct dma_extended_desc *dma_etx ____cacheline_aligned_in_smp; struct dma_edesc *dma_entx; struct dma_desc *dma_tx; - struct sk_buff **tx_skbuff; + union { + struct sk_buff **tx_skbuff; + struct xdp_frame **xdpf; + }; struct stmmac_tx_info *tx_skbuff_dma; + struct xsk_buff_pool *xsk_pool; + u32 xsk_frames_done; unsigned int cur_tx; unsigned int dirty_tx; dma_addr_t dma_tx_phy; - u32 tx_tail_addr; + dma_addr_t tx_tail_addr; u32 mss; }; struct stmmac_rx_buffer { - struct page *page; + union { + struct { + struct page *page; + dma_addr_t addr; + __u32 page_offset; + }; + struct xdp_buff *xdp; + }; struct page *sec_page; - dma_addr_t addr; dma_addr_t sec_addr; }; struct stmmac_rx_queue { u32 rx_count_frames; u32 queue_index; + struct xdp_rxq_info xdp_rxq; + struct xsk_buff_pool *xsk_pool; struct page_pool *page_pool; struct stmmac_rx_buffer *buf_pool; struct stmmac_priv *priv_data; @@ -78,6 +104,7 @@ struct stmmac_rx_queue { struct dma_desc *dma_rx ____cacheline_aligned_in_smp; unsigned int cur_rx; unsigned int dirty_rx; + unsigned int buf_alloc_num; u32 rx_zeroc_thresh; dma_addr_t dma_rx_phy; u32 rx_tail_addr; @@ -92,6 +119,7 @@ struct stmmac_rx_queue { struct stmmac_channel { struct napi_struct rx_napi ____cacheline_aligned_in_smp; struct napi_struct tx_napi ____cacheline_aligned_in_smp; + struct napi_struct rxtx_napi ____cacheline_aligned_in_smp; struct stmmac_priv *priv_data; spinlock_t lock; u32 index; @@ -144,22 +172,49 @@ struct stmmac_flow_entry { int is_l4; }; +/* Rx Frame Steering */ +enum stmmac_rfs_type { + STMMAC_RFS_T_VLAN, + STMMAC_RFS_T_LLDP, + STMMAC_RFS_T_1588, + STMMAC_RFS_T_MAX, +}; + +struct stmmac_rfs_entry { + unsigned long cookie; + u16 etype; + int in_use; + int type; + int tc; +}; + +struct stmmac_dma_conf { + unsigned int dma_buf_sz; + + /* RX Queue */ + struct stmmac_rx_queue rx_queue[MTL_MAX_RX_QUEUES]; + unsigned int dma_rx_size; + + /* TX Queue */ + struct stmmac_tx_queue tx_queue[MTL_MAX_TX_QUEUES]; + unsigned int dma_tx_size; +}; + struct stmmac_priv { /* Frequently used values are kept adjacent for cache effect */ - u32 tx_coal_frames; - u32 tx_coal_timer; - u32 rx_coal_frames; + u32 tx_coal_frames[MTL_MAX_TX_QUEUES]; + u32 tx_coal_timer[MTL_MAX_TX_QUEUES]; + u32 rx_coal_frames[MTL_MAX_TX_QUEUES]; - int tx_coalesce; int hwts_tx_en; bool tx_path_in_lpi_mode; bool tso; int sph; + int sph_cap; u32 sarc_type; - unsigned int dma_buf_sz; unsigned int rx_copybreak; - u32 rx_riwt; + u32 rx_riwt[MTL_MAX_TX_QUEUES]; int hwts_rx_en; void __iomem *ioaddr; @@ -169,11 +224,7 @@ struct stmmac_priv { int (*hwif_quirks)(struct stmmac_priv *priv); struct mutex lock; - /* RX Queue */ - struct stmmac_rx_queue rx_queue[MTL_MAX_RX_QUEUES]; - - /* TX Queue */ - struct stmmac_tx_queue tx_queue[MTL_MAX_TX_QUEUES]; + struct stmmac_dma_conf dma_conf; /* Generic channel for NAPI */ struct stmmac_channel channel[STMMAC_CH_MAX]; @@ -182,7 +233,6 @@ struct stmmac_priv { unsigned int flow_ctrl; unsigned int pause; struct mii_bus *mii; - int mii_irq[PHY_MAX_ADDR]; struct phylink_config phylink_config; struct phylink *phylink; @@ -203,6 +253,9 @@ struct stmmac_priv { int eee_enabled; int eee_active; int tx_lpi_timer; + int tx_lpi_enabled; + int eee_tw_timer; + bool eee_sw_timer_en; unsigned int mode; unsigned int chain_mode; int extend_desc; @@ -215,10 +268,26 @@ struct stmmac_priv { u32 adv_ts; int use_riwt; int irq_wake; - spinlock_t ptp_lock; + rwlock_t ptp_lock; + /* Protects auxiliary snapshot registers from concurrent access. */ + struct mutex aux_ts_lock; + wait_queue_head_t tstamp_busy_wait; + void __iomem *mmcaddr; void __iomem *ptpaddr; unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)]; + int sfty_ce_irq; + int sfty_ue_irq; + int rx_irq[MTL_MAX_RX_QUEUES]; + int tx_irq[MTL_MAX_TX_QUEUES]; + /*irq name */ + char int_name_mac[IFNAMSIZ + 9]; + char int_name_wol[IFNAMSIZ + 9]; + char int_name_lpi[IFNAMSIZ + 9]; + char int_name_sfty_ce[IFNAMSIZ + 10]; + char int_name_sfty_ue[IFNAMSIZ + 10]; + char int_name_rx_irq[MTL_MAX_TX_QUEUES][IFNAMSIZ + 14]; + char int_name_tx_irq[MTL_MAX_TX_QUEUES][IFNAMSIZ + 18]; #ifdef CONFIG_DEBUG_FS struct dentry *dbgfs_dir; @@ -228,18 +297,32 @@ struct stmmac_priv { struct workqueue_struct *wq; struct work_struct service_task; + /* Workqueue for handling FPE hand-shaking */ + unsigned long fpe_task_state; + struct workqueue_struct *fpe_wq; + struct work_struct fpe_task; + char wq_name[IFNAMSIZ + 4]; + /* TC Handling */ unsigned int tc_entries_max; unsigned int tc_off_max; struct stmmac_tc_entry *tc_entries; unsigned int flow_entries_max; struct stmmac_flow_entry *flow_entries; + unsigned int rfs_entries_max[STMMAC_RFS_T_MAX]; + unsigned int rfs_entries_cnt[STMMAC_RFS_T_MAX]; + unsigned int rfs_entries_total; + struct stmmac_rfs_entry *rfs_entries; /* Pulse Per Second output */ struct stmmac_pps_cfg pps[STMMAC_PPS_MAX]; /* Receive Side Scaling */ struct stmmac_rss rss; + + /* XDP BPF Program */ + unsigned long *af_xdp_zc_qps; + struct bpf_prog *xdp_prog; }; enum stmmac_state { @@ -252,10 +335,14 @@ enum stmmac_state { int stmmac_mdio_unregister(struct net_device *ndev); int stmmac_mdio_register(struct net_device *ndev); int stmmac_mdio_reset(struct mii_bus *mii); +int stmmac_xpcs_setup(struct mii_bus *mii); void stmmac_set_ethtool_ops(struct net_device *netdev); +int stmmac_init_tstamp_counter(struct stmmac_priv *priv, u32 systime_flags); void stmmac_ptp_register(struct stmmac_priv *priv); void stmmac_ptp_unregister(struct stmmac_priv *priv); +int stmmac_xdp_open(struct net_device *dev); +void stmmac_xdp_release(struct net_device *dev); int stmmac_resume(struct device *dev); int stmmac_suspend(struct device *dev); int stmmac_dvr_remove(struct device *dev); @@ -264,6 +351,32 @@ int stmmac_dvr_probe(struct device *device, struct stmmac_resources *res); void stmmac_disable_eee_mode(struct stmmac_priv *priv); bool stmmac_eee_init(struct stmmac_priv *priv); +int stmmac_reinit_queues(struct net_device *dev, u32 rx_cnt, u32 tx_cnt); +int stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size); +int stmmac_bus_clks_config(struct stmmac_priv *priv, bool enabled); +void stmmac_fpe_handshake(struct stmmac_priv *priv, bool enable); + +static inline bool stmmac_xdp_is_enabled(struct stmmac_priv *priv) +{ + return !!priv->xdp_prog; +} + +static inline unsigned int stmmac_rx_offset(struct stmmac_priv *priv) +{ + if (stmmac_xdp_is_enabled(priv)) + return XDP_PACKET_HEADROOM; + + return 0; +} + +void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue); +void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue); +void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue); +void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue); +int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags); +struct timespec64 stmmac_calc_tas_basetime(ktime_t old_base_time, + ktime_t current_time, + u64 cycle_time); #if IS_ENABLED(CONFIG_STMMAC_SELFTESTS) void stmmac_selftest_run(struct net_device *dev, diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c index b29603ec744c..f453b0d09366 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c @@ -21,10 +21,18 @@ #include "dwxgmac2.h" #define REG_SPACE_SIZE 0x1060 +#define GMAC4_REG_SPACE_SIZE 0x116C #define MAC100_ETHTOOL_NAME "st_mac100" #define GMAC_ETHTOOL_NAME "st_gmac" #define XGMAC_ETHTOOL_NAME "st_xgmac" +/* Same as DMA_CHAN_BASE_ADDR defined in dwmac4_dma.h + * + * It is here because dwmac_dma.h and dwmac4_dam.h can not be included at the + * same time due to the conflicting macro names. + */ +#define GMAC4_DMA_CHAN_BASE_ADDR 0x00001100 + #define ETHTOOL_DMA_OFFSET 55 struct stmmac_stats { @@ -158,6 +166,12 @@ static const struct stmmac_stats stmmac_gstrings_stats[] = { /* TSO */ STMMAC_STAT(tx_tso_frames), STMMAC_STAT(tx_tso_nfrags), + /* EST */ + STMMAC_STAT(mtl_est_cgce), + STMMAC_STAT(mtl_est_hlbs), + STMMAC_STAT(mtl_est_hlbf), + STMMAC_STAT(mtl_est_btre), + STMMAC_STAT(mtl_est_btrlm), }; #define STMMAC_STATS_LEN ARRAY_SIZE(stmmac_gstrings_stats) @@ -255,20 +269,35 @@ static const struct stmmac_stats stmmac_mmc[] = { }; #define STMMAC_MMC_STATS_LEN ARRAY_SIZE(stmmac_mmc) +static const char stmmac_qstats_tx_string[][ETH_GSTRING_LEN] = { + "tx_pkt_n", + "tx_irq_n", +#define STMMAC_TXQ_STATS ARRAY_SIZE(stmmac_qstats_tx_string) +}; + +static const char stmmac_qstats_rx_string[][ETH_GSTRING_LEN] = { + "rx_pkt_n", + "rx_irq_n", +#define STMMAC_RXQ_STATS ARRAY_SIZE(stmmac_qstats_rx_string) +}; + static void stmmac_ethtool_getdrvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { struct stmmac_priv *priv = netdev_priv(dev); if (priv->plat->has_gmac || priv->plat->has_gmac4) - strlcpy(info->driver, GMAC_ETHTOOL_NAME, sizeof(info->driver)); + strscpy(info->driver, GMAC_ETHTOOL_NAME, sizeof(info->driver)); else if (priv->plat->has_xgmac) - strlcpy(info->driver, XGMAC_ETHTOOL_NAME, sizeof(info->driver)); + strscpy(info->driver, XGMAC_ETHTOOL_NAME, sizeof(info->driver)); else - strlcpy(info->driver, MAC100_ETHTOOL_NAME, + strscpy(info->driver, MAC100_ETHTOOL_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_MODULE_VERSION, sizeof(info->version)); + if (priv->plat->pdev) { + strscpy(info->bus_info, pci_name(priv->plat->pdev), + sizeof(info->bus_info)); + } } static int stmmac_ethtool_get_link_ksettings(struct net_device *dev, @@ -413,6 +442,8 @@ static int stmmac_ethtool_get_regs_len(struct net_device *dev) if (priv->plat->has_xgmac) return XGMAC_REGSIZE * 4; + else if (priv->plat->has_gmac4) + return GMAC4_REG_SPACE_SIZE; return REG_SPACE_SIZE; } @@ -425,8 +456,13 @@ static void stmmac_ethtool_gregs(struct net_device *dev, stmmac_dump_mac_regs(priv, priv->hw, reg_space); stmmac_dump_dma_regs(priv, priv->ioaddr, reg_space); - if (!priv->plat->has_xgmac) { - /* Copy DMA registers to where ethtool expects them */ + /* Copy DMA registers to where ethtool expects them */ + if (priv->plat->has_gmac4) { + /* GMAC4 dumps its DMA registers at its DMA_CHAN_BASE_ADDR */ + memcpy(®_space[ETHTOOL_DMA_OFFSET], + ®_space[GMAC4_DMA_CHAN_BASE_ADDR / 4], + NUM_DWMAC4_DMA_REGS * 4); + } else if (!priv->plat->has_xgmac) { memcpy(®_space[ETHTOOL_DMA_OFFSET], ®_space[DMA_BUS_MODE / 4], NUM_DWMAC1000_DMA_REGS * 4); @@ -440,6 +476,37 @@ static int stmmac_nway_reset(struct net_device *dev) return phylink_ethtool_nway_reset(priv->phylink); } +static void stmmac_get_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) +{ + struct stmmac_priv *priv = netdev_priv(netdev); + + ring->rx_max_pending = DMA_MAX_RX_SIZE; + ring->tx_max_pending = DMA_MAX_TX_SIZE; + ring->rx_pending = priv->dma_conf.dma_rx_size; + ring->tx_pending = priv->dma_conf.dma_tx_size; +} + +static int stmmac_set_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) +{ + if (ring->rx_mini_pending || ring->rx_jumbo_pending || + ring->rx_pending < DMA_MIN_RX_SIZE || + ring->rx_pending > DMA_MAX_RX_SIZE || + !is_power_of_2(ring->rx_pending) || + ring->tx_pending < DMA_MIN_TX_SIZE || + ring->tx_pending > DMA_MAX_TX_SIZE || + !is_power_of_2(ring->tx_pending)) + return -EINVAL; + + return stmmac_reinit_ringparam(netdev, ring->rx_pending, + ring->tx_pending); +} + static void stmmac_get_pauseparam(struct net_device *netdev, struct ethtool_pauseparam *pause) @@ -473,6 +540,31 @@ stmmac_set_pauseparam(struct net_device *netdev, } } +static void stmmac_get_per_qstats(struct stmmac_priv *priv, u64 *data) +{ + u32 tx_cnt = priv->plat->tx_queues_to_use; + u32 rx_cnt = priv->plat->rx_queues_to_use; + int q, stat; + char *p; + + for (q = 0; q < tx_cnt; q++) { + p = (char *)priv + offsetof(struct stmmac_priv, + xstats.txq_stats[q].tx_pkt_n); + for (stat = 0; stat < STMMAC_TXQ_STATS; stat++) { + *data++ = (*(u64 *)p); + p += sizeof(u64 *); + } + } + for (q = 0; q < rx_cnt; q++) { + p = (char *)priv + offsetof(struct stmmac_priv, + xstats.rxq_stats[q].rx_pkt_n); + for (stat = 0; stat < STMMAC_RXQ_STATS; stat++) { + *data++ = (*(u64 *)p); + p += sizeof(u64 *); + } + } +} + static void stmmac_get_ethtool_stats(struct net_device *dev, struct ethtool_stats *dummy, u64 *data) { @@ -523,16 +615,21 @@ static void stmmac_get_ethtool_stats(struct net_device *dev, data[j++] = (stmmac_gstrings_stats[i].sizeof_stat == sizeof(u64)) ? (*(u64 *)p) : (*(u32 *)p); } + stmmac_get_per_qstats(priv, &data[j]); } static int stmmac_get_sset_count(struct net_device *netdev, int sset) { struct stmmac_priv *priv = netdev_priv(netdev); + u32 tx_cnt = priv->plat->tx_queues_to_use; + u32 rx_cnt = priv->plat->rx_queues_to_use; int i, len, safety_len = 0; switch (sset) { case ETH_SS_STATS: - len = STMMAC_STATS_LEN; + len = STMMAC_STATS_LEN + + STMMAC_TXQ_STATS * tx_cnt + + STMMAC_RXQ_STATS * rx_cnt; if (priv->dma_cap.rmon) len += STMMAC_MMC_STATS_LEN; @@ -555,6 +652,28 @@ static int stmmac_get_sset_count(struct net_device *netdev, int sset) } } +static void stmmac_get_qstats_string(struct stmmac_priv *priv, u8 *data) +{ + u32 tx_cnt = priv->plat->tx_queues_to_use; + u32 rx_cnt = priv->plat->rx_queues_to_use; + int q, stat; + + for (q = 0; q < tx_cnt; q++) { + for (stat = 0; stat < STMMAC_TXQ_STATS; stat++) { + snprintf(data, ETH_GSTRING_LEN, "q%d_%s", q, + stmmac_qstats_tx_string[stat]); + data += ETH_GSTRING_LEN; + } + } + for (q = 0; q < rx_cnt; q++) { + for (stat = 0; stat < STMMAC_RXQ_STATS; stat++) { + snprintf(data, ETH_GSTRING_LEN, "q%d_%s", q, + stmmac_qstats_rx_string[stat]); + data += ETH_GSTRING_LEN; + } + } +} + static void stmmac_get_strings(struct net_device *dev, u32 stringset, u8 *data) { int i; @@ -585,6 +704,7 @@ static void stmmac_get_strings(struct net_device *dev, u32 stringset, u8 *data) ETH_GSTRING_LEN); p += ETH_GSTRING_LEN; } + stmmac_get_qstats_string(priv, p); break; case ETH_SS_TEST: stmmac_selftest_get_strings(priv, p); @@ -600,9 +720,14 @@ static void stmmac_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol) { struct stmmac_priv *priv = netdev_priv(dev); + if (!priv->plat->pmt) + return phylink_ethtool_get_wol(priv->phylink, wol); + mutex_lock(&priv->lock); if (device_can_wakeup(priv->device)) { wol->supported = WAKE_MAGIC | WAKE_UCAST; + if (priv->hw_cap_support && !priv->dma_cap.pmt_magic_frame) + wol->supported &= ~WAKE_MAGIC; wol->wolopts = priv->wolopts; } mutex_unlock(&priv->lock); @@ -613,15 +738,23 @@ static int stmmac_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol) struct stmmac_priv *priv = netdev_priv(dev); u32 support = WAKE_MAGIC | WAKE_UCAST; + if (!device_can_wakeup(priv->device)) + return -EOPNOTSUPP; + + if (!priv->plat->pmt) { + int ret = phylink_ethtool_set_wol(priv->phylink, wol); + + if (!ret) + device_set_wakeup_enable(priv->device, !!wol->wolopts); + return ret; + } + /* By default almost all GMAC devices support the WoL via * magic frame but we can disable it if the HW capability * register shows no support for pmt_magic_frame. */ if ((priv->hw_cap_support) && (!priv->dma_cap.pmt_magic_frame)) wol->wolopts &= ~WAKE_MAGIC; - if (!device_can_wakeup(priv->device)) - return -EINVAL; - if (wol->wolopts & ~support) return -EINVAL; @@ -652,6 +785,7 @@ static int stmmac_ethtool_op_get_eee(struct net_device *dev, edata->eee_enabled = priv->eee_enabled; edata->eee_active = priv->eee_active; edata->tx_lpi_timer = priv->tx_lpi_timer; + edata->tx_lpi_enabled = priv->tx_lpi_enabled; return phylink_ethtool_get_eee(priv->phylink, edata); } @@ -662,24 +796,26 @@ static int stmmac_ethtool_op_set_eee(struct net_device *dev, struct stmmac_priv *priv = netdev_priv(dev); int ret; - if (!edata->eee_enabled) { + if (!priv->dma_cap.eee) + return -EOPNOTSUPP; + + if (priv->tx_lpi_enabled != edata->tx_lpi_enabled) + netdev_warn(priv->dev, + "Setting EEE tx-lpi is not supported\n"); + + if (!edata->eee_enabled) stmmac_disable_eee_mode(priv); - } else { - /* We are asking for enabling the EEE but it is safe - * to verify all by invoking the eee_init function. - * In case of failure it will return an error. - */ - edata->eee_enabled = stmmac_eee_init(priv); - if (!edata->eee_enabled) - return -EOPNOTSUPP; - } ret = phylink_ethtool_set_eee(priv->phylink, edata); if (ret) return ret; - priv->eee_enabled = edata->eee_enabled; - priv->tx_lpi_timer = edata->tx_lpi_timer; + if (edata->eee_enabled && + priv->tx_lpi_timer != edata->tx_lpi_timer) { + priv->tx_lpi_timer = edata->tx_lpi_timer; + stmmac_eee_init(priv); + } + return 0; } @@ -709,42 +845,77 @@ static u32 stmmac_riwt2usec(u32 riwt, struct stmmac_priv *priv) return (riwt * 256) / (clk / 1000000); } -static int stmmac_get_coalesce(struct net_device *dev, - struct ethtool_coalesce *ec) +static int __stmmac_get_coalesce(struct net_device *dev, + struct ethtool_coalesce *ec, + int queue) { struct stmmac_priv *priv = netdev_priv(dev); + u32 max_cnt; + u32 rx_cnt; + u32 tx_cnt; - ec->tx_coalesce_usecs = priv->tx_coal_timer; - ec->tx_max_coalesced_frames = priv->tx_coal_frames; + rx_cnt = priv->plat->rx_queues_to_use; + tx_cnt = priv->plat->tx_queues_to_use; + max_cnt = max(rx_cnt, tx_cnt); - if (priv->use_riwt) { - ec->rx_max_coalesced_frames = priv->rx_coal_frames; - ec->rx_coalesce_usecs = stmmac_riwt2usec(priv->rx_riwt, priv); + if (queue < 0) + queue = 0; + else if (queue >= max_cnt) + return -EINVAL; + + if (queue < tx_cnt) { + ec->tx_coalesce_usecs = priv->tx_coal_timer[queue]; + ec->tx_max_coalesced_frames = priv->tx_coal_frames[queue]; + } else { + ec->tx_coalesce_usecs = 0; + ec->tx_max_coalesced_frames = 0; + } + + if (priv->use_riwt && queue < rx_cnt) { + ec->rx_max_coalesced_frames = priv->rx_coal_frames[queue]; + ec->rx_coalesce_usecs = stmmac_riwt2usec(priv->rx_riwt[queue], + priv); + } else { + ec->rx_max_coalesced_frames = 0; + ec->rx_coalesce_usecs = 0; } return 0; } -static int stmmac_set_coalesce(struct net_device *dev, - struct ethtool_coalesce *ec) +static int stmmac_get_coalesce(struct net_device *dev, + struct ethtool_coalesce *ec, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) +{ + return __stmmac_get_coalesce(dev, ec, -1); +} + +static int stmmac_get_per_queue_coalesce(struct net_device *dev, u32 queue, + struct ethtool_coalesce *ec) +{ + return __stmmac_get_coalesce(dev, ec, queue); +} + +static int __stmmac_set_coalesce(struct net_device *dev, + struct ethtool_coalesce *ec, + int queue) { struct stmmac_priv *priv = netdev_priv(dev); - u32 rx_cnt = priv->plat->rx_queues_to_use; + bool all_queues = false; unsigned int rx_riwt; + u32 max_cnt; + u32 rx_cnt; + u32 tx_cnt; - /* Check not supported parameters */ - if ((ec->rx_coalesce_usecs_irq) || - (ec->rx_max_coalesced_frames_irq) || (ec->tx_coalesce_usecs_irq) || - (ec->use_adaptive_rx_coalesce) || (ec->use_adaptive_tx_coalesce) || - (ec->pkt_rate_low) || (ec->rx_coalesce_usecs_low) || - (ec->rx_max_coalesced_frames_low) || (ec->tx_coalesce_usecs_high) || - (ec->tx_max_coalesced_frames_low) || (ec->pkt_rate_high) || - (ec->tx_coalesce_usecs_low) || (ec->rx_coalesce_usecs_high) || - (ec->rx_max_coalesced_frames_high) || - (ec->tx_max_coalesced_frames_irq) || - (ec->stats_block_coalesce_usecs) || - (ec->tx_max_coalesced_frames_high) || (ec->rate_sample_interval)) - return -EOPNOTSUPP; + rx_cnt = priv->plat->rx_queues_to_use; + tx_cnt = priv->plat->tx_queues_to_use; + max_cnt = max(rx_cnt, tx_cnt); + + if (queue < 0) + all_queues = true; + else if (queue >= max_cnt) + return -EINVAL; if (priv->use_riwt && (ec->rx_coalesce_usecs > 0)) { rx_riwt = stmmac_usec2riwt(ec->rx_coalesce_usecs, priv); @@ -752,8 +923,23 @@ static int stmmac_set_coalesce(struct net_device *dev, if ((rx_riwt > MAX_DMA_RIWT) || (rx_riwt < MIN_DMA_RIWT)) return -EINVAL; - priv->rx_riwt = rx_riwt; - stmmac_rx_watchdog(priv, priv->ioaddr, priv->rx_riwt, rx_cnt); + if (all_queues) { + int i; + + for (i = 0; i < rx_cnt; i++) { + priv->rx_riwt[i] = rx_riwt; + stmmac_rx_watchdog(priv, priv->ioaddr, + rx_riwt, i); + priv->rx_coal_frames[i] = + ec->rx_max_coalesced_frames; + } + } else if (queue < rx_cnt) { + priv->rx_riwt[queue] = rx_riwt; + stmmac_rx_watchdog(priv, priv->ioaddr, + rx_riwt, queue); + priv->rx_coal_frames[queue] = + ec->rx_max_coalesced_frames; + } } if ((ec->tx_coalesce_usecs == 0) && @@ -764,13 +950,39 @@ static int stmmac_set_coalesce(struct net_device *dev, (ec->tx_max_coalesced_frames > STMMAC_TX_MAX_FRAMES)) return -EINVAL; - /* Only copy relevant parameters, ignore all others. */ - priv->tx_coal_frames = ec->tx_max_coalesced_frames; - priv->tx_coal_timer = ec->tx_coalesce_usecs; - priv->rx_coal_frames = ec->rx_max_coalesced_frames; + if (all_queues) { + int i; + + for (i = 0; i < tx_cnt; i++) { + priv->tx_coal_frames[i] = + ec->tx_max_coalesced_frames; + priv->tx_coal_timer[i] = + ec->tx_coalesce_usecs; + } + } else if (queue < tx_cnt) { + priv->tx_coal_frames[queue] = + ec->tx_max_coalesced_frames; + priv->tx_coal_timer[queue] = + ec->tx_coalesce_usecs; + } + return 0; } +static int stmmac_set_coalesce(struct net_device *dev, + struct ethtool_coalesce *ec, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) +{ + return __stmmac_set_coalesce(dev, ec, -1); +} + +static int stmmac_set_per_queue_coalesce(struct net_device *dev, u32 queue, + struct ethtool_coalesce *ec) +{ + return __stmmac_set_coalesce(dev, ec, queue); +} + static int stmmac_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *rxnfc, u32 *rule_locs) { @@ -841,6 +1053,30 @@ static int stmmac_set_rxfh(struct net_device *dev, const u32 *indir, priv->plat->rx_queues_to_use); } +static void stmmac_get_channels(struct net_device *dev, + struct ethtool_channels *chan) +{ + struct stmmac_priv *priv = netdev_priv(dev); + + chan->rx_count = priv->plat->rx_queues_to_use; + chan->tx_count = priv->plat->tx_queues_to_use; + chan->max_rx = priv->dma_cap.number_rx_queues; + chan->max_tx = priv->dma_cap.number_tx_queues; +} + +static int stmmac_set_channels(struct net_device *dev, + struct ethtool_channels *chan) +{ + struct stmmac_priv *priv = netdev_priv(dev); + + if (chan->rx_count > priv->dma_cap.number_rx_queues || + chan->tx_count > priv->dma_cap.number_tx_queues || + !chan->rx_count || !chan->tx_count) + return -EINVAL; + + return stmmac_reinit_queues(dev, chan->rx_count, chan->tx_count); +} + static int stmmac_get_ts_info(struct net_device *dev, struct ethtool_ts_info *info) { @@ -914,6 +1150,8 @@ static int stmmac_set_tunable(struct net_device *dev, } static const struct ethtool_ops stmmac_ethtool_ops = { + .supported_coalesce_params = ETHTOOL_COALESCE_USECS | + ETHTOOL_COALESCE_MAX_FRAMES, .begin = stmmac_check_if_running, .get_drvinfo = stmmac_ethtool_getdrvinfo, .get_msglevel = stmmac_ethtool_getmsglevel, @@ -922,6 +1160,8 @@ static const struct ethtool_ops stmmac_ethtool_ops = { .get_regs_len = stmmac_ethtool_get_regs_len, .get_link = ethtool_op_get_link, .nway_reset = stmmac_nway_reset, + .get_ringparam = stmmac_get_ringparam, + .set_ringparam = stmmac_set_ringparam, .get_pauseparam = stmmac_get_pauseparam, .set_pauseparam = stmmac_set_pauseparam, .self_test = stmmac_selftest_run, @@ -940,6 +1180,10 @@ static const struct ethtool_ops stmmac_ethtool_ops = { .get_ts_info = stmmac_get_ts_info, .get_coalesce = stmmac_get_coalesce, .set_coalesce = stmmac_set_coalesce, + .get_per_queue_coalesce = stmmac_get_per_queue_coalesce, + .set_per_queue_coalesce = stmmac_set_per_queue_coalesce, + .get_channels = stmmac_get_channels, + .set_channels = stmmac_set_channels, .get_tunable = stmmac_get_tunable, .set_tunable = stmmac_set_tunable, .get_link_ksettings = stmmac_ethtool_get_link_ksettings, diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_hwtstamp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_hwtstamp.c index 020159622559..764832f4dae1 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_hwtstamp.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_hwtstamp.c @@ -10,9 +10,13 @@ *******************************************************************************/ #include <linux/io.h> +#include <linux/iopoll.h> #include <linux/delay.h> +#include <linux/ptp_clock_kernel.h> #include "common.h" #include "stmmac_ptp.h" +#include "dwmac4.h" +#include "stmmac.h" static void config_hw_tstamping(void __iomem *ioaddr, u32 data) { @@ -26,12 +30,16 @@ static void config_sub_second_increment(void __iomem *ioaddr, unsigned long data; u32 reg_value; - /* For GMAC3.x, 4.x versions, convert the ptp_clock to nano second - * formula = (1/ptp_clock) * 1000000000 - * where ptp_clock is 50MHz if fine method is used to update system + /* For GMAC3.x, 4.x versions, in "fine adjustement mode" set sub-second + * increment to twice the number of nanoseconds of a clock cycle. + * The calculation of the default_addend value by the caller will set it + * to mid-range = 2^31 when the remainder of this division is zero, + * which will make the accumulator overflow once every 2 ptp_clock + * cycles, adding twice the number of nanoseconds of a clock cycle : + * 2000000000ULL / ptp_clock. */ if (value & PTP_TCR_TSCFUPDT) - data = (1000000000ULL / 50000000); + data = (2000000000ULL / ptp_clock); else data = (1000000000ULL / ptp_clock); @@ -53,7 +61,6 @@ static void config_sub_second_increment(void __iomem *ioaddr, static int init_systime(void __iomem *ioaddr, u32 sec, u32 nsec) { - int limit; u32 value; writel(sec, ioaddr + PTP_STSUR); @@ -64,16 +71,9 @@ static int init_systime(void __iomem *ioaddr, u32 sec, u32 nsec) writel(value, ioaddr + PTP_TCR); /* wait for present system time initialize to complete */ - limit = 10; - while (limit--) { - if (!(readl(ioaddr + PTP_TCR) & PTP_TCR_TSINIT)) - break; - mdelay(10); - } - if (limit < 0) - return -EBUSY; - - return 0; + return readl_poll_timeout_atomic(ioaddr + PTP_TCR, value, + !(value & PTP_TCR_TSINIT), + 10, 100000); } static int config_addend(void __iomem *ioaddr, u32 addend) @@ -145,15 +145,70 @@ static int adjust_systime(void __iomem *ioaddr, u32 sec, u32 nsec, static void get_systime(void __iomem *ioaddr, u64 *systime) { + u64 ns, sec0, sec1; + + /* Get the TSS value */ + sec1 = readl_relaxed(ioaddr + PTP_STSR); + do { + sec0 = sec1; + /* Get the TSSS value */ + ns = readl_relaxed(ioaddr + PTP_STNSR); + /* Get the TSS value */ + sec1 = readl_relaxed(ioaddr + PTP_STSR); + } while (sec0 != sec1); + + if (systime) + *systime = ns + (sec1 * 1000000000ULL); +} + +static void get_ptptime(void __iomem *ptpaddr, u64 *ptp_time) +{ u64 ns; - /* Get the TSSS value */ - ns = readl(ioaddr + PTP_STNSR); - /* Get the TSS and convert sec time value to nanosecond */ - ns += readl(ioaddr + PTP_STSR) * 1000000000ULL; + ns = readl(ptpaddr + PTP_ATNR); + ns += readl(ptpaddr + PTP_ATSR) * NSEC_PER_SEC; - if (systime) - *systime = ns; + *ptp_time = ns; +} + +static void timestamp_interrupt(struct stmmac_priv *priv) +{ + u32 num_snapshot, ts_status, tsync_int; + struct ptp_clock_event event; + unsigned long flags; + u64 ptp_time; + int i; + + if (priv->plat->int_snapshot_en) { + wake_up(&priv->tstamp_busy_wait); + return; + } + + tsync_int = readl(priv->ioaddr + GMAC_INT_STATUS) & GMAC_INT_TSIE; + + if (!tsync_int) + return; + + /* Read timestamp status to clear interrupt from either external + * timestamp or start/end of PPS. + */ + ts_status = readl(priv->ioaddr + GMAC_TIMESTAMP_STATUS); + + if (!priv->plat->ext_snapshot_en) + return; + + num_snapshot = (ts_status & GMAC_TIMESTAMP_ATSNS_MASK) >> + GMAC_TIMESTAMP_ATSNS_SHIFT; + + for (i = 0; i < num_snapshot; i++) { + read_lock_irqsave(&priv->ptp_lock, flags); + get_ptptime(priv->ptpaddr, &ptp_time); + read_unlock_irqrestore(&priv->ptp_lock, flags); + event.type = PTP_CLOCK_EXTTS; + event.index = 0; + event.timestamp = ptp_time; + ptp_clock_event(priv->ptp_clock, &event); + } } const struct stmmac_hwtimestamp stmmac_ptp = { @@ -163,4 +218,6 @@ const struct stmmac_hwtimestamp stmmac_ptp = { .config_addend = config_addend, .adjust_systime = adjust_systime, .get_systime = get_systime, + .get_ptptime = get_ptptime, + .timestamp_interrupt = timestamp_interrupt, }; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 7da18c9afa01..8273e6a175c8 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -28,6 +28,7 @@ #include <linux/if_vlan.h> #include <linux/dma-mapping.h> #include <linux/slab.h> +#include <linux/pm_runtime.h> #include <linux/prefetch.h> #include <linux/pinctrl/consumer.h> #ifdef CONFIG_DEBUG_FS @@ -37,15 +38,25 @@ #include <linux/net_tstamp.h> #include <linux/phylink.h> #include <linux/udp.h> +#include <linux/bpf_trace.h> #include <net/pkt_cls.h> +#include <net/xdp_sock_drv.h> #include "stmmac_ptp.h" #include "stmmac.h" +#include "stmmac_xdp.h" #include <linux/reset.h> #include <linux/of_mdio.h> #include "dwmac1000.h" #include "dwxgmac2.h" #include "hwif.h" +/* As long as the interface is active, we keep the timestamping counter enabled + * with fine resolution and binary rollover. This avoid non-monotonic behavior + * (clock jumps) when changing timestamping settings at runtime. + */ +#define STMMAC_HWTS_ACTIVE (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | \ + PTP_TCR_TSCTRLSSR) + #define STMMAC_ALIGN(x) ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16) #define TSO_MAX_BUFF_SIZE (SZ_16K - 1) @@ -63,8 +74,18 @@ static int phyaddr = -1; module_param(phyaddr, int, 0444); MODULE_PARM_DESC(phyaddr, "Physical device address"); -#define STMMAC_TX_THRESH (DMA_TX_SIZE / 4) -#define STMMAC_RX_THRESH (DMA_RX_SIZE / 4) +#define STMMAC_TX_THRESH(x) ((x)->dma_conf.dma_tx_size / 4) +#define STMMAC_RX_THRESH(x) ((x)->dma_conf.dma_rx_size / 4) + +/* Limit to make sure XDP TX and slow path can coexist */ +#define STMMAC_XSK_TX_BUDGET_MAX 256 +#define STMMAC_TX_XSK_AVAIL 16 +#define STMMAC_RX_FILL_BATCH 16 + +#define STMMAC_XDP_PASS 0 +#define STMMAC_XDP_CONSUMED BIT(0) +#define STMMAC_XDP_TX BIT(1) +#define STMMAC_XDP_REDIRECT BIT(2) static int flow_ctrl = FLOW_AUTO; module_param(flow_ctrl, int, 0644); @@ -94,7 +115,7 @@ static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE | static int eee_timer = STMMAC_DEFAULT_LPI_TIMER; module_param(eee_timer, int, 0644); MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec"); -#define STMMAC_LPI_T(x) (jiffies + msecs_to_jiffies(x)) +#define STMMAC_LPI_T(x) (jiffies + usecs_to_jiffies(x)) /* By default the driver will use the ring mode to manage tx and rx descriptors, * but allow user to force to use the chain instead of the ring @@ -104,6 +125,18 @@ module_param(chain_mode, int, 0444); MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode"); static irqreturn_t stmmac_interrupt(int irq, void *dev_id); +/* For MSI interrupts handling */ +static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id); +static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id); +static irqreturn_t stmmac_msi_intr_tx(int irq, void *data); +static irqreturn_t stmmac_msi_intr_rx(int irq, void *data); +static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue); +static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue); +static void stmmac_reset_queues_param(struct stmmac_priv *priv); +static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue); +static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue); +static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, + u32 rxmode, u32 chan); #ifdef CONFIG_DEBUG_FS static const struct net_device_ops stmmac_netdev_ops; @@ -111,7 +144,39 @@ static void stmmac_init_fs(struct net_device *dev); static void stmmac_exit_fs(struct net_device *dev); #endif -#define STMMAC_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x)) +#define STMMAC_COAL_TIMER(x) (ns_to_ktime((x) * NSEC_PER_USEC)) + +int stmmac_bus_clks_config(struct stmmac_priv *priv, bool enabled) +{ + int ret = 0; + + if (enabled) { + ret = clk_prepare_enable(priv->plat->stmmac_clk); + if (ret) + return ret; + ret = clk_prepare_enable(priv->plat->pclk); + if (ret) { + clk_disable_unprepare(priv->plat->stmmac_clk); + return ret; + } + if (priv->plat->clks_config) { + ret = priv->plat->clks_config(priv->plat->bsp_priv, enabled); + if (ret) { + clk_disable_unprepare(priv->plat->stmmac_clk); + clk_disable_unprepare(priv->plat->pclk); + return ret; + } + } + } else { + clk_disable_unprepare(priv->plat->stmmac_clk); + clk_disable_unprepare(priv->plat->pclk); + if (priv->plat->clks_config) + priv->plat->clks_config(priv->plat->bsp_priv, enabled); + } + + return ret; +} +EXPORT_SYMBOL_GPL(stmmac_bus_clks_config); /** * stmmac_verify_args - verify the driver parameters. @@ -134,11 +199,7 @@ static void stmmac_verify_args(void) eee_timer = STMMAC_DEFAULT_LPI_TIMER; } -/** - * stmmac_disable_all_queues - Disable all queues - * @priv: driver private structure - */ -static void stmmac_disable_all_queues(struct stmmac_priv *priv) +static void __stmmac_disable_all_queues(struct stmmac_priv *priv) { u32 rx_queues_cnt = priv->plat->rx_queues_to_use; u32 tx_queues_cnt = priv->plat->tx_queues_to_use; @@ -148,6 +209,12 @@ static void stmmac_disable_all_queues(struct stmmac_priv *priv) for (queue = 0; queue < maxq; queue++) { struct stmmac_channel *ch = &priv->channel[queue]; + if (stmmac_xdp_is_enabled(priv) && + test_bit(queue, priv->af_xdp_zc_qps)) { + napi_disable(&ch->rxtx_napi); + continue; + } + if (queue < rx_queues_cnt) napi_disable(&ch->rx_napi); if (queue < tx_queues_cnt) @@ -156,50 +223,52 @@ static void stmmac_disable_all_queues(struct stmmac_priv *priv) } /** - * stmmac_enable_all_queues - Enable all queues + * stmmac_disable_all_queues - Disable all queues * @priv: driver private structure */ -static void stmmac_enable_all_queues(struct stmmac_priv *priv) +static void stmmac_disable_all_queues(struct stmmac_priv *priv) { u32 rx_queues_cnt = priv->plat->rx_queues_to_use; - u32 tx_queues_cnt = priv->plat->tx_queues_to_use; - u32 maxq = max(rx_queues_cnt, tx_queues_cnt); + struct stmmac_rx_queue *rx_q; u32 queue; - for (queue = 0; queue < maxq; queue++) { - struct stmmac_channel *ch = &priv->channel[queue]; - - if (queue < rx_queues_cnt) - napi_enable(&ch->rx_napi); - if (queue < tx_queues_cnt) - napi_enable(&ch->tx_napi); + /* synchronize_rcu() needed for pending XDP buffers to drain */ + for (queue = 0; queue < rx_queues_cnt; queue++) { + rx_q = &priv->dma_conf.rx_queue[queue]; + if (rx_q->xsk_pool) { + synchronize_rcu(); + break; + } } + + __stmmac_disable_all_queues(priv); } /** - * stmmac_stop_all_queues - Stop all queues + * stmmac_enable_all_queues - Enable all queues * @priv: driver private structure */ -static void stmmac_stop_all_queues(struct stmmac_priv *priv) +static void stmmac_enable_all_queues(struct stmmac_priv *priv) { + u32 rx_queues_cnt = priv->plat->rx_queues_to_use; u32 tx_queues_cnt = priv->plat->tx_queues_to_use; + u32 maxq = max(rx_queues_cnt, tx_queues_cnt); u32 queue; - for (queue = 0; queue < tx_queues_cnt; queue++) - netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); -} + for (queue = 0; queue < maxq; queue++) { + struct stmmac_channel *ch = &priv->channel[queue]; -/** - * stmmac_start_all_queues - Start all queues - * @priv: driver private structure - */ -static void stmmac_start_all_queues(struct stmmac_priv *priv) -{ - u32 tx_queues_cnt = priv->plat->tx_queues_to_use; - u32 queue; + if (stmmac_xdp_is_enabled(priv) && + test_bit(queue, priv->af_xdp_zc_qps)) { + napi_enable(&ch->rxtx_napi); + continue; + } - for (queue = 0; queue < tx_queues_cnt; queue++) - netif_tx_start_queue(netdev_get_tx_queue(priv->dev, queue)); + if (queue < rx_queues_cnt) + napi_enable(&ch->rx_napi); + if (queue < tx_queues_cnt) + napi_enable(&ch->tx_napi); + } } static void stmmac_service_event_schedule(struct stmmac_priv *priv) @@ -252,7 +321,7 @@ static void stmmac_clk_csr_set(struct stmmac_priv *priv) priv->clk_csr = STMMAC_CSR_100_150M; else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M)) priv->clk_csr = STMMAC_CSR_150_250M; - else if ((clk_rate >= CSR_F_250M) && (clk_rate < CSR_F_300M)) + else if ((clk_rate >= CSR_F_250M) && (clk_rate <= CSR_F_300M)) priv->clk_csr = STMMAC_CSR_250_300M; } @@ -291,13 +360,13 @@ static void print_pkt(unsigned char *buf, int len) static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) { - struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; + struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; u32 avail; if (tx_q->dirty_tx > tx_q->cur_tx) avail = tx_q->dirty_tx - tx_q->cur_tx - 1; else - avail = DMA_TX_SIZE - tx_q->cur_tx + tx_q->dirty_tx - 1; + avail = priv->dma_conf.dma_tx_size - tx_q->cur_tx + tx_q->dirty_tx - 1; return avail; } @@ -309,40 +378,51 @@ static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) */ static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue) { - struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; + struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; u32 dirty; if (rx_q->dirty_rx <= rx_q->cur_rx) dirty = rx_q->cur_rx - rx_q->dirty_rx; else - dirty = DMA_RX_SIZE - rx_q->dirty_rx + rx_q->cur_rx; + dirty = priv->dma_conf.dma_rx_size - rx_q->dirty_rx + rx_q->cur_rx; return dirty; } +static void stmmac_lpi_entry_timer_config(struct stmmac_priv *priv, bool en) +{ + int tx_lpi_timer; + + /* Clear/set the SW EEE timer flag based on LPI ET enablement */ + priv->eee_sw_timer_en = en ? 0 : 1; + tx_lpi_timer = en ? priv->tx_lpi_timer : 0; + stmmac_set_eee_lpi_timer(priv, priv->hw, tx_lpi_timer); +} + /** * stmmac_enable_eee_mode - check and enter in LPI mode * @priv: driver private structure * Description: this function is to verify and enter in LPI mode in case of * EEE. */ -static void stmmac_enable_eee_mode(struct stmmac_priv *priv) +static int stmmac_enable_eee_mode(struct stmmac_priv *priv) { u32 tx_cnt = priv->plat->tx_queues_to_use; u32 queue; /* check if all TX queues have the work finished */ for (queue = 0; queue < tx_cnt; queue++) { - struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; + struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; if (tx_q->dirty_tx != tx_q->cur_tx) - return; /* still unfinished work */ + return -EBUSY; /* still unfinished work */ } /* Check and enter in LPI mode */ if (!priv->tx_path_in_lpi_mode) stmmac_set_eee_mode(priv, priv->hw, priv->plat->en_tx_lpi_clockgating); + return 0; } /** @@ -353,6 +433,11 @@ static void stmmac_enable_eee_mode(struct stmmac_priv *priv) */ void stmmac_disable_eee_mode(struct stmmac_priv *priv) { + if (!priv->eee_sw_timer_en) { + stmmac_lpi_entry_timer_config(priv, 0); + return; + } + stmmac_reset_eee_mode(priv, priv->hw); del_timer_sync(&priv->eee_ctrl_timer); priv->tx_path_in_lpi_mode = false; @@ -360,7 +445,7 @@ void stmmac_disable_eee_mode(struct stmmac_priv *priv) /** * stmmac_eee_ctrl_timer - EEE TX SW timer. - * @arg : data hook + * @t: timer_list struct containing private info * Description: * if there is no data transfer and if we are not in LPI state, * then MAC Transmitter can be moved to LPI state. @@ -369,8 +454,8 @@ static void stmmac_eee_ctrl_timer(struct timer_list *t) { struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer); - stmmac_enable_eee_mode(priv); - mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer)); + if (stmmac_enable_eee_mode(priv)) + mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); } /** @@ -383,7 +468,7 @@ static void stmmac_eee_ctrl_timer(struct timer_list *t) */ bool stmmac_eee_init(struct stmmac_priv *priv) { - int tx_lpi_timer = priv->tx_lpi_timer; + int eee_tw_timer = priv->eee_tw_timer; /* Using PCS we cannot dial with the phy registers at this stage * so we do not support extra feature like EEE. @@ -402,8 +487,13 @@ bool stmmac_eee_init(struct stmmac_priv *priv) if (!priv->eee_active) { if (priv->eee_enabled) { netdev_dbg(priv->dev, "disable EEE\n"); + stmmac_lpi_entry_timer_config(priv, 0); del_timer_sync(&priv->eee_ctrl_timer); - stmmac_set_eee_timer(priv, priv->hw, 0, tx_lpi_timer); + stmmac_set_eee_timer(priv, priv->hw, 0, eee_tw_timer); + if (priv->hw->xpcs) + xpcs_config_eee(priv->hw->xpcs, + priv->plat->mult_fact_100ns, + false); } mutex_unlock(&priv->lock); return false; @@ -411,9 +501,22 @@ bool stmmac_eee_init(struct stmmac_priv *priv) if (priv->eee_active && !priv->eee_enabled) { timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0); - mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer)); stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS, - tx_lpi_timer); + eee_tw_timer); + if (priv->hw->xpcs) + xpcs_config_eee(priv->hw->xpcs, + priv->plat->mult_fact_100ns, + true); + } + + if (priv->plat->has_gmac4 && priv->tx_lpi_timer <= STMMAC_ET_MAX) { + del_timer_sync(&priv->eee_ctrl_timer); + priv->tx_path_in_lpi_mode = false; + stmmac_lpi_entry_timer_config(priv, 1); + } else { + stmmac_lpi_entry_timer_config(priv, 0); + mod_timer(&priv->eee_ctrl_timer, + STMMAC_LPI_T(priv->tx_lpi_timer)); } mutex_unlock(&priv->lock); @@ -452,6 +555,8 @@ static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, } if (found) { + ns -= priv->plat->cdc_error_adj; + memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); shhwtstamp.hwtstamp = ns_to_ktime(ns); @@ -486,6 +591,9 @@ static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, /* Check if timestamp is available */ if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) { stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); + + ns -= priv->plat->cdc_error_adj; + netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns); shhwtstamp = skb_hwtstamps(skb); memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); @@ -510,8 +618,6 @@ static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr) { struct stmmac_priv *priv = netdev_priv(dev); struct hwtstamp_config config; - struct timespec64 now; - u64 temp = 0; u32 ptp_v2 = 0; u32 tstamp_all = 0; u32 ptp_over_ipv4_udp = 0; @@ -520,11 +626,6 @@ static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr) u32 snap_type_sel = 0; u32 ts_master_en = 0; u32 ts_event_en = 0; - u32 sec_inc = 0; - u32 value = 0; - bool xmac; - - xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; if (!(priv->dma_cap.time_stamp || priv->adv_ts)) { netdev_alert(priv->dev, "No support for HW time stamping\n"); @@ -541,10 +642,6 @@ static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr) netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n", __func__, config.flags, config.tx_type, config.rx_filter); - /* reserved for future extensions */ - if (config.flags) - return -EINVAL; - if (config.tx_type != HWTSTAMP_TX_OFF && config.tx_type != HWTSTAMP_TX_ON) return -ERANGE; @@ -630,7 +727,8 @@ static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr) config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; ptp_v2 = PTP_TCR_TSVER2ENA; snap_type_sel = PTP_TCR_SNAPTYPSEL_1; - ts_event_en = PTP_TCR_TSEVNTENA; + if (priv->synopsys_id < DWMAC_CORE_4_10) + ts_event_en = PTP_TCR_TSEVNTENA; ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; ptp_over_ethernet = PTP_TCR_TSIPENA; @@ -685,42 +783,17 @@ static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr) priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1); priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON; - if (!priv->hwts_tx_en && !priv->hwts_rx_en) - stmmac_config_hw_tstamping(priv, priv->ptpaddr, 0); - else { - value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR | - tstamp_all | ptp_v2 | ptp_over_ethernet | - ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en | - ts_master_en | snap_type_sel); - stmmac_config_hw_tstamping(priv, priv->ptpaddr, value); - - /* program Sub Second Increment reg */ - stmmac_config_sub_second_increment(priv, - priv->ptpaddr, priv->plat->clk_ptp_rate, - xmac, &sec_inc); - temp = div_u64(1000000000ULL, sec_inc); - - /* Store sub second increment and flags for later use */ - priv->sub_second_inc = sec_inc; - priv->systime_flags = value; - - /* calculate default added value: - * formula is : - * addend = (2^32)/freq_div_ratio; - * where, freq_div_ratio = 1e9ns/sec_inc - */ - temp = (u64)(temp << 32); - priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate); - stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend); - - /* initialize system time */ - ktime_get_real_ts64(&now); + priv->systime_flags = STMMAC_HWTS_ACTIVE; - /* lower 32 bits of tv_sec are safe until y2106 */ - stmmac_init_systime(priv, priv->ptpaddr, - (u32)now.tv_sec, now.tv_nsec); + if (priv->hwts_tx_en || priv->hwts_rx_en) { + priv->systime_flags |= tstamp_all | ptp_v2 | + ptp_over_ethernet | ptp_over_ipv6_udp | + ptp_over_ipv4_udp | ts_event_en | + ts_master_en | snap_type_sel; } + stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags); + memcpy(&priv->tstamp_config, &config, sizeof(config)); return copy_to_user(ifr->ifr_data, &config, @@ -734,7 +807,7 @@ static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr) * a proprietary structure used to pass information to the driver. * Description: * This function obtain the current hardware timestamping settings - as requested. + * as requested. */ static int stmmac_hwtstamp_get(struct net_device *dev, struct ifreq *ifr) { @@ -749,6 +822,57 @@ static int stmmac_hwtstamp_get(struct net_device *dev, struct ifreq *ifr) } /** + * stmmac_init_tstamp_counter - init hardware timestamping counter + * @priv: driver private structure + * @systime_flags: timestamping flags + * Description: + * Initialize hardware counter for packet timestamping. + * This is valid as long as the interface is open and not suspended. + * Will be rerun after resuming from suspend, case in which the timestamping + * flags updated by stmmac_hwtstamp_set() also need to be restored. + */ +int stmmac_init_tstamp_counter(struct stmmac_priv *priv, u32 systime_flags) +{ + bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; + struct timespec64 now; + u32 sec_inc = 0; + u64 temp = 0; + + if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) + return -EOPNOTSUPP; + + stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags); + priv->systime_flags = systime_flags; + + /* program Sub Second Increment reg */ + stmmac_config_sub_second_increment(priv, priv->ptpaddr, + priv->plat->clk_ptp_rate, + xmac, &sec_inc); + temp = div_u64(1000000000ULL, sec_inc); + + /* Store sub second increment for later use */ + priv->sub_second_inc = sec_inc; + + /* calculate default added value: + * formula is : + * addend = (2^32)/freq_div_ratio; + * where, freq_div_ratio = 1e9ns/sec_inc + */ + temp = (u64)(temp << 32); + priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate); + stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend); + + /* initialize system time */ + ktime_get_real_ts64(&now); + + /* lower 32 bits of tv_sec are safe until y2106 */ + stmmac_init_systime(priv, priv->ptpaddr, (u32)now.tv_sec, now.tv_nsec); + + return 0; +} +EXPORT_SYMBOL_GPL(stmmac_init_tstamp_counter); + +/** * stmmac_init_ptp - init PTP * @priv: driver private structure * Description: this is to verify if the HW supports the PTPv1 or PTPv2. @@ -758,9 +882,14 @@ static int stmmac_hwtstamp_get(struct net_device *dev, struct ifreq *ifr) static int stmmac_init_ptp(struct stmmac_priv *priv) { bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; + int ret; - if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) - return -EOPNOTSUPP; + if (priv->plat->ptp_clk_freq_config) + priv->plat->ptp_clk_freq_config(priv); + + ret = stmmac_init_tstamp_counter(priv, STMMAC_HWTS_ACTIVE); + if (ret) + return ret; priv->adv_ts = 0; /* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */ @@ -780,21 +909,19 @@ static int stmmac_init_ptp(struct stmmac_priv *priv) priv->hwts_tx_en = 0; priv->hwts_rx_en = 0; - stmmac_ptp_register(priv); - return 0; } static void stmmac_release_ptp(struct stmmac_priv *priv) { - if (priv->plat->clk_ptp_ref) - clk_disable_unprepare(priv->plat->clk_ptp_ref); + clk_disable_unprepare(priv->plat->clk_ptp_ref); stmmac_ptp_unregister(priv); } /** * stmmac_mac_flow_ctrl - Configure flow control in all queues * @priv: driver private structure + * @duplex: duplex passed to the next function * Description: It is used for configuring the flow control in all queues */ static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex) @@ -805,86 +932,67 @@ static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex) priv->pause, tx_cnt); } -static void stmmac_validate(struct phylink_config *config, - unsigned long *supported, - struct phylink_link_state *state) +static struct phylink_pcs *stmmac_mac_select_pcs(struct phylink_config *config, + phy_interface_t interface) { struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); - __ETHTOOL_DECLARE_LINK_MODE_MASK(mac_supported) = { 0, }; - __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; - int tx_cnt = priv->plat->tx_queues_to_use; - int max_speed = priv->plat->max_speed; - phylink_set(mac_supported, 10baseT_Half); - phylink_set(mac_supported, 10baseT_Full); - phylink_set(mac_supported, 100baseT_Half); - phylink_set(mac_supported, 100baseT_Full); - phylink_set(mac_supported, 1000baseT_Half); - phylink_set(mac_supported, 1000baseT_Full); - phylink_set(mac_supported, 1000baseKX_Full); - - phylink_set(mac_supported, Autoneg); - phylink_set(mac_supported, Pause); - phylink_set(mac_supported, Asym_Pause); - phylink_set_port_modes(mac_supported); - - /* Cut down 1G if asked to */ - if ((max_speed > 0) && (max_speed < 1000)) { - phylink_set(mask, 1000baseT_Full); - phylink_set(mask, 1000baseX_Full); - } else if (priv->plat->has_xgmac) { - if (!max_speed || (max_speed >= 2500)) { - phylink_set(mac_supported, 2500baseT_Full); - phylink_set(mac_supported, 2500baseX_Full); - } - if (!max_speed || (max_speed >= 5000)) { - phylink_set(mac_supported, 5000baseT_Full); - } - if (!max_speed || (max_speed >= 10000)) { - phylink_set(mac_supported, 10000baseSR_Full); - phylink_set(mac_supported, 10000baseLR_Full); - phylink_set(mac_supported, 10000baseER_Full); - phylink_set(mac_supported, 10000baseLRM_Full); - phylink_set(mac_supported, 10000baseT_Full); - phylink_set(mac_supported, 10000baseKX4_Full); - phylink_set(mac_supported, 10000baseKR_Full); - } - } + if (!priv->hw->xpcs) + return NULL; - /* Half-Duplex can only work with single queue */ - if (tx_cnt > 1) { - phylink_set(mask, 10baseT_Half); - phylink_set(mask, 100baseT_Half); - phylink_set(mask, 1000baseT_Half); - } + return &priv->hw->xpcs->pcs; +} - bitmap_and(supported, supported, mac_supported, - __ETHTOOL_LINK_MODE_MASK_NBITS); - bitmap_andnot(supported, supported, mask, - __ETHTOOL_LINK_MODE_MASK_NBITS); - bitmap_and(state->advertising, state->advertising, mac_supported, - __ETHTOOL_LINK_MODE_MASK_NBITS); - bitmap_andnot(state->advertising, state->advertising, mask, - __ETHTOOL_LINK_MODE_MASK_NBITS); +static void stmmac_mac_config(struct phylink_config *config, unsigned int mode, + const struct phylink_link_state *state) +{ + /* Nothing to do, xpcs_config() handles everything */ } -static void stmmac_mac_pcs_get_state(struct phylink_config *config, - struct phylink_link_state *state) +static void stmmac_fpe_link_state_handle(struct stmmac_priv *priv, bool is_up) { - state->link = 0; + struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; + enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; + enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; + bool *hs_enable = &fpe_cfg->hs_enable; + + if (is_up && *hs_enable) { + stmmac_fpe_send_mpacket(priv, priv->ioaddr, MPACKET_VERIFY); + } else { + *lo_state = FPE_STATE_OFF; + *lp_state = FPE_STATE_OFF; + } } -static void stmmac_mac_config(struct phylink_config *config, unsigned int mode, - const struct phylink_link_state *state) +static void stmmac_mac_link_down(struct phylink_config *config, + unsigned int mode, phy_interface_t interface) { struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); - u32 ctrl; - ctrl = readl(priv->ioaddr + MAC_CTRL_REG); - ctrl &= ~priv->hw->link.speed_mask; + stmmac_mac_set(priv, priv->ioaddr, false); + priv->eee_active = false; + priv->tx_lpi_enabled = false; + priv->eee_enabled = stmmac_eee_init(priv); + stmmac_set_eee_pls(priv, priv->hw, false); + + if (priv->dma_cap.fpesel) + stmmac_fpe_link_state_handle(priv, false); +} - if (state->interface == PHY_INTERFACE_MODE_USXGMII) { - switch (state->speed) { +static void stmmac_mac_link_up(struct phylink_config *config, + struct phy_device *phy, + unsigned int mode, phy_interface_t interface, + int speed, int duplex, + bool tx_pause, bool rx_pause) +{ + struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); + u32 old_ctrl, ctrl; + + old_ctrl = readl(priv->ioaddr + MAC_CTRL_REG); + ctrl = old_ctrl & ~priv->hw->link.speed_mask; + + if (interface == PHY_INTERFACE_MODE_USXGMII) { + switch (speed) { case SPEED_10000: ctrl |= priv->hw->link.xgmii.speed10000; break; @@ -897,8 +1005,34 @@ static void stmmac_mac_config(struct phylink_config *config, unsigned int mode, default: return; } + } else if (interface == PHY_INTERFACE_MODE_XLGMII) { + switch (speed) { + case SPEED_100000: + ctrl |= priv->hw->link.xlgmii.speed100000; + break; + case SPEED_50000: + ctrl |= priv->hw->link.xlgmii.speed50000; + break; + case SPEED_40000: + ctrl |= priv->hw->link.xlgmii.speed40000; + break; + case SPEED_25000: + ctrl |= priv->hw->link.xlgmii.speed25000; + break; + case SPEED_10000: + ctrl |= priv->hw->link.xgmii.speed10000; + break; + case SPEED_2500: + ctrl |= priv->hw->link.speed2500; + break; + case SPEED_1000: + ctrl |= priv->hw->link.speed1000; + break; + default: + return; + } } else { - switch (state->speed) { + switch (speed) { case SPEED_2500: ctrl |= priv->hw->link.speed2500; break; @@ -916,58 +1050,39 @@ static void stmmac_mac_config(struct phylink_config *config, unsigned int mode, } } - priv->speed = state->speed; + priv->speed = speed; if (priv->plat->fix_mac_speed) - priv->plat->fix_mac_speed(priv->plat->bsp_priv, state->speed); + priv->plat->fix_mac_speed(priv->plat->bsp_priv, speed); - if (!state->duplex) + if (!duplex) ctrl &= ~priv->hw->link.duplex; else ctrl |= priv->hw->link.duplex; /* Flow Control operation */ - if (state->pause) - stmmac_mac_flow_ctrl(priv, state->duplex); - - writel(ctrl, priv->ioaddr + MAC_CTRL_REG); -} - -static void stmmac_mac_an_restart(struct phylink_config *config) -{ - /* Not Supported */ -} + if (tx_pause && rx_pause) + stmmac_mac_flow_ctrl(priv, duplex); -static void stmmac_mac_link_down(struct phylink_config *config, - unsigned int mode, phy_interface_t interface) -{ - struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); - - stmmac_mac_set(priv, priv->ioaddr, false); - priv->eee_active = false; - stmmac_eee_init(priv); - stmmac_set_eee_pls(priv, priv->hw, false); -} - -static void stmmac_mac_link_up(struct phylink_config *config, - unsigned int mode, phy_interface_t interface, - struct phy_device *phy) -{ - struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); + if (ctrl != old_ctrl) + writel(ctrl, priv->ioaddr + MAC_CTRL_REG); stmmac_mac_set(priv, priv->ioaddr, true); if (phy && priv->dma_cap.eee) { priv->eee_active = phy_init_eee(phy, 1) >= 0; priv->eee_enabled = stmmac_eee_init(priv); + priv->tx_lpi_enabled = priv->eee_enabled; stmmac_set_eee_pls(priv, priv->hw, true); } + + if (priv->dma_cap.fpesel) + stmmac_fpe_link_state_handle(priv, true); } static const struct phylink_mac_ops stmmac_phylink_mac_ops = { - .validate = stmmac_validate, - .mac_pcs_get_state = stmmac_mac_pcs_get_state, + .validate = phylink_generic_validate, + .mac_select_pcs = stmmac_mac_select_pcs, .mac_config = stmmac_mac_config, - .mac_an_restart = stmmac_mac_an_restart, .mac_link_down = stmmac_mac_link_down, .mac_link_up = stmmac_mac_link_up, }; @@ -1008,18 +1123,20 @@ static void stmmac_check_pcs_mode(struct stmmac_priv *priv) static int stmmac_init_phy(struct net_device *dev) { struct stmmac_priv *priv = netdev_priv(dev); - struct device_node *node; + struct fwnode_handle *fwnode; int ret; - node = priv->plat->phylink_node; + fwnode = of_fwnode_handle(priv->plat->phylink_node); + if (!fwnode) + fwnode = dev_fwnode(priv->device); - if (node) - ret = phylink_of_phy_connect(priv->phylink, node, 0); + if (fwnode) + ret = phylink_fwnode_phy_connect(priv->phylink, fwnode, 0); /* Some DT bindings do not set-up the PHY handle. Let's try to * manually parse it */ - if (!node || ret) { + if (!fwnode || ret) { int addr = priv->plat->phy_addr; struct phy_device *phydev; @@ -1032,17 +1149,72 @@ static int stmmac_init_phy(struct net_device *dev) ret = phylink_connect_phy(priv->phylink, phydev); } + if (!priv->plat->pmt) { + struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL }; + + phylink_ethtool_get_wol(priv->phylink, &wol); + device_set_wakeup_capable(priv->device, !!wol.supported); + } + return ret; } static int stmmac_phy_setup(struct stmmac_priv *priv) { + struct stmmac_mdio_bus_data *mdio_bus_data = priv->plat->mdio_bus_data; struct fwnode_handle *fwnode = of_fwnode_handle(priv->plat->phylink_node); + int max_speed = priv->plat->max_speed; int mode = priv->plat->phy_interface; struct phylink *phylink; priv->phylink_config.dev = &priv->dev->dev; priv->phylink_config.type = PHYLINK_NETDEV; + if (priv->plat->mdio_bus_data) + priv->phylink_config.ovr_an_inband = + mdio_bus_data->xpcs_an_inband; + + if (!fwnode) + fwnode = dev_fwnode(priv->device); + + /* Set the platform/firmware specified interface mode */ + __set_bit(mode, priv->phylink_config.supported_interfaces); + + /* If we have an xpcs, it defines which PHY interfaces are supported. */ + if (priv->hw->xpcs) + xpcs_get_interfaces(priv->hw->xpcs, + priv->phylink_config.supported_interfaces); + + priv->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE | + MAC_10 | MAC_100; + + if (!max_speed || max_speed >= 1000) + priv->phylink_config.mac_capabilities |= MAC_1000; + + if (priv->plat->has_gmac4) { + if (!max_speed || max_speed >= 2500) + priv->phylink_config.mac_capabilities |= MAC_2500FD; + } else if (priv->plat->has_xgmac) { + if (!max_speed || max_speed >= 2500) + priv->phylink_config.mac_capabilities |= MAC_2500FD; + if (!max_speed || max_speed >= 5000) + priv->phylink_config.mac_capabilities |= MAC_5000FD; + if (!max_speed || max_speed >= 10000) + priv->phylink_config.mac_capabilities |= MAC_10000FD; + if (!max_speed || max_speed >= 25000) + priv->phylink_config.mac_capabilities |= MAC_25000FD; + if (!max_speed || max_speed >= 40000) + priv->phylink_config.mac_capabilities |= MAC_40000FD; + if (!max_speed || max_speed >= 50000) + priv->phylink_config.mac_capabilities |= MAC_50000FD; + if (!max_speed || max_speed >= 100000) + priv->phylink_config.mac_capabilities |= MAC_100000FD; + } + + /* Half-Duplex can only work with single queue */ + if (priv->plat->tx_queues_to_use > 1) + priv->phylink_config.mac_capabilities &= + ~(MAC_10HD | MAC_100HD | MAC_1000HD); + priv->phylink_config.mac_managed_pm = true; phylink = phylink_create(&priv->phylink_config, fwnode, mode, &stmmac_phylink_mac_ops); @@ -1053,58 +1225,72 @@ static int stmmac_phy_setup(struct stmmac_priv *priv) return 0; } -static void stmmac_display_rx_rings(struct stmmac_priv *priv) +static void stmmac_display_rx_rings(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf) { u32 rx_cnt = priv->plat->rx_queues_to_use; + unsigned int desc_size; void *head_rx; u32 queue; /* Display RX rings */ for (queue = 0; queue < rx_cnt; queue++) { - struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; + struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; pr_info("\tRX Queue %u rings\n", queue); - if (priv->extend_desc) + if (priv->extend_desc) { head_rx = (void *)rx_q->dma_erx; - else + desc_size = sizeof(struct dma_extended_desc); + } else { head_rx = (void *)rx_q->dma_rx; + desc_size = sizeof(struct dma_desc); + } /* Display RX ring */ - stmmac_display_ring(priv, head_rx, DMA_RX_SIZE, true); + stmmac_display_ring(priv, head_rx, dma_conf->dma_rx_size, true, + rx_q->dma_rx_phy, desc_size); } } -static void stmmac_display_tx_rings(struct stmmac_priv *priv) +static void stmmac_display_tx_rings(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf) { u32 tx_cnt = priv->plat->tx_queues_to_use; + unsigned int desc_size; void *head_tx; u32 queue; /* Display TX rings */ for (queue = 0; queue < tx_cnt; queue++) { - struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; + struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; pr_info("\tTX Queue %d rings\n", queue); - if (priv->extend_desc) + if (priv->extend_desc) { head_tx = (void *)tx_q->dma_etx; - else if (tx_q->tbs & STMMAC_TBS_AVAIL) + desc_size = sizeof(struct dma_extended_desc); + } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { head_tx = (void *)tx_q->dma_entx; - else + desc_size = sizeof(struct dma_edesc); + } else { head_tx = (void *)tx_q->dma_tx; + desc_size = sizeof(struct dma_desc); + } - stmmac_display_ring(priv, head_tx, DMA_TX_SIZE, false); + stmmac_display_ring(priv, head_tx, dma_conf->dma_tx_size, false, + tx_q->dma_tx_phy, desc_size); } } -static void stmmac_display_rings(struct stmmac_priv *priv) +static void stmmac_display_rings(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf) { /* Display RX ring */ - stmmac_display_rx_rings(priv); + stmmac_display_rx_rings(priv, dma_conf); /* Display TX ring */ - stmmac_display_tx_rings(priv); + stmmac_display_tx_rings(priv, dma_conf); } static int stmmac_set_bfsize(int mtu, int bufsize) @@ -1128,44 +1314,50 @@ static int stmmac_set_bfsize(int mtu, int bufsize) /** * stmmac_clear_rx_descriptors - clear RX descriptors * @priv: driver private structure + * @dma_conf: structure to take the dma data * @queue: RX queue index * Description: this function is called to clear the RX descriptors * in case of both basic and extended descriptors are used. */ -static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue) +static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf, + u32 queue) { - struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; + struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; int i; /* Clear the RX descriptors */ - for (i = 0; i < DMA_RX_SIZE; i++) + for (i = 0; i < dma_conf->dma_rx_size; i++) if (priv->extend_desc) stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic, priv->use_riwt, priv->mode, - (i == DMA_RX_SIZE - 1), - priv->dma_buf_sz); + (i == dma_conf->dma_rx_size - 1), + dma_conf->dma_buf_sz); else stmmac_init_rx_desc(priv, &rx_q->dma_rx[i], priv->use_riwt, priv->mode, - (i == DMA_RX_SIZE - 1), - priv->dma_buf_sz); + (i == dma_conf->dma_rx_size - 1), + dma_conf->dma_buf_sz); } /** * stmmac_clear_tx_descriptors - clear tx descriptors * @priv: driver private structure + * @dma_conf: structure to take the dma data * @queue: TX queue index. * Description: this function is called to clear the TX descriptors * in case of both basic and extended descriptors are used. */ -static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue) +static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf, + u32 queue) { - struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; + struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; int i; /* Clear the TX descriptors */ - for (i = 0; i < DMA_TX_SIZE; i++) { - int last = (i == (DMA_TX_SIZE - 1)); + for (i = 0; i < dma_conf->dma_tx_size; i++) { + int last = (i == (dma_conf->dma_tx_size - 1)); struct dma_desc *p; if (priv->extend_desc) @@ -1182,10 +1374,12 @@ static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue) /** * stmmac_clear_descriptors - clear descriptors * @priv: driver private structure + * @dma_conf: structure to take the dma data * Description: this function is called to clear the TX and RX descriptors * in case of both basic and extended descriptors are used. */ -static void stmmac_clear_descriptors(struct stmmac_priv *priv) +static void stmmac_clear_descriptors(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf) { u32 rx_queue_cnt = priv->plat->rx_queues_to_use; u32 tx_queue_cnt = priv->plat->tx_queues_to_use; @@ -1193,16 +1387,17 @@ static void stmmac_clear_descriptors(struct stmmac_priv *priv) /* Clear the RX descriptors */ for (queue = 0; queue < rx_queue_cnt; queue++) - stmmac_clear_rx_descriptors(priv, queue); + stmmac_clear_rx_descriptors(priv, dma_conf, queue); /* Clear the TX descriptors */ for (queue = 0; queue < tx_queue_cnt; queue++) - stmmac_clear_tx_descriptors(priv, queue); + stmmac_clear_tx_descriptors(priv, dma_conf, queue); } /** * stmmac_init_rx_buffers - init the RX descriptor buffer. * @priv: driver private structure + * @dma_conf: structure to take the dma data * @p: descriptor pointer * @i: descriptor index * @flags: gfp flag @@ -1210,30 +1405,41 @@ static void stmmac_clear_descriptors(struct stmmac_priv *priv) * Description: this function is called to allocate a receive buffer, perform * the DMA mapping and init the descriptor. */ -static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p, +static int stmmac_init_rx_buffers(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf, + struct dma_desc *p, int i, gfp_t flags, u32 queue) { - struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; + struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; + gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); - buf->page = page_pool_dev_alloc_pages(rx_q->page_pool); - if (!buf->page) - return -ENOMEM; + if (priv->dma_cap.addr64 <= 32) + gfp |= GFP_DMA32; + + if (!buf->page) { + buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); + if (!buf->page) + return -ENOMEM; + buf->page_offset = stmmac_rx_offset(priv); + } - if (priv->sph) { - buf->sec_page = page_pool_dev_alloc_pages(rx_q->page_pool); + if (priv->sph && !buf->sec_page) { + buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); if (!buf->sec_page) return -ENOMEM; buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); - stmmac_set_desc_sec_addr(priv, p, buf->sec_addr); + stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); } else { buf->sec_page = NULL; + stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); } - buf->addr = page_pool_get_dma_addr(buf->page); + buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; + stmmac_set_desc_addr(priv, p, buf->addr); - if (priv->dma_buf_sz == BUF_SIZE_16KiB) + if (dma_conf->dma_buf_sz == BUF_SIZE_16KiB) stmmac_init_desc3(priv, p); return 0; @@ -1242,34 +1448,39 @@ static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p, /** * stmmac_free_rx_buffer - free RX dma buffers * @priv: private structure - * @queue: RX queue index + * @rx_q: RX queue * @i: buffer index. */ -static void stmmac_free_rx_buffer(struct stmmac_priv *priv, u32 queue, int i) +static void stmmac_free_rx_buffer(struct stmmac_priv *priv, + struct stmmac_rx_queue *rx_q, + int i) { - struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; if (buf->page) - page_pool_put_page(rx_q->page_pool, buf->page, false); + page_pool_put_full_page(rx_q->page_pool, buf->page, false); buf->page = NULL; if (buf->sec_page) - page_pool_put_page(rx_q->page_pool, buf->sec_page, false); + page_pool_put_full_page(rx_q->page_pool, buf->sec_page, false); buf->sec_page = NULL; } /** * stmmac_free_tx_buffer - free RX dma buffers * @priv: private structure + * @dma_conf: structure to take the dma data * @queue: RX queue index * @i: buffer index. */ -static void stmmac_free_tx_buffer(struct stmmac_priv *priv, u32 queue, int i) +static void stmmac_free_tx_buffer(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf, + u32 queue, int i) { - struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; + struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; - if (tx_q->tx_skbuff_dma[i].buf) { + if (tx_q->tx_skbuff_dma[i].buf && + tx_q->tx_skbuff_dma[i].buf_type != STMMAC_TXBUF_T_XDP_TX) { if (tx_q->tx_skbuff_dma[i].map_as_page) dma_unmap_page(priv->device, tx_q->tx_skbuff_dma[i].buf, @@ -1282,82 +1493,236 @@ static void stmmac_free_tx_buffer(struct stmmac_priv *priv, u32 queue, int i) DMA_TO_DEVICE); } - if (tx_q->tx_skbuff[i]) { + if (tx_q->xdpf[i] && + (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_TX || + tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_NDO)) { + xdp_return_frame(tx_q->xdpf[i]); + tx_q->xdpf[i] = NULL; + } + + if (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XSK_TX) + tx_q->xsk_frames_done++; + + if (tx_q->tx_skbuff[i] && + tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_SKB) { dev_kfree_skb_any(tx_q->tx_skbuff[i]); tx_q->tx_skbuff[i] = NULL; - tx_q->tx_skbuff_dma[i].buf = 0; - tx_q->tx_skbuff_dma[i].map_as_page = false; } + + tx_q->tx_skbuff_dma[i].buf = 0; + tx_q->tx_skbuff_dma[i].map_as_page = false; } /** - * init_dma_rx_desc_rings - init the RX descriptor rings - * @dev: net device structure + * dma_free_rx_skbufs - free RX dma buffers + * @priv: private structure + * @dma_conf: structure to take the dma data + * @queue: RX queue index + */ +static void dma_free_rx_skbufs(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf, + u32 queue) +{ + struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; + int i; + + for (i = 0; i < dma_conf->dma_rx_size; i++) + stmmac_free_rx_buffer(priv, rx_q, i); +} + +static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf, + u32 queue, gfp_t flags) +{ + struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; + int i; + + for (i = 0; i < dma_conf->dma_rx_size; i++) { + struct dma_desc *p; + int ret; + + if (priv->extend_desc) + p = &((rx_q->dma_erx + i)->basic); + else + p = rx_q->dma_rx + i; + + ret = stmmac_init_rx_buffers(priv, dma_conf, p, i, flags, + queue); + if (ret) + return ret; + + rx_q->buf_alloc_num++; + } + + return 0; +} + +/** + * dma_free_rx_xskbufs - free RX dma buffers from XSK pool + * @priv: private structure + * @dma_conf: structure to take the dma data + * @queue: RX queue index + */ +static void dma_free_rx_xskbufs(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf, + u32 queue) +{ + struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; + int i; + + for (i = 0; i < dma_conf->dma_rx_size; i++) { + struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; + + if (!buf->xdp) + continue; + + xsk_buff_free(buf->xdp); + buf->xdp = NULL; + } +} + +static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf, + u32 queue) +{ + struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; + int i; + + for (i = 0; i < dma_conf->dma_rx_size; i++) { + struct stmmac_rx_buffer *buf; + dma_addr_t dma_addr; + struct dma_desc *p; + + if (priv->extend_desc) + p = (struct dma_desc *)(rx_q->dma_erx + i); + else + p = rx_q->dma_rx + i; + + buf = &rx_q->buf_pool[i]; + + buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); + if (!buf->xdp) + return -ENOMEM; + + dma_addr = xsk_buff_xdp_get_dma(buf->xdp); + stmmac_set_desc_addr(priv, p, dma_addr); + rx_q->buf_alloc_num++; + } + + return 0; +} + +static struct xsk_buff_pool *stmmac_get_xsk_pool(struct stmmac_priv *priv, u32 queue) +{ + if (!stmmac_xdp_is_enabled(priv) || !test_bit(queue, priv->af_xdp_zc_qps)) + return NULL; + + return xsk_get_pool_from_qid(priv->dev, queue); +} + +/** + * __init_dma_rx_desc_rings - init the RX descriptor ring (per queue) + * @priv: driver private structure + * @dma_conf: structure to take the dma data + * @queue: RX queue index * @flags: gfp flag. * Description: this function initializes the DMA RX descriptors * and allocates the socket buffers. It supports the chained and ring * modes. */ -static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags) +static int __init_dma_rx_desc_rings(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf, + u32 queue, gfp_t flags) { - struct stmmac_priv *priv = netdev_priv(dev); - u32 rx_count = priv->plat->rx_queues_to_use; - int ret = -ENOMEM; - int queue; - int i; + struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; + int ret; - /* RX INITIALIZATION */ netif_dbg(priv, probe, priv->dev, - "SKB addresses:\nskb\t\tskb data\tdma data\n"); + "(%s) dma_rx_phy=0x%08x\n", __func__, + (u32)rx_q->dma_rx_phy); - for (queue = 0; queue < rx_count; queue++) { - struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; + stmmac_clear_rx_descriptors(priv, dma_conf, queue); - netif_dbg(priv, probe, priv->dev, - "(%s) dma_rx_phy=0x%08x\n", __func__, - (u32)rx_q->dma_rx_phy); + xdp_rxq_info_unreg_mem_model(&rx_q->xdp_rxq); - stmmac_clear_rx_descriptors(priv, queue); + rx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); - for (i = 0; i < DMA_RX_SIZE; i++) { - struct dma_desc *p; + if (rx_q->xsk_pool) { + WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, + MEM_TYPE_XSK_BUFF_POOL, + NULL)); + netdev_info(priv->dev, + "Register MEM_TYPE_XSK_BUFF_POOL RxQ-%d\n", + rx_q->queue_index); + xsk_pool_set_rxq_info(rx_q->xsk_pool, &rx_q->xdp_rxq); + } else { + WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, + MEM_TYPE_PAGE_POOL, + rx_q->page_pool)); + netdev_info(priv->dev, + "Register MEM_TYPE_PAGE_POOL RxQ-%d\n", + rx_q->queue_index); + } - if (priv->extend_desc) - p = &((rx_q->dma_erx + i)->basic); - else - p = rx_q->dma_rx + i; + if (rx_q->xsk_pool) { + /* RX XDP ZC buffer pool may not be populated, e.g. + * xdpsock TX-only. + */ + stmmac_alloc_rx_buffers_zc(priv, dma_conf, queue); + } else { + ret = stmmac_alloc_rx_buffers(priv, dma_conf, queue, flags); + if (ret < 0) + return -ENOMEM; + } - ret = stmmac_init_rx_buffers(priv, p, i, flags, - queue); - if (ret) - goto err_init_rx_buffers; - } + /* Setup the chained descriptor addresses */ + if (priv->mode == STMMAC_CHAIN_MODE) { + if (priv->extend_desc) + stmmac_mode_init(priv, rx_q->dma_erx, + rx_q->dma_rx_phy, + dma_conf->dma_rx_size, 1); + else + stmmac_mode_init(priv, rx_q->dma_rx, + rx_q->dma_rx_phy, + dma_conf->dma_rx_size, 0); + } - rx_q->cur_rx = 0; - rx_q->dirty_rx = (unsigned int)(i - DMA_RX_SIZE); + return 0; +} - /* Setup the chained descriptor addresses */ - if (priv->mode == STMMAC_CHAIN_MODE) { - if (priv->extend_desc) - stmmac_mode_init(priv, rx_q->dma_erx, - rx_q->dma_rx_phy, DMA_RX_SIZE, 1); - else - stmmac_mode_init(priv, rx_q->dma_rx, - rx_q->dma_rx_phy, DMA_RX_SIZE, 0); - } +static int init_dma_rx_desc_rings(struct net_device *dev, + struct stmmac_dma_conf *dma_conf, + gfp_t flags) +{ + struct stmmac_priv *priv = netdev_priv(dev); + u32 rx_count = priv->plat->rx_queues_to_use; + int queue; + int ret; + + /* RX INITIALIZATION */ + netif_dbg(priv, probe, priv->dev, + "SKB addresses:\nskb\t\tskb data\tdma data\n"); + + for (queue = 0; queue < rx_count; queue++) { + ret = __init_dma_rx_desc_rings(priv, dma_conf, queue, flags); + if (ret) + goto err_init_rx_buffers; } return 0; err_init_rx_buffers: while (queue >= 0) { - while (--i >= 0) - stmmac_free_rx_buffer(priv, queue, i); + struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; - if (queue == 0) - break; + if (rx_q->xsk_pool) + dma_free_rx_xskbufs(priv, dma_conf, queue); + else + dma_free_rx_skbufs(priv, dma_conf, queue); + + rx_q->buf_alloc_num = 0; + rx_q->xsk_pool = NULL; - i = DMA_RX_SIZE; queue--; } @@ -1365,337 +1730,444 @@ err_init_rx_buffers: } /** - * init_dma_tx_desc_rings - init the TX descriptor rings - * @dev: net device structure. + * __init_dma_tx_desc_rings - init the TX descriptor ring (per queue) + * @priv: driver private structure + * @dma_conf: structure to take the dma data + * @queue: TX queue index * Description: this function initializes the DMA TX descriptors * and allocates the socket buffers. It supports the chained and ring * modes. */ -static int init_dma_tx_desc_rings(struct net_device *dev) +static int __init_dma_tx_desc_rings(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf, + u32 queue) { - struct stmmac_priv *priv = netdev_priv(dev); - u32 tx_queue_cnt = priv->plat->tx_queues_to_use; - u32 queue; + struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; int i; - for (queue = 0; queue < tx_queue_cnt; queue++) { - struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; - - netif_dbg(priv, probe, priv->dev, - "(%s) dma_tx_phy=0x%08x\n", __func__, - (u32)tx_q->dma_tx_phy); - - /* Setup the chained descriptor addresses */ - if (priv->mode == STMMAC_CHAIN_MODE) { - if (priv->extend_desc) - stmmac_mode_init(priv, tx_q->dma_etx, - tx_q->dma_tx_phy, DMA_TX_SIZE, 1); - else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) - stmmac_mode_init(priv, tx_q->dma_tx, - tx_q->dma_tx_phy, DMA_TX_SIZE, 0); - } + netif_dbg(priv, probe, priv->dev, + "(%s) dma_tx_phy=0x%08x\n", __func__, + (u32)tx_q->dma_tx_phy); - for (i = 0; i < DMA_TX_SIZE; i++) { - struct dma_desc *p; - if (priv->extend_desc) - p = &((tx_q->dma_etx + i)->basic); - else if (tx_q->tbs & STMMAC_TBS_AVAIL) - p = &((tx_q->dma_entx + i)->basic); - else - p = tx_q->dma_tx + i; + /* Setup the chained descriptor addresses */ + if (priv->mode == STMMAC_CHAIN_MODE) { + if (priv->extend_desc) + stmmac_mode_init(priv, tx_q->dma_etx, + tx_q->dma_tx_phy, + dma_conf->dma_tx_size, 1); + else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) + stmmac_mode_init(priv, tx_q->dma_tx, + tx_q->dma_tx_phy, + dma_conf->dma_tx_size, 0); + } - stmmac_clear_desc(priv, p); + tx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); - tx_q->tx_skbuff_dma[i].buf = 0; - tx_q->tx_skbuff_dma[i].map_as_page = false; - tx_q->tx_skbuff_dma[i].len = 0; - tx_q->tx_skbuff_dma[i].last_segment = false; - tx_q->tx_skbuff[i] = NULL; - } + for (i = 0; i < dma_conf->dma_tx_size; i++) { + struct dma_desc *p; + + if (priv->extend_desc) + p = &((tx_q->dma_etx + i)->basic); + else if (tx_q->tbs & STMMAC_TBS_AVAIL) + p = &((tx_q->dma_entx + i)->basic); + else + p = tx_q->dma_tx + i; - tx_q->dirty_tx = 0; - tx_q->cur_tx = 0; - tx_q->mss = 0; + stmmac_clear_desc(priv, p); - netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); + tx_q->tx_skbuff_dma[i].buf = 0; + tx_q->tx_skbuff_dma[i].map_as_page = false; + tx_q->tx_skbuff_dma[i].len = 0; + tx_q->tx_skbuff_dma[i].last_segment = false; + tx_q->tx_skbuff[i] = NULL; } return 0; } +static int init_dma_tx_desc_rings(struct net_device *dev, + struct stmmac_dma_conf *dma_conf) +{ + struct stmmac_priv *priv = netdev_priv(dev); + u32 tx_queue_cnt; + u32 queue; + + tx_queue_cnt = priv->plat->tx_queues_to_use; + + for (queue = 0; queue < tx_queue_cnt; queue++) + __init_dma_tx_desc_rings(priv, dma_conf, queue); + + return 0; +} + /** * init_dma_desc_rings - init the RX/TX descriptor rings * @dev: net device structure + * @dma_conf: structure to take the dma data * @flags: gfp flag. * Description: this function initializes the DMA RX/TX descriptors * and allocates the socket buffers. It supports the chained and ring * modes. */ -static int init_dma_desc_rings(struct net_device *dev, gfp_t flags) +static int init_dma_desc_rings(struct net_device *dev, + struct stmmac_dma_conf *dma_conf, + gfp_t flags) { struct stmmac_priv *priv = netdev_priv(dev); int ret; - ret = init_dma_rx_desc_rings(dev, flags); + ret = init_dma_rx_desc_rings(dev, dma_conf, flags); if (ret) return ret; - ret = init_dma_tx_desc_rings(dev); + ret = init_dma_tx_desc_rings(dev, dma_conf); - stmmac_clear_descriptors(priv); + stmmac_clear_descriptors(priv, dma_conf); if (netif_msg_hw(priv)) - stmmac_display_rings(priv); + stmmac_display_rings(priv, dma_conf); return ret; } /** - * dma_free_rx_skbufs - free RX dma buffers + * dma_free_tx_skbufs - free TX dma buffers * @priv: private structure - * @queue: RX queue index + * @dma_conf: structure to take the dma data + * @queue: TX queue index */ -static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue) +static void dma_free_tx_skbufs(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf, + u32 queue) { + struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; int i; - for (i = 0; i < DMA_RX_SIZE; i++) - stmmac_free_rx_buffer(priv, queue, i); + tx_q->xsk_frames_done = 0; + + for (i = 0; i < dma_conf->dma_tx_size; i++) + stmmac_free_tx_buffer(priv, dma_conf, queue, i); + + if (tx_q->xsk_pool && tx_q->xsk_frames_done) { + xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); + tx_q->xsk_frames_done = 0; + tx_q->xsk_pool = NULL; + } } /** - * dma_free_tx_skbufs - free TX dma buffers + * stmmac_free_tx_skbufs - free TX skb buffers * @priv: private structure - * @queue: TX queue index */ -static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue) +static void stmmac_free_tx_skbufs(struct stmmac_priv *priv) { - int i; + u32 tx_queue_cnt = priv->plat->tx_queues_to_use; + u32 queue; - for (i = 0; i < DMA_TX_SIZE; i++) - stmmac_free_tx_buffer(priv, queue, i); + for (queue = 0; queue < tx_queue_cnt; queue++) + dma_free_tx_skbufs(priv, &priv->dma_conf, queue); } /** - * free_dma_rx_desc_resources - free RX dma desc resources + * __free_dma_rx_desc_resources - free RX dma desc resources (per queue) * @priv: private structure + * @dma_conf: structure to take the dma data + * @queue: RX queue index */ -static void free_dma_rx_desc_resources(struct stmmac_priv *priv) +static void __free_dma_rx_desc_resources(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf, + u32 queue) { - u32 rx_count = priv->plat->rx_queues_to_use; - u32 queue; + struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; - /* Free RX queue resources */ - for (queue = 0; queue < rx_count; queue++) { - struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; + /* Release the DMA RX socket buffers */ + if (rx_q->xsk_pool) + dma_free_rx_xskbufs(priv, dma_conf, queue); + else + dma_free_rx_skbufs(priv, dma_conf, queue); - /* Release the DMA RX socket buffers */ - dma_free_rx_skbufs(priv, queue); + rx_q->buf_alloc_num = 0; + rx_q->xsk_pool = NULL; - /* Free DMA regions of consistent memory previously allocated */ - if (!priv->extend_desc) - dma_free_coherent(priv->device, - DMA_RX_SIZE * sizeof(struct dma_desc), - rx_q->dma_rx, rx_q->dma_rx_phy); - else - dma_free_coherent(priv->device, DMA_RX_SIZE * - sizeof(struct dma_extended_desc), - rx_q->dma_erx, rx_q->dma_rx_phy); + /* Free DMA regions of consistent memory previously allocated */ + if (!priv->extend_desc) + dma_free_coherent(priv->device, dma_conf->dma_rx_size * + sizeof(struct dma_desc), + rx_q->dma_rx, rx_q->dma_rx_phy); + else + dma_free_coherent(priv->device, dma_conf->dma_rx_size * + sizeof(struct dma_extended_desc), + rx_q->dma_erx, rx_q->dma_rx_phy); - kfree(rx_q->buf_pool); - if (rx_q->page_pool) - page_pool_destroy(rx_q->page_pool); - } + if (xdp_rxq_info_is_reg(&rx_q->xdp_rxq)) + xdp_rxq_info_unreg(&rx_q->xdp_rxq); + + kfree(rx_q->buf_pool); + if (rx_q->page_pool) + page_pool_destroy(rx_q->page_pool); +} + +static void free_dma_rx_desc_resources(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf) +{ + u32 rx_count = priv->plat->rx_queues_to_use; + u32 queue; + + /* Free RX queue resources */ + for (queue = 0; queue < rx_count; queue++) + __free_dma_rx_desc_resources(priv, dma_conf, queue); } /** - * free_dma_tx_desc_resources - free TX dma desc resources + * __free_dma_tx_desc_resources - free TX dma desc resources (per queue) * @priv: private structure + * @dma_conf: structure to take the dma data + * @queue: TX queue index */ -static void free_dma_tx_desc_resources(struct stmmac_priv *priv) +static void __free_dma_tx_desc_resources(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf, + u32 queue) { - u32 tx_count = priv->plat->tx_queues_to_use; - u32 queue; + struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; + size_t size; + void *addr; - /* Free TX queue resources */ - for (queue = 0; queue < tx_count; queue++) { - struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; - size_t size; - void *addr; + /* Release the DMA TX socket buffers */ + dma_free_tx_skbufs(priv, dma_conf, queue); + + if (priv->extend_desc) { + size = sizeof(struct dma_extended_desc); + addr = tx_q->dma_etx; + } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { + size = sizeof(struct dma_edesc); + addr = tx_q->dma_entx; + } else { + size = sizeof(struct dma_desc); + addr = tx_q->dma_tx; + } - /* Release the DMA TX socket buffers */ - dma_free_tx_skbufs(priv, queue); + size *= dma_conf->dma_tx_size; - if (priv->extend_desc) { - size = sizeof(struct dma_extended_desc); - addr = tx_q->dma_etx; - } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { - size = sizeof(struct dma_edesc); - addr = tx_q->dma_entx; - } else { - size = sizeof(struct dma_desc); - addr = tx_q->dma_tx; - } + dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy); - size *= DMA_TX_SIZE; + kfree(tx_q->tx_skbuff_dma); + kfree(tx_q->tx_skbuff); +} - dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy); +static void free_dma_tx_desc_resources(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf) +{ + u32 tx_count = priv->plat->tx_queues_to_use; + u32 queue; - kfree(tx_q->tx_skbuff_dma); - kfree(tx_q->tx_skbuff); - } + /* Free TX queue resources */ + for (queue = 0; queue < tx_count; queue++) + __free_dma_tx_desc_resources(priv, dma_conf, queue); } /** - * alloc_dma_rx_desc_resources - alloc RX resources. + * __alloc_dma_rx_desc_resources - alloc RX resources (per queue). * @priv: private structure + * @dma_conf: structure to take the dma data + * @queue: RX queue index * Description: according to which descriptor can be used (extend or basic) * this function allocates the resources for TX and RX paths. In case of * reception, for example, it pre-allocated the RX socket buffer in order to * allow zero-copy mechanism. */ -static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv) +static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf, + u32 queue) +{ + struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; + struct stmmac_channel *ch = &priv->channel[queue]; + bool xdp_prog = stmmac_xdp_is_enabled(priv); + struct page_pool_params pp_params = { 0 }; + unsigned int num_pages; + unsigned int napi_id; + int ret; + + rx_q->queue_index = queue; + rx_q->priv_data = priv; + + pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; + pp_params.pool_size = dma_conf->dma_rx_size; + num_pages = DIV_ROUND_UP(dma_conf->dma_buf_sz, PAGE_SIZE); + pp_params.order = ilog2(num_pages); + pp_params.nid = dev_to_node(priv->device); + pp_params.dev = priv->device; + pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; + pp_params.offset = stmmac_rx_offset(priv); + pp_params.max_len = STMMAC_MAX_RX_BUF_SIZE(num_pages); + + rx_q->page_pool = page_pool_create(&pp_params); + if (IS_ERR(rx_q->page_pool)) { + ret = PTR_ERR(rx_q->page_pool); + rx_q->page_pool = NULL; + return ret; + } + + rx_q->buf_pool = kcalloc(dma_conf->dma_rx_size, + sizeof(*rx_q->buf_pool), + GFP_KERNEL); + if (!rx_q->buf_pool) + return -ENOMEM; + + if (priv->extend_desc) { + rx_q->dma_erx = dma_alloc_coherent(priv->device, + dma_conf->dma_rx_size * + sizeof(struct dma_extended_desc), + &rx_q->dma_rx_phy, + GFP_KERNEL); + if (!rx_q->dma_erx) + return -ENOMEM; + + } else { + rx_q->dma_rx = dma_alloc_coherent(priv->device, + dma_conf->dma_rx_size * + sizeof(struct dma_desc), + &rx_q->dma_rx_phy, + GFP_KERNEL); + if (!rx_q->dma_rx) + return -ENOMEM; + } + + if (stmmac_xdp_is_enabled(priv) && + test_bit(queue, priv->af_xdp_zc_qps)) + napi_id = ch->rxtx_napi.napi_id; + else + napi_id = ch->rx_napi.napi_id; + + ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev, + rx_q->queue_index, + napi_id); + if (ret) { + netdev_err(priv->dev, "Failed to register xdp rxq info\n"); + return -EINVAL; + } + + return 0; +} + +static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf) { u32 rx_count = priv->plat->rx_queues_to_use; - int ret = -ENOMEM; u32 queue; + int ret; /* RX queues buffers and DMA */ for (queue = 0; queue < rx_count; queue++) { - struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; - struct page_pool_params pp_params = { 0 }; - unsigned int num_pages; - - rx_q->queue_index = queue; - rx_q->priv_data = priv; - - pp_params.flags = PP_FLAG_DMA_MAP; - pp_params.pool_size = DMA_RX_SIZE; - num_pages = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE); - pp_params.order = ilog2(num_pages); - pp_params.nid = dev_to_node(priv->device); - pp_params.dev = priv->device; - pp_params.dma_dir = DMA_FROM_DEVICE; - - rx_q->page_pool = page_pool_create(&pp_params); - if (IS_ERR(rx_q->page_pool)) { - ret = PTR_ERR(rx_q->page_pool); - rx_q->page_pool = NULL; - goto err_dma; - } - - rx_q->buf_pool = kcalloc(DMA_RX_SIZE, sizeof(*rx_q->buf_pool), - GFP_KERNEL); - if (!rx_q->buf_pool) + ret = __alloc_dma_rx_desc_resources(priv, dma_conf, queue); + if (ret) goto err_dma; - - if (priv->extend_desc) { - rx_q->dma_erx = dma_alloc_coherent(priv->device, - DMA_RX_SIZE * sizeof(struct dma_extended_desc), - &rx_q->dma_rx_phy, - GFP_KERNEL); - if (!rx_q->dma_erx) - goto err_dma; - - } else { - rx_q->dma_rx = dma_alloc_coherent(priv->device, - DMA_RX_SIZE * sizeof(struct dma_desc), - &rx_q->dma_rx_phy, - GFP_KERNEL); - if (!rx_q->dma_rx) - goto err_dma; - } } return 0; err_dma: - free_dma_rx_desc_resources(priv); + free_dma_rx_desc_resources(priv, dma_conf); return ret; } /** - * alloc_dma_tx_desc_resources - alloc TX resources. + * __alloc_dma_tx_desc_resources - alloc TX resources (per queue). * @priv: private structure + * @dma_conf: structure to take the dma data + * @queue: TX queue index * Description: according to which descriptor can be used (extend or basic) * this function allocates the resources for TX and RX paths. In case of * reception, for example, it pre-allocated the RX socket buffer in order to * allow zero-copy mechanism. */ -static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv) +static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf, + u32 queue) { - u32 tx_count = priv->plat->tx_queues_to_use; - int ret = -ENOMEM; - u32 queue; + struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; + size_t size; + void *addr; - /* TX queues buffers and DMA */ - for (queue = 0; queue < tx_count; queue++) { - struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; - size_t size; - void *addr; + tx_q->queue_index = queue; + tx_q->priv_data = priv; - tx_q->queue_index = queue; - tx_q->priv_data = priv; + tx_q->tx_skbuff_dma = kcalloc(dma_conf->dma_tx_size, + sizeof(*tx_q->tx_skbuff_dma), + GFP_KERNEL); + if (!tx_q->tx_skbuff_dma) + return -ENOMEM; - tx_q->tx_skbuff_dma = kcalloc(DMA_TX_SIZE, - sizeof(*tx_q->tx_skbuff_dma), - GFP_KERNEL); - if (!tx_q->tx_skbuff_dma) - goto err_dma; + tx_q->tx_skbuff = kcalloc(dma_conf->dma_tx_size, + sizeof(struct sk_buff *), + GFP_KERNEL); + if (!tx_q->tx_skbuff) + return -ENOMEM; - tx_q->tx_skbuff = kcalloc(DMA_TX_SIZE, - sizeof(struct sk_buff *), - GFP_KERNEL); - if (!tx_q->tx_skbuff) - goto err_dma; + if (priv->extend_desc) + size = sizeof(struct dma_extended_desc); + else if (tx_q->tbs & STMMAC_TBS_AVAIL) + size = sizeof(struct dma_edesc); + else + size = sizeof(struct dma_desc); - if (priv->extend_desc) - size = sizeof(struct dma_extended_desc); - else if (tx_q->tbs & STMMAC_TBS_AVAIL) - size = sizeof(struct dma_edesc); - else - size = sizeof(struct dma_desc); + size *= dma_conf->dma_tx_size; - size *= DMA_TX_SIZE; + addr = dma_alloc_coherent(priv->device, size, + &tx_q->dma_tx_phy, GFP_KERNEL); + if (!addr) + return -ENOMEM; - addr = dma_alloc_coherent(priv->device, size, - &tx_q->dma_tx_phy, GFP_KERNEL); - if (!addr) - goto err_dma; + if (priv->extend_desc) + tx_q->dma_etx = addr; + else if (tx_q->tbs & STMMAC_TBS_AVAIL) + tx_q->dma_entx = addr; + else + tx_q->dma_tx = addr; - if (priv->extend_desc) - tx_q->dma_etx = addr; - else if (tx_q->tbs & STMMAC_TBS_AVAIL) - tx_q->dma_entx = addr; - else - tx_q->dma_tx = addr; + return 0; +} + +static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf) +{ + u32 tx_count = priv->plat->tx_queues_to_use; + u32 queue; + int ret; + + /* TX queues buffers and DMA */ + for (queue = 0; queue < tx_count; queue++) { + ret = __alloc_dma_tx_desc_resources(priv, dma_conf, queue); + if (ret) + goto err_dma; } return 0; err_dma: - free_dma_tx_desc_resources(priv); + free_dma_tx_desc_resources(priv, dma_conf); return ret; } /** * alloc_dma_desc_resources - alloc TX/RX resources. * @priv: private structure + * @dma_conf: structure to take the dma data * Description: according to which descriptor can be used (extend or basic) * this function allocates the resources for TX and RX paths. In case of * reception, for example, it pre-allocated the RX socket buffer in order to * allow zero-copy mechanism. */ -static int alloc_dma_desc_resources(struct stmmac_priv *priv) +static int alloc_dma_desc_resources(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf) { /* RX Allocation */ - int ret = alloc_dma_rx_desc_resources(priv); + int ret = alloc_dma_rx_desc_resources(priv, dma_conf); if (ret) return ret; - ret = alloc_dma_tx_desc_resources(priv); + ret = alloc_dma_tx_desc_resources(priv, dma_conf); return ret; } @@ -1703,14 +2175,18 @@ static int alloc_dma_desc_resources(struct stmmac_priv *priv) /** * free_dma_desc_resources - free dma desc resources * @priv: private structure + * @dma_conf: structure to take the dma data */ -static void free_dma_desc_resources(struct stmmac_priv *priv) +static void free_dma_desc_resources(struct stmmac_priv *priv, + struct stmmac_dma_conf *dma_conf) { - /* Release the DMA RX socket buffers */ - free_dma_rx_desc_resources(priv); - /* Release the DMA TX socket buffers */ - free_dma_tx_desc_resources(priv); + free_dma_tx_desc_resources(priv, dma_conf); + + /* Release the DMA RX socket buffers later + * to ensure all pending XDP_TX buffers are returned. + */ + free_dma_rx_desc_resources(priv, dma_conf); } /** @@ -1782,6 +2258,23 @@ static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan) stmmac_stop_tx(priv, priv->ioaddr, chan); } +static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv) +{ + u32 rx_channels_count = priv->plat->rx_queues_to_use; + u32 tx_channels_count = priv->plat->tx_queues_to_use; + u32 dma_csr_ch = max(rx_channels_count, tx_channels_count); + u32 chan; + + for (chan = 0; chan < dma_csr_ch; chan++) { + struct stmmac_channel *ch = &priv->channel[chan]; + unsigned long flags; + + spin_lock_irqsave(&ch->lock, flags); + stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); + spin_unlock_irqrestore(&ch->lock, flags); + } +} + /** * stmmac_start_all_dma - start all RX and TX DMA channels * @priv: driver private structure @@ -1867,12 +2360,24 @@ static void stmmac_dma_operation_mode(struct stmmac_priv *priv) /* configure all channels */ for (chan = 0; chan < rx_channels_count; chan++) { + struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; + u32 buf_size; + qmode = priv->plat->rx_queues_cfg[chan].mode_to_use; stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, qmode); - stmmac_set_dma_bfsize(priv, priv->ioaddr, priv->dma_buf_sz, - chan); + + if (rx_q->xsk_pool) { + buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); + stmmac_set_dma_bfsize(priv, priv->ioaddr, + buf_size, + chan); + } else { + stmmac_set_dma_bfsize(priv, priv->ioaddr, + priv->dma_conf.dma_buf_sz, + chan); + } } for (chan = 0; chan < tx_channels_count; chan++) { @@ -1883,28 +2388,156 @@ static void stmmac_dma_operation_mode(struct stmmac_priv *priv) } } +static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) +{ + struct netdev_queue *nq = netdev_get_tx_queue(priv->dev, queue); + struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; + struct xsk_buff_pool *pool = tx_q->xsk_pool; + unsigned int entry = tx_q->cur_tx; + struct dma_desc *tx_desc = NULL; + struct xdp_desc xdp_desc; + bool work_done = true; + + /* Avoids TX time-out as we are sharing with slow path */ + txq_trans_cond_update(nq); + + budget = min(budget, stmmac_tx_avail(priv, queue)); + + while (budget-- > 0) { + dma_addr_t dma_addr; + bool set_ic; + + /* We are sharing with slow path and stop XSK TX desc submission when + * available TX ring is less than threshold. + */ + if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL) || + !netif_carrier_ok(priv->dev)) { + work_done = false; + break; + } + + if (!xsk_tx_peek_desc(pool, &xdp_desc)) + break; + + if (likely(priv->extend_desc)) + tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); + else if (tx_q->tbs & STMMAC_TBS_AVAIL) + tx_desc = &tx_q->dma_entx[entry].basic; + else + tx_desc = tx_q->dma_tx + entry; + + dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc.addr); + xsk_buff_raw_dma_sync_for_device(pool, dma_addr, xdp_desc.len); + + tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XSK_TX; + + /* To return XDP buffer to XSK pool, we simple call + * xsk_tx_completed(), so we don't need to fill up + * 'buf' and 'xdpf'. + */ + tx_q->tx_skbuff_dma[entry].buf = 0; + tx_q->xdpf[entry] = NULL; + + tx_q->tx_skbuff_dma[entry].map_as_page = false; + tx_q->tx_skbuff_dma[entry].len = xdp_desc.len; + tx_q->tx_skbuff_dma[entry].last_segment = true; + tx_q->tx_skbuff_dma[entry].is_jumbo = false; + + stmmac_set_desc_addr(priv, tx_desc, dma_addr); + + tx_q->tx_count_frames++; + + if (!priv->tx_coal_frames[queue]) + set_ic = false; + else if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) + set_ic = true; + else + set_ic = false; + + if (set_ic) { + tx_q->tx_count_frames = 0; + stmmac_set_tx_ic(priv, tx_desc); + priv->xstats.tx_set_ic_bit++; + } + + stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len, + true, priv->mode, true, true, + xdp_desc.len); + + stmmac_enable_dma_transmission(priv, priv->ioaddr); + + tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); + entry = tx_q->cur_tx; + } + + if (tx_desc) { + stmmac_flush_tx_descriptors(priv, queue); + xsk_tx_release(pool); + } + + /* Return true if all of the 3 conditions are met + * a) TX Budget is still available + * b) work_done = true when XSK TX desc peek is empty (no more + * pending XSK TX for transmission) + */ + return !!budget && work_done; +} + +static void stmmac_bump_dma_threshold(struct stmmac_priv *priv, u32 chan) +{ + if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && tc <= 256) { + tc += 64; + + if (priv->plat->force_thresh_dma_mode) + stmmac_set_dma_operation_mode(priv, tc, tc, chan); + else + stmmac_set_dma_operation_mode(priv, tc, SF_DMA_MODE, + chan); + + priv->xstats.threshold = tc; + } +} + /** * stmmac_tx_clean - to manage the transmission completion * @priv: driver private structure + * @budget: napi budget limiting this functions packet handling * @queue: TX queue index * Description: it reclaims the transmit resources after transmission completes. */ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue) { - struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; + struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; unsigned int bytes_compl = 0, pkts_compl = 0; - unsigned int entry, count = 0; + unsigned int entry, xmits = 0, count = 0; __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue)); priv->xstats.tx_clean++; + tx_q->xsk_frames_done = 0; + entry = tx_q->dirty_tx; - while ((entry != tx_q->cur_tx) && (count < budget)) { - struct sk_buff *skb = tx_q->tx_skbuff[entry]; + + /* Try to clean all TX complete frame in 1 shot */ + while ((entry != tx_q->cur_tx) && count < priv->dma_conf.dma_tx_size) { + struct xdp_frame *xdpf; + struct sk_buff *skb; struct dma_desc *p; int status; + if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX || + tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { + xdpf = tx_q->xdpf[entry]; + skb = NULL; + } else if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { + xdpf = NULL; + skb = tx_q->tx_skbuff[entry]; + } else { + xdpf = NULL; + skb = NULL; + } + if (priv->extend_desc) p = (struct dma_desc *)(tx_q->dma_etx + entry); else if (tx_q->tbs & STMMAC_TBS_AVAIL) @@ -1930,14 +2563,19 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue) /* ... verify the status error condition */ if (unlikely(status & tx_err)) { priv->dev->stats.tx_errors++; + if (unlikely(status & tx_err_bump_tc)) + stmmac_bump_dma_threshold(priv, queue); } else { priv->dev->stats.tx_packets++; priv->xstats.tx_pkt_n++; + priv->xstats.txq_stats[queue].tx_pkt_n++; } - stmmac_get_tx_hwtstamp(priv, p, skb); + if (skb) + stmmac_get_tx_hwtstamp(priv, p, skb); } - if (likely(tx_q->tx_skbuff_dma[entry].buf)) { + if (likely(tx_q->tx_skbuff_dma[entry].buf && + tx_q->tx_skbuff_dma[entry].buf_type != STMMAC_TXBUF_T_XDP_TX)) { if (tx_q->tx_skbuff_dma[entry].map_as_page) dma_unmap_page(priv->device, tx_q->tx_skbuff_dma[entry].buf, @@ -1958,16 +2596,33 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue) tx_q->tx_skbuff_dma[entry].last_segment = false; tx_q->tx_skbuff_dma[entry].is_jumbo = false; - if (likely(skb != NULL)) { - pkts_compl++; - bytes_compl += skb->len; - dev_consume_skb_any(skb); - tx_q->tx_skbuff[entry] = NULL; + if (xdpf && + tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX) { + xdp_return_frame_rx_napi(xdpf); + tx_q->xdpf[entry] = NULL; + } + + if (xdpf && + tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { + xdp_return_frame(xdpf); + tx_q->xdpf[entry] = NULL; + } + + if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XSK_TX) + tx_q->xsk_frames_done++; + + if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { + if (likely(skb)) { + pkts_compl++; + bytes_compl += skb->len; + dev_consume_skb_any(skb); + tx_q->tx_skbuff[entry] = NULL; + } } stmmac_release_tx_desc(priv, p, priv->mode); - entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE); + entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); } tx_q->dirty_tx = entry; @@ -1976,25 +2631,51 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue) if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev, queue))) && - stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH) { + stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH(priv)) { netif_dbg(priv, tx_done, priv->dev, "%s: restart transmit\n", __func__); netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue)); } - if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) { - stmmac_enable_eee_mode(priv); - mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer)); + if (tx_q->xsk_pool) { + bool work_done; + + if (tx_q->xsk_frames_done) + xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); + + if (xsk_uses_need_wakeup(tx_q->xsk_pool)) + xsk_set_tx_need_wakeup(tx_q->xsk_pool); + + /* For XSK TX, we try to send as many as possible. + * If XSK work done (XSK TX desc empty and budget still + * available), return "budget - 1" to reenable TX IRQ. + * Else, return "budget" to make NAPI continue polling. + */ + work_done = stmmac_xdp_xmit_zc(priv, queue, + STMMAC_XSK_TX_BUDGET_MAX); + if (work_done) + xmits = budget - 1; + else + xmits = budget; + } + + if (priv->eee_enabled && !priv->tx_path_in_lpi_mode && + priv->eee_sw_timer_en) { + if (stmmac_enable_eee_mode(priv)) + mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); } /* We still have pending packets, let's call for a new scheduling */ if (tx_q->dirty_tx != tx_q->cur_tx) - mod_timer(&tx_q->txtimer, STMMAC_COAL_TIMER(priv->tx_coal_timer)); + hrtimer_start(&tx_q->txtimer, + STMMAC_COAL_TIMER(priv->tx_coal_timer[queue]), + HRTIMER_MODE_REL); __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue)); - return count; + /* Combine decisions from TX clean and XSK TX */ + return max(count, xmits); } /** @@ -2006,17 +2687,14 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue) */ static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan) { - struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; + struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan)); stmmac_stop_tx_dma(priv, chan); - dma_free_tx_skbufs(priv, chan); - stmmac_clear_tx_descriptors(priv, chan); - tx_q->dirty_tx = 0; - tx_q->cur_tx = 0; - tx_q->mss = 0; - netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, chan)); + dma_free_tx_skbufs(priv, &priv->dma_conf, chan); + stmmac_clear_tx_descriptors(priv, &priv->dma_conf, chan); + stmmac_reset_tx_queue(priv, chan); stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, tx_q->dma_tx_phy, chan); stmmac_start_tx_dma(priv, chan); @@ -2072,28 +2750,35 @@ static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv) return false; } -static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan) +static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir) { int status = stmmac_dma_interrupt_status(priv, priv->ioaddr, - &priv->xstats, chan); + &priv->xstats, chan, dir); + struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; + struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; struct stmmac_channel *ch = &priv->channel[chan]; + struct napi_struct *rx_napi; + struct napi_struct *tx_napi; unsigned long flags; + rx_napi = rx_q->xsk_pool ? &ch->rxtx_napi : &ch->rx_napi; + tx_napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; + if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) { - if (napi_schedule_prep(&ch->rx_napi)) { + if (napi_schedule_prep(rx_napi)) { spin_lock_irqsave(&ch->lock, flags); stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0); spin_unlock_irqrestore(&ch->lock, flags); - __napi_schedule_irqoff(&ch->rx_napi); + __napi_schedule(rx_napi); } } if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) { - if (napi_schedule_prep(&ch->tx_napi)) { + if (napi_schedule_prep(tx_napi)) { spin_lock_irqsave(&ch->lock, flags); stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1); spin_unlock_irqrestore(&ch->lock, flags); - __napi_schedule_irqoff(&ch->tx_napi); + __napi_schedule(tx_napi); } } @@ -2121,26 +2806,13 @@ static void stmmac_dma_interrupt(struct stmmac_priv *priv) channels_to_check = ARRAY_SIZE(status); for (chan = 0; chan < channels_to_check; chan++) - status[chan] = stmmac_napi_check(priv, chan); + status[chan] = stmmac_napi_check(priv, chan, + DMA_DIR_RXTX); for (chan = 0; chan < tx_channel_count; chan++) { if (unlikely(status[chan] & tx_hard_error_bump_tc)) { /* Try to bump up the dma threshold on this failure */ - if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && - (tc <= 256)) { - tc += 64; - if (priv->plat->force_thresh_dma_mode) - stmmac_set_dma_operation_mode(priv, - tc, - tc, - chan); - else - stmmac_set_dma_operation_mode(priv, - tc, - SF_DMA_MODE, - chan); - priv->xstats.threshold = tc; - } + stmmac_bump_dma_threshold(priv, chan); } else if (unlikely(status[chan] == tx_hard_error)) { stmmac_tx_err(priv, chan); } @@ -2189,9 +2861,13 @@ static int stmmac_get_hw_features(struct stmmac_priv *priv) */ static void stmmac_check_ether_addr(struct stmmac_priv *priv) { + u8 addr[ETH_ALEN]; + if (!is_valid_ether_addr(priv->dev->dev_addr)) { - stmmac_get_umac_addr(priv, priv->hw, priv->dev->dev_addr, 0); - if (!is_valid_ether_addr(priv->dev->dev_addr)) + stmmac_get_umac_addr(priv, priv->hw, addr, 0); + if (is_valid_ether_addr(addr)) + eth_hw_addr_set(priv->dev, addr); + else eth_hw_addr_random(priv->dev); dev_info(priv->device, "device MAC address %pM\n", priv->dev->dev_addr); @@ -2238,25 +2914,28 @@ static int stmmac_init_dma_engine(struct stmmac_priv *priv) stmmac_axi(priv, priv->ioaddr, priv->plat->axi); /* DMA CSR Channel configuration */ - for (chan = 0; chan < dma_csr_ch; chan++) + for (chan = 0; chan < dma_csr_ch; chan++) { stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); + stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); + } /* DMA RX Channel Configuration */ for (chan = 0; chan < rx_channels_count; chan++) { - rx_q = &priv->rx_queue[chan]; + rx_q = &priv->dma_conf.rx_queue[chan]; stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, rx_q->dma_rx_phy, chan); rx_q->rx_tail_addr = rx_q->dma_rx_phy + - (DMA_RX_SIZE * sizeof(struct dma_desc)); + (rx_q->buf_alloc_num * + sizeof(struct dma_desc)); stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, chan); } /* DMA TX Channel Configuration */ for (chan = 0; chan < tx_channels_count; chan++) { - tx_q = &priv->tx_queue[chan]; + tx_q = &priv->dma_conf.tx_queue[chan]; stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, tx_q->dma_tx_phy, chan); @@ -2271,33 +2950,39 @@ static int stmmac_init_dma_engine(struct stmmac_priv *priv) static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue) { - struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; + struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; - mod_timer(&tx_q->txtimer, STMMAC_COAL_TIMER(priv->tx_coal_timer)); + hrtimer_start(&tx_q->txtimer, + STMMAC_COAL_TIMER(priv->tx_coal_timer[queue]), + HRTIMER_MODE_REL); } /** * stmmac_tx_timer - mitigation sw timer for tx. - * @data: data pointer + * @t: data pointer * Description: * This is the timer handler to directly invoke the stmmac_tx_clean. */ -static void stmmac_tx_timer(struct timer_list *t) +static enum hrtimer_restart stmmac_tx_timer(struct hrtimer *t) { - struct stmmac_tx_queue *tx_q = from_timer(tx_q, t, txtimer); + struct stmmac_tx_queue *tx_q = container_of(t, struct stmmac_tx_queue, txtimer); struct stmmac_priv *priv = tx_q->priv_data; struct stmmac_channel *ch; + struct napi_struct *napi; ch = &priv->channel[tx_q->queue_index]; + napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; - if (likely(napi_schedule_prep(&ch->tx_napi))) { + if (likely(napi_schedule_prep(napi))) { unsigned long flags; spin_lock_irqsave(&ch->lock, flags); stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1); spin_unlock_irqrestore(&ch->lock, flags); - __napi_schedule(&ch->tx_napi); + __napi_schedule(napi); } + + return HRTIMER_NORESTART; } /** @@ -2311,17 +2996,21 @@ static void stmmac_tx_timer(struct timer_list *t) static void stmmac_init_coalesce(struct stmmac_priv *priv) { u32 tx_channel_count = priv->plat->tx_queues_to_use; + u32 rx_channel_count = priv->plat->rx_queues_to_use; u32 chan; - priv->tx_coal_frames = STMMAC_TX_FRAMES; - priv->tx_coal_timer = STMMAC_COAL_TX_TIMER; - priv->rx_coal_frames = STMMAC_RX_FRAMES; - for (chan = 0; chan < tx_channel_count; chan++) { - struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; + struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; - timer_setup(&tx_q->txtimer, stmmac_tx_timer, 0); + priv->tx_coal_frames[chan] = STMMAC_TX_FRAMES; + priv->tx_coal_timer[chan] = STMMAC_COAL_TX_TIMER; + + hrtimer_init(&tx_q->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + tx_q->txtimer.function = stmmac_tx_timer; } + + for (chan = 0; chan < rx_channel_count; chan++) + priv->rx_coal_frames[chan] = STMMAC_RX_FRAMES; } static void stmmac_set_rings_length(struct stmmac_priv *priv) @@ -2333,12 +3022,12 @@ static void stmmac_set_rings_length(struct stmmac_priv *priv) /* set TX ring length */ for (chan = 0; chan < tx_channels_count; chan++) stmmac_set_tx_ring_len(priv, priv->ioaddr, - (DMA_TX_SIZE - 1), chan); + (priv->dma_conf.dma_tx_size - 1), chan); /* set RX ring length */ for (chan = 0; chan < rx_channels_count; chan++) stmmac_set_rx_ring_len(priv, priv->ioaddr, - (DMA_RX_SIZE - 1), chan); + (priv->dma_conf.dma_rx_size - 1), chan); } /** @@ -2532,15 +3221,38 @@ static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) { if (priv->dma_cap.asp) { netdev_info(priv->dev, "Enabling Safety Features\n"); - stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp); + stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp, + priv->plat->safety_feat_cfg); } else { netdev_info(priv->dev, "No Safety Features support found\n"); } } +static int stmmac_fpe_start_wq(struct stmmac_priv *priv) +{ + char *name; + + clear_bit(__FPE_TASK_SCHED, &priv->fpe_task_state); + clear_bit(__FPE_REMOVING, &priv->fpe_task_state); + + name = priv->wq_name; + sprintf(name, "%s-fpe", priv->dev->name); + + priv->fpe_wq = create_singlethread_workqueue(name); + if (!priv->fpe_wq) { + netdev_err(priv->dev, "%s: Failed to create workqueue\n", name); + + return -ENOMEM; + } + netdev_info(priv->dev, "FPE workqueue start"); + + return 0; +} + /** * stmmac_hw_setup - setup mac in a usable state. * @dev : pointer to the device structure. + * @ptp_register: register PTP if set * Description: * this is the main function to setup the HW in a usable state because the * dma engine is reset, the core registers are configured (e.g. AXI, @@ -2550,11 +3262,12 @@ static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) * 0 on success and an appropriate (-)ve integer as defined in errno.h * file on failure. */ -static int stmmac_hw_setup(struct net_device *dev, bool init_ptp) +static int stmmac_hw_setup(struct net_device *dev, bool ptp_register) { struct stmmac_priv *priv = netdev_priv(dev); u32 rx_cnt = priv->plat->rx_queues_to_use; u32 tx_cnt = priv->plat->tx_queues_to_use; + bool sph_en; u32 chan; int ret; @@ -2606,25 +3319,38 @@ static int stmmac_hw_setup(struct net_device *dev, bool init_ptp) stmmac_mmc_setup(priv); - if (init_ptp) { + if (ptp_register) { ret = clk_prepare_enable(priv->plat->clk_ptp_ref); if (ret < 0) - netdev_warn(priv->dev, "failed to enable PTP reference clock: %d\n", ret); - - ret = stmmac_init_ptp(priv); - if (ret == -EOPNOTSUPP) - netdev_warn(priv->dev, "PTP not supported by HW\n"); - else if (ret) - netdev_warn(priv->dev, "PTP init failed\n"); + netdev_warn(priv->dev, + "failed to enable PTP reference clock: %pe\n", + ERR_PTR(ret)); } - priv->tx_lpi_timer = STMMAC_DEFAULT_TWT_LS; + ret = stmmac_init_ptp(priv); + if (ret == -EOPNOTSUPP) + netdev_info(priv->dev, "PTP not supported by HW\n"); + else if (ret) + netdev_warn(priv->dev, "PTP init failed\n"); + else if (ptp_register) + stmmac_ptp_register(priv); + + priv->eee_tw_timer = STMMAC_DEFAULT_TWT_LS; + + /* Convert the timer from msec to usec */ + if (!priv->tx_lpi_timer) + priv->tx_lpi_timer = eee_timer * 1000; if (priv->use_riwt) { - if (!priv->rx_riwt) - priv->rx_riwt = DEF_DMA_RIWT; + u32 queue; + + for (queue = 0; queue < rx_cnt; queue++) { + if (!priv->rx_riwt[queue]) + priv->rx_riwt[queue] = DEF_DMA_RIWT; - ret = stmmac_rx_watchdog(priv, priv->ioaddr, priv->rx_riwt, rx_cnt); + stmmac_rx_watchdog(priv, priv->ioaddr, + priv->rx_riwt[queue], queue); + } } if (priv->hw->pcs) @@ -2635,15 +3361,22 @@ static int stmmac_hw_setup(struct net_device *dev, bool init_ptp) /* Enable TSO */ if (priv->tso) { - for (chan = 0; chan < tx_cnt; chan++) + for (chan = 0; chan < tx_cnt; chan++) { + struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; + + /* TSO and TBS cannot co-exist */ + if (tx_q->tbs & STMMAC_TBS_AVAIL) + continue; + stmmac_enable_tso(priv, priv->ioaddr, 1, chan); + } } /* Enable Split Header */ - if (priv->sph && priv->hw->rx_csum) { - for (chan = 0; chan < rx_cnt; chan++) - stmmac_enable_sph(priv, priv->ioaddr, 1, chan); - } + sph_en = (priv->hw->rx_csum > 0) && priv->sph; + for (chan = 0; chan < rx_cnt; chan++) + stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); + /* VLAN Tag Insertion */ if (priv->dma_cap.vlins) @@ -2651,15 +3384,26 @@ static int stmmac_hw_setup(struct net_device *dev, bool init_ptp) /* TBS */ for (chan = 0; chan < tx_cnt; chan++) { - struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; + struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; int enable = tx_q->tbs & STMMAC_TBS_AVAIL; stmmac_enable_tbs(priv, priv->ioaddr, enable, chan); } + /* Configure real RX and TX queues */ + netif_set_real_num_rx_queues(dev, priv->plat->rx_queues_to_use); + netif_set_real_num_tx_queues(dev, priv->plat->tx_queues_to_use); + /* Start the ball rolling... */ stmmac_start_all_dma(priv); + if (priv->dma_cap.fpesel) { + stmmac_fpe_start_wq(priv); + + if (priv->plat->fpe_cfg->enable) + stmmac_fpe_handshake(priv, true); + } + return 0; } @@ -2670,141 +3414,465 @@ static void stmmac_hw_teardown(struct net_device *dev) clk_disable_unprepare(priv->plat->clk_ptp_ref); } -/** - * stmmac_open - open entry point of the driver - * @dev : pointer to the device structure. - * Description: - * This function is the open entry point of the driver. - * Return value: - * 0 on success and an appropriate (-)ve integer as defined in errno.h - * file on failure. - */ -static int stmmac_open(struct net_device *dev) +static void stmmac_free_irq(struct net_device *dev, + enum request_irq_err irq_err, int irq_idx) { struct stmmac_priv *priv = netdev_priv(dev); - int bfsize = 0; - u32 chan; + int j; + + switch (irq_err) { + case REQ_IRQ_ERR_ALL: + irq_idx = priv->plat->tx_queues_to_use; + fallthrough; + case REQ_IRQ_ERR_TX: + for (j = irq_idx - 1; j >= 0; j--) { + if (priv->tx_irq[j] > 0) { + irq_set_affinity_hint(priv->tx_irq[j], NULL); + free_irq(priv->tx_irq[j], &priv->dma_conf.tx_queue[j]); + } + } + irq_idx = priv->plat->rx_queues_to_use; + fallthrough; + case REQ_IRQ_ERR_RX: + for (j = irq_idx - 1; j >= 0; j--) { + if (priv->rx_irq[j] > 0) { + irq_set_affinity_hint(priv->rx_irq[j], NULL); + free_irq(priv->rx_irq[j], &priv->dma_conf.rx_queue[j]); + } + } + + if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) + free_irq(priv->sfty_ue_irq, dev); + fallthrough; + case REQ_IRQ_ERR_SFTY_UE: + if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) + free_irq(priv->sfty_ce_irq, dev); + fallthrough; + case REQ_IRQ_ERR_SFTY_CE: + if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) + free_irq(priv->lpi_irq, dev); + fallthrough; + case REQ_IRQ_ERR_LPI: + if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) + free_irq(priv->wol_irq, dev); + fallthrough; + case REQ_IRQ_ERR_WOL: + free_irq(dev->irq, dev); + fallthrough; + case REQ_IRQ_ERR_MAC: + case REQ_IRQ_ERR_NO: + /* If MAC IRQ request error, no more IRQ to free */ + break; + } +} + +static int stmmac_request_irq_multi_msi(struct net_device *dev) +{ + struct stmmac_priv *priv = netdev_priv(dev); + enum request_irq_err irq_err; + cpumask_t cpu_mask; + int irq_idx = 0; + char *int_name; int ret; + int i; - if (priv->hw->pcs != STMMAC_PCS_TBI && - priv->hw->pcs != STMMAC_PCS_RTBI) { - ret = stmmac_init_phy(dev); - if (ret) { + /* For common interrupt */ + int_name = priv->int_name_mac; + sprintf(int_name, "%s:%s", dev->name, "mac"); + ret = request_irq(dev->irq, stmmac_mac_interrupt, + 0, int_name, dev); + if (unlikely(ret < 0)) { + netdev_err(priv->dev, + "%s: alloc mac MSI %d (error: %d)\n", + __func__, dev->irq, ret); + irq_err = REQ_IRQ_ERR_MAC; + goto irq_error; + } + + /* Request the Wake IRQ in case of another line + * is used for WoL + */ + if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { + int_name = priv->int_name_wol; + sprintf(int_name, "%s:%s", dev->name, "wol"); + ret = request_irq(priv->wol_irq, + stmmac_mac_interrupt, + 0, int_name, dev); + if (unlikely(ret < 0)) { netdev_err(priv->dev, - "%s: Cannot attach to PHY (error: %d)\n", - __func__, ret); - return ret; + "%s: alloc wol MSI %d (error: %d)\n", + __func__, priv->wol_irq, ret); + irq_err = REQ_IRQ_ERR_WOL; + goto irq_error; } } - /* Extra statistics */ - memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats)); - priv->xstats.threshold = tc; - - bfsize = stmmac_set_16kib_bfsize(priv, dev->mtu); - if (bfsize < 0) - bfsize = 0; - - if (bfsize < BUF_SIZE_16KiB) - bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz); + /* Request the LPI IRQ in case of another line + * is used for LPI + */ + if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { + int_name = priv->int_name_lpi; + sprintf(int_name, "%s:%s", dev->name, "lpi"); + ret = request_irq(priv->lpi_irq, + stmmac_mac_interrupt, + 0, int_name, dev); + if (unlikely(ret < 0)) { + netdev_err(priv->dev, + "%s: alloc lpi MSI %d (error: %d)\n", + __func__, priv->lpi_irq, ret); + irq_err = REQ_IRQ_ERR_LPI; + goto irq_error; + } + } - priv->dma_buf_sz = bfsize; - buf_sz = bfsize; + /* Request the Safety Feature Correctible Error line in + * case of another line is used + */ + if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) { + int_name = priv->int_name_sfty_ce; + sprintf(int_name, "%s:%s", dev->name, "safety-ce"); + ret = request_irq(priv->sfty_ce_irq, + stmmac_safety_interrupt, + 0, int_name, dev); + if (unlikely(ret < 0)) { + netdev_err(priv->dev, + "%s: alloc sfty ce MSI %d (error: %d)\n", + __func__, priv->sfty_ce_irq, ret); + irq_err = REQ_IRQ_ERR_SFTY_CE; + goto irq_error; + } + } - priv->rx_copybreak = STMMAC_RX_COPYBREAK; + /* Request the Safety Feature Uncorrectible Error line in + * case of another line is used + */ + if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) { + int_name = priv->int_name_sfty_ue; + sprintf(int_name, "%s:%s", dev->name, "safety-ue"); + ret = request_irq(priv->sfty_ue_irq, + stmmac_safety_interrupt, + 0, int_name, dev); + if (unlikely(ret < 0)) { + netdev_err(priv->dev, + "%s: alloc sfty ue MSI %d (error: %d)\n", + __func__, priv->sfty_ue_irq, ret); + irq_err = REQ_IRQ_ERR_SFTY_UE; + goto irq_error; + } + } - /* Earlier check for TBS */ - for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) { - struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; - int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en; + /* Request Rx MSI irq */ + for (i = 0; i < priv->plat->rx_queues_to_use; i++) { + if (i >= MTL_MAX_RX_QUEUES) + break; + if (priv->rx_irq[i] == 0) + continue; - tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0; - if (stmmac_enable_tbs(priv, priv->ioaddr, tbs_en, chan)) - tx_q->tbs &= ~STMMAC_TBS_AVAIL; + int_name = priv->int_name_rx_irq[i]; + sprintf(int_name, "%s:%s-%d", dev->name, "rx", i); + ret = request_irq(priv->rx_irq[i], + stmmac_msi_intr_rx, + 0, int_name, &priv->dma_conf.rx_queue[i]); + if (unlikely(ret < 0)) { + netdev_err(priv->dev, + "%s: alloc rx-%d MSI %d (error: %d)\n", + __func__, i, priv->rx_irq[i], ret); + irq_err = REQ_IRQ_ERR_RX; + irq_idx = i; + goto irq_error; + } + cpumask_clear(&cpu_mask); + cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); + irq_set_affinity_hint(priv->rx_irq[i], &cpu_mask); } - ret = alloc_dma_desc_resources(priv); - if (ret < 0) { - netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n", - __func__); - goto dma_desc_error; - } + /* Request Tx MSI irq */ + for (i = 0; i < priv->plat->tx_queues_to_use; i++) { + if (i >= MTL_MAX_TX_QUEUES) + break; + if (priv->tx_irq[i] == 0) + continue; - ret = init_dma_desc_rings(dev, GFP_KERNEL); - if (ret < 0) { - netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n", - __func__); - goto init_error; + int_name = priv->int_name_tx_irq[i]; + sprintf(int_name, "%s:%s-%d", dev->name, "tx", i); + ret = request_irq(priv->tx_irq[i], + stmmac_msi_intr_tx, + 0, int_name, &priv->dma_conf.tx_queue[i]); + if (unlikely(ret < 0)) { + netdev_err(priv->dev, + "%s: alloc tx-%d MSI %d (error: %d)\n", + __func__, i, priv->tx_irq[i], ret); + irq_err = REQ_IRQ_ERR_TX; + irq_idx = i; + goto irq_error; + } + cpumask_clear(&cpu_mask); + cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); + irq_set_affinity_hint(priv->tx_irq[i], &cpu_mask); } - ret = stmmac_hw_setup(dev, true); - if (ret < 0) { - netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); - goto init_error; - } + return 0; - stmmac_init_coalesce(priv); +irq_error: + stmmac_free_irq(dev, irq_err, irq_idx); + return ret; +} - phylink_start(priv->phylink); +static int stmmac_request_irq_single(struct net_device *dev) +{ + struct stmmac_priv *priv = netdev_priv(dev); + enum request_irq_err irq_err; + int ret; - /* Request the IRQ lines */ ret = request_irq(dev->irq, stmmac_interrupt, IRQF_SHARED, dev->name, dev); if (unlikely(ret < 0)) { netdev_err(priv->dev, "%s: ERROR: allocating the IRQ %d (error: %d)\n", __func__, dev->irq, ret); + irq_err = REQ_IRQ_ERR_MAC; goto irq_error; } - /* Request the Wake IRQ in case of another line is used for WoL */ - if (priv->wol_irq != dev->irq) { + /* Request the Wake IRQ in case of another line + * is used for WoL + */ + if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { ret = request_irq(priv->wol_irq, stmmac_interrupt, IRQF_SHARED, dev->name, dev); if (unlikely(ret < 0)) { netdev_err(priv->dev, "%s: ERROR: allocating the WoL IRQ %d (%d)\n", __func__, priv->wol_irq, ret); - goto wolirq_error; + irq_err = REQ_IRQ_ERR_WOL; + goto irq_error; } } /* Request the IRQ lines */ - if (priv->lpi_irq > 0) { - ret = request_irq(priv->lpi_irq, stmmac_interrupt, IRQF_SHARED, - dev->name, dev); + if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { + ret = request_irq(priv->lpi_irq, stmmac_interrupt, + IRQF_SHARED, dev->name, dev); if (unlikely(ret < 0)) { netdev_err(priv->dev, "%s: ERROR: allocating the LPI IRQ %d (%d)\n", __func__, priv->lpi_irq, ret); - goto lpiirq_error; + irq_err = REQ_IRQ_ERR_LPI; + goto irq_error; } } + return 0; + +irq_error: + stmmac_free_irq(dev, irq_err, 0); + return ret; +} + +static int stmmac_request_irq(struct net_device *dev) +{ + struct stmmac_priv *priv = netdev_priv(dev); + int ret; + + /* Request the IRQ lines */ + if (priv->plat->multi_msi_en) + ret = stmmac_request_irq_multi_msi(dev); + else + ret = stmmac_request_irq_single(dev); + + return ret; +} + +/** + * stmmac_setup_dma_desc - Generate a dma_conf and allocate DMA queue + * @priv: driver private structure + * @mtu: MTU to setup the dma queue and buf with + * Description: Allocate and generate a dma_conf based on the provided MTU. + * Allocate the Tx/Rx DMA queue and init them. + * Return value: + * the dma_conf allocated struct on success and an appropriate ERR_PTR on failure. + */ +static struct stmmac_dma_conf * +stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu) +{ + struct stmmac_dma_conf *dma_conf; + int chan, bfsize, ret; + + dma_conf = kzalloc(sizeof(*dma_conf), GFP_KERNEL); + if (!dma_conf) { + netdev_err(priv->dev, "%s: DMA conf allocation failed\n", + __func__); + return ERR_PTR(-ENOMEM); + } + + bfsize = stmmac_set_16kib_bfsize(priv, mtu); + if (bfsize < 0) + bfsize = 0; + + if (bfsize < BUF_SIZE_16KiB) + bfsize = stmmac_set_bfsize(mtu, 0); + + dma_conf->dma_buf_sz = bfsize; + /* Chose the tx/rx size from the already defined one in the + * priv struct. (if defined) + */ + dma_conf->dma_tx_size = priv->dma_conf.dma_tx_size; + dma_conf->dma_rx_size = priv->dma_conf.dma_rx_size; + + if (!dma_conf->dma_tx_size) + dma_conf->dma_tx_size = DMA_DEFAULT_TX_SIZE; + if (!dma_conf->dma_rx_size) + dma_conf->dma_rx_size = DMA_DEFAULT_RX_SIZE; + + /* Earlier check for TBS */ + for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) { + struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[chan]; + int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en; + + /* Setup per-TXQ tbs flag before TX descriptor alloc */ + tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0; + } + + ret = alloc_dma_desc_resources(priv, dma_conf); + if (ret < 0) { + netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n", + __func__); + goto alloc_error; + } + + ret = init_dma_desc_rings(priv->dev, dma_conf, GFP_KERNEL); + if (ret < 0) { + netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n", + __func__); + goto init_error; + } + + return dma_conf; + +init_error: + free_dma_desc_resources(priv, dma_conf); +alloc_error: + kfree(dma_conf); + return ERR_PTR(ret); +} + +/** + * __stmmac_open - open entry point of the driver + * @dev : pointer to the device structure. + * @dma_conf : structure to take the dma data + * Description: + * This function is the open entry point of the driver. + * Return value: + * 0 on success and an appropriate (-)ve integer as defined in errno.h + * file on failure. + */ +static int __stmmac_open(struct net_device *dev, + struct stmmac_dma_conf *dma_conf) +{ + struct stmmac_priv *priv = netdev_priv(dev); + int mode = priv->plat->phy_interface; + u32 chan; + int ret; + + ret = pm_runtime_resume_and_get(priv->device); + if (ret < 0) + return ret; + + if (priv->hw->pcs != STMMAC_PCS_TBI && + priv->hw->pcs != STMMAC_PCS_RTBI && + (!priv->hw->xpcs || + xpcs_get_an_mode(priv->hw->xpcs, mode) != DW_AN_C73)) { + ret = stmmac_init_phy(dev); + if (ret) { + netdev_err(priv->dev, + "%s: Cannot attach to PHY (error: %d)\n", + __func__, ret); + goto init_phy_error; + } + } + + /* Extra statistics */ + memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats)); + priv->xstats.threshold = tc; + + priv->rx_copybreak = STMMAC_RX_COPYBREAK; + + buf_sz = dma_conf->dma_buf_sz; + memcpy(&priv->dma_conf, dma_conf, sizeof(*dma_conf)); + + stmmac_reset_queues_param(priv); + + if (priv->plat->serdes_powerup) { + ret = priv->plat->serdes_powerup(dev, priv->plat->bsp_priv); + if (ret < 0) { + netdev_err(priv->dev, "%s: Serdes powerup failed\n", + __func__); + goto init_error; + } + } + + ret = stmmac_hw_setup(dev, true); + if (ret < 0) { + netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); + goto init_error; + } + + stmmac_init_coalesce(priv); + + phylink_start(priv->phylink); + /* We may have called phylink_speed_down before */ + phylink_speed_up(priv->phylink); + + ret = stmmac_request_irq(dev); + if (ret) + goto irq_error; + stmmac_enable_all_queues(priv); - stmmac_start_all_queues(priv); + netif_tx_start_all_queues(priv->dev); + stmmac_enable_all_dma_irq(priv); return 0; -lpiirq_error: - if (priv->wol_irq != dev->irq) - free_irq(priv->wol_irq, dev); -wolirq_error: - free_irq(dev->irq, dev); irq_error: phylink_stop(priv->phylink); for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) - del_timer_sync(&priv->tx_queue[chan].txtimer); + hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); stmmac_hw_teardown(dev); init_error: - free_dma_desc_resources(priv); -dma_desc_error: + free_dma_desc_resources(priv, &priv->dma_conf); phylink_disconnect_phy(priv->phylink); +init_phy_error: + pm_runtime_put(priv->device); return ret; } +static int stmmac_open(struct net_device *dev) +{ + struct stmmac_priv *priv = netdev_priv(dev); + struct stmmac_dma_conf *dma_conf; + int ret; + + dma_conf = stmmac_setup_dma_desc(priv, dev->mtu); + if (IS_ERR(dma_conf)) + return PTR_ERR(dma_conf); + + ret = __stmmac_open(dev, dma_conf); + kfree(dma_conf); + return ret; +} + +static void stmmac_fpe_stop_wq(struct stmmac_priv *priv) +{ + set_bit(__FPE_REMOVING, &priv->fpe_task_state); + + if (priv->fpe_wq) + destroy_workqueue(priv->fpe_wq); + + netdev_info(priv->dev, "FPE workqueue stop"); +} + /** * stmmac_release - close entry point of the driver * @dev : device pointer. @@ -2816,40 +3884,49 @@ static int stmmac_release(struct net_device *dev) struct stmmac_priv *priv = netdev_priv(dev); u32 chan; - if (priv->eee_enabled) - del_timer_sync(&priv->eee_ctrl_timer); - + if (device_may_wakeup(priv->device)) + phylink_speed_down(priv->phylink, false); /* Stop and disconnect the PHY */ phylink_stop(priv->phylink); phylink_disconnect_phy(priv->phylink); - stmmac_stop_all_queues(priv); - stmmac_disable_all_queues(priv); for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) - del_timer_sync(&priv->tx_queue[chan].txtimer); + hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); + + netif_tx_disable(dev); /* Free the IRQ lines */ - free_irq(dev->irq, dev); - if (priv->wol_irq != dev->irq) - free_irq(priv->wol_irq, dev); - if (priv->lpi_irq > 0) - free_irq(priv->lpi_irq, dev); + stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); + + if (priv->eee_enabled) { + priv->tx_path_in_lpi_mode = false; + del_timer_sync(&priv->eee_ctrl_timer); + } /* Stop TX/RX DMA and clear the descriptors */ stmmac_stop_all_dma(priv); /* Release and free the Rx/Tx resources */ - free_dma_desc_resources(priv); + free_dma_desc_resources(priv, &priv->dma_conf); /* Disable the MAC Rx/Tx */ stmmac_mac_set(priv, priv->ioaddr, false); + /* Powerdown Serdes if there is */ + if (priv->plat->serdes_powerdown) + priv->plat->serdes_powerdown(dev, priv->plat->bsp_priv); + netif_carrier_off(dev); stmmac_release_ptp(priv); + pm_runtime_put(priv->device); + + if (priv->dma_cap.fpesel) + stmmac_fpe_stop_wq(priv); + return 0; } @@ -2880,7 +3957,7 @@ static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb, return false; stmmac_set_tx_owner(priv, p); - tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE); + tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); return true; } @@ -2889,7 +3966,7 @@ static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb, * @priv: driver private structure * @des: buffer start address * @total_len: total length to fill in descriptors - * @last_segmant: condition for the last descriptor + * @last_segment: condition for the last descriptor * @queue: TX queue index * Description: * This function fills descriptor and request new descriptors according to @@ -2898,7 +3975,7 @@ static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb, static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des, int total_len, bool last_segment, u32 queue) { - struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; + struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; struct dma_desc *desc; u32 buff_size; int tmp_len; @@ -2908,7 +3985,8 @@ static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des, while (tmp_len > 0) { dma_addr_t curr_addr; - tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE); + tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, + priv->dma_conf.dma_tx_size); WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); if (tx_q->tbs & STMMAC_TBS_AVAIL) @@ -2934,6 +4012,28 @@ static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des, } } +static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue) +{ + struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; + int desc_size; + + if (likely(priv->extend_desc)) + desc_size = sizeof(struct dma_extended_desc); + else if (tx_q->tbs & STMMAC_TBS_AVAIL) + desc_size = sizeof(struct dma_edesc); + else + desc_size = sizeof(struct dma_desc); + + /* The own bit must be the latest setting done when prepare the + * descriptor and then barrier is needed to make sure that + * all is coherent before granting the DMA engine. + */ + wmb(); + + tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size); + stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue); +} + /** * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO) * @skb : the socket buffer @@ -2965,10 +4065,10 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) { struct dma_desc *desc, *first, *mss_desc = NULL; struct stmmac_priv *priv = netdev_priv(dev); - int desc_size, tmp_pay_len = 0, first_tx; int nfrags = skb_shinfo(skb)->nr_frags; u32 queue = skb_get_queue_mapping(skb); unsigned int first_entry, tx_packets; + int tmp_pay_len = 0, first_tx; struct stmmac_tx_queue *tx_q; bool has_vlan, set_ic; u8 proto_hdr_len, hdr; @@ -2976,7 +4076,7 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) dma_addr_t des; int i; - tx_q = &priv->tx_queue[queue]; + tx_q = &priv->dma_conf.tx_queue[queue]; first_tx = tx_q->cur_tx; /* Compute header lengths */ @@ -2984,7 +4084,7 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) proto_hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr); hdr = sizeof(struct udphdr); } else { - proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + proto_hdr_len = skb_tcp_all_headers(skb); hdr = tcp_hdrlen(skb); } @@ -3015,7 +4115,8 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) stmmac_set_mss(priv, mss_desc, mss); tx_q->mss = mss; - tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE); + tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, + priv->dma_conf.dma_tx_size); WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); } @@ -3049,6 +4150,8 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) tx_q->tx_skbuff_dma[first_entry].buf = des; tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb); + tx_q->tx_skbuff_dma[first_entry].map_as_page = false; + tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB; if (priv->dma_cap.addr64 <= 32) { first->des0 = cpu_to_le32(des); @@ -3084,12 +4187,14 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag); tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true; + tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; } tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true; /* Only the last descriptor gets to point to the skb. */ tx_q->tx_skbuff[tx_q->cur_tx] = skb; + tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; /* Manage tx mitigation */ tx_packets = (tx_q->cur_tx + 1) - first_tx; @@ -3097,11 +4202,12 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) set_ic = true; - else if (!priv->tx_coal_frames) + else if (!priv->tx_coal_frames[queue]) set_ic = false; - else if (tx_packets > priv->tx_coal_frames) + else if (tx_packets > priv->tx_coal_frames[queue]) set_ic = true; - else if ((tx_q->tx_count_frames % priv->tx_coal_frames) < tx_packets) + else if ((tx_q->tx_count_frames % + priv->tx_coal_frames[queue]) < tx_packets) set_ic = true; else set_ic = false; @@ -3122,7 +4228,7 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) * ndo_start_xmit will fill this descriptor the next time it's * called and stmmac_tx_clean may clean up to this descriptor. */ - tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE); + tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", @@ -3164,12 +4270,6 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) stmmac_set_tx_owner(priv, mss_desc); } - /* The own bit must be the latest setting done when prepare the - * descriptor and then barrier is needed to make sure that - * all is coherent before granting the DMA engine. - */ - wmb(); - if (netif_msg_pktdata(priv)) { pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n", __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, @@ -3180,13 +4280,7 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); - if (tx_q->tbs & STMMAC_TBS_AVAIL) - desc_size = sizeof(struct dma_edesc); - else - desc_size = sizeof(struct dma_desc); - - tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size); - stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue); + stmmac_flush_tx_descriptors(priv, queue); stmmac_tx_timer_arm(priv, queue); return NETDEV_TX_OK; @@ -3216,16 +4310,16 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) int nfrags = skb_shinfo(skb)->nr_frags; int gso = skb_shinfo(skb)->gso_type; struct dma_edesc *tbs_desc = NULL; - int entry, desc_size, first_tx; struct dma_desc *desc, *first; struct stmmac_tx_queue *tx_q; bool has_vlan, set_ic; + int entry, first_tx; dma_addr_t des; - tx_q = &priv->tx_queue[queue]; + tx_q = &priv->dma_conf.tx_queue[queue]; first_tx = tx_q->cur_tx; - if (priv->tx_path_in_lpi_mode) + if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en) stmmac_disable_eee_mode(priv); /* Manage oversized TCP frames for GMAC4 device */ @@ -3285,7 +4379,7 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) int len = skb_frag_size(frag); bool last_segment = (i == (nfrags - 1)); - entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE); + entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); WARN_ON(tx_q->tx_skbuff[entry]); if (likely(priv->extend_desc)) @@ -3307,6 +4401,7 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) tx_q->tx_skbuff_dma[entry].map_as_page = true; tx_q->tx_skbuff_dma[entry].len = len; tx_q->tx_skbuff_dma[entry].last_segment = last_segment; + tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; /* Prepare the descriptor and set the own bit too */ stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion, @@ -3315,6 +4410,7 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) /* Only the last descriptor gets to point to the skb. */ tx_q->tx_skbuff[entry] = skb; + tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; /* According to the coalesce parameter the IC bit for the latest * segment is reset and the timer re-started to clean the tx status. @@ -3326,11 +4422,12 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) set_ic = true; - else if (!priv->tx_coal_frames) + else if (!priv->tx_coal_frames[queue]) set_ic = false; - else if (tx_packets > priv->tx_coal_frames) + else if (tx_packets > priv->tx_coal_frames[queue]) set_ic = true; - else if ((tx_q->tx_count_frames % priv->tx_coal_frames) < tx_packets) + else if ((tx_q->tx_count_frames % + priv->tx_coal_frames[queue]) < tx_packets) set_ic = true; else set_ic = false; @@ -3353,7 +4450,7 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) * ndo_start_xmit will fill this descriptor the next time it's * called and stmmac_tx_clean may clean up to this descriptor. */ - entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE); + entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); tx_q->cur_tx = entry; if (netif_msg_pktdata(priv)) { @@ -3392,6 +4489,8 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) goto dma_map_err; tx_q->tx_skbuff_dma[first_entry].buf = des; + tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB; + tx_q->tx_skbuff_dma[first_entry].map_as_page = false; stmmac_set_desc_addr(priv, first, des); @@ -3420,25 +4519,11 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) stmmac_set_tx_owner(priv, first); - /* The own bit must be the latest setting done when prepare the - * descriptor and then barrier is needed to make sure that - * all is coherent before granting the DMA engine. - */ - wmb(); - netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); stmmac_enable_dma_transmission(priv, priv->ioaddr); - if (likely(priv->extend_desc)) - desc_size = sizeof(struct dma_extended_desc); - else if (tx_q->tbs & STMMAC_TBS_AVAIL) - desc_size = sizeof(struct dma_edesc); - else - desc_size = sizeof(struct dma_desc); - - tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size); - stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue); + stmmac_flush_tx_descriptors(priv, queue); stmmac_tx_timer_arm(priv, queue); return NETDEV_TX_OK; @@ -3471,15 +4556,6 @@ static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) } } - -static inline int stmmac_rx_threshold_count(struct stmmac_rx_queue *rx_q) -{ - if (rx_q->rx_zeroc_thresh < STMMAC_RX_THRESH) - return 0; - - return 1; -} - /** * stmmac_rx_refill - refill used skb preallocated buffers * @priv: driver private structure @@ -3489,11 +4565,13 @@ static inline int stmmac_rx_threshold_count(struct stmmac_rx_queue *rx_q) */ static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) { - struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; - int len, dirty = stmmac_rx_dirty(priv, queue); + struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; + int dirty = stmmac_rx_dirty(priv, queue); unsigned int entry = rx_q->dirty_rx; + gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); - len = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE) * PAGE_SIZE; + if (priv->dma_cap.addr64 <= 32) + gfp |= GFP_DMA32; while (dirty-- > 0) { struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; @@ -3506,40 +4584,34 @@ static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) p = rx_q->dma_rx + entry; if (!buf->page) { - buf->page = page_pool_dev_alloc_pages(rx_q->page_pool); + buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); if (!buf->page) break; } if (priv->sph && !buf->sec_page) { - buf->sec_page = page_pool_dev_alloc_pages(rx_q->page_pool); + buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); if (!buf->sec_page) break; buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); - - dma_sync_single_for_device(priv->device, buf->sec_addr, - len, DMA_FROM_DEVICE); } - buf->addr = page_pool_get_dma_addr(buf->page); - - /* Sync whole allocation to device. This will invalidate old - * data. - */ - dma_sync_single_for_device(priv->device, buf->addr, len, - DMA_FROM_DEVICE); + buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; stmmac_set_desc_addr(priv, p, buf->addr); - stmmac_set_desc_sec_addr(priv, p, buf->sec_addr); + if (priv->sph) + stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); + else + stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); stmmac_refill_desc3(priv, rx_q, p); rx_q->rx_count_frames++; - rx_q->rx_count_frames += priv->rx_coal_frames; - if (rx_q->rx_count_frames > priv->rx_coal_frames) + rx_q->rx_count_frames += priv->rx_coal_frames[queue]; + if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) rx_q->rx_count_frames = 0; - use_rx_wd = !priv->rx_coal_frames; + use_rx_wd = !priv->rx_coal_frames[queue]; use_rx_wd |= rx_q->rx_count_frames > 0; if (!priv->use_riwt) use_rx_wd = false; @@ -3547,7 +4619,7 @@ static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) dma_wmb(); stmmac_set_rx_owner(priv, p, use_rx_wd); - entry = STMMAC_GET_ENTRY(entry, DMA_RX_SIZE); + entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size); } rx_q->dirty_rx = entry; rx_q->rx_tail_addr = rx_q->dma_rx_phy + @@ -3559,15 +4631,15 @@ static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv, struct dma_desc *p, int status, unsigned int len) { - int ret, coe = priv->hw->rx_csum; unsigned int plen = 0, hlen = 0; + int coe = priv->hw->rx_csum; /* Not first descriptor, buffer is always zero */ if (priv->sph && len) return 0; /* First descriptor, get split header length */ - ret = stmmac_get_rx_header_len(priv, p, &hlen); + stmmac_get_rx_header_len(priv, p, &hlen); if (priv->sph && hlen) { priv->xstats.rx_split_hdr_pkt_n++; return hlen; @@ -3575,12 +4647,12 @@ static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv, /* First descriptor, not last descriptor and not split header */ if (status & rx_not_ls) - return priv->dma_buf_sz; + return priv->dma_conf.dma_buf_sz; plen = stmmac_get_rx_frame_len(priv, p, coe); /* First descriptor and last descriptor and not split header */ - return min_t(unsigned int, priv->dma_buf_sz, plen); + return min_t(unsigned int, priv->dma_conf.dma_buf_sz, plen); } static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, @@ -3596,7 +4668,7 @@ static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, /* Not last descriptor */ if (status & rx_not_ls) - return priv->dma_buf_sz; + return priv->dma_conf.dma_buf_sz; plen = stmmac_get_rx_frame_len(priv, p, coe); @@ -3604,6 +4676,476 @@ static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, return plen - len; } +static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, + struct xdp_frame *xdpf, bool dma_map) +{ + struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; + unsigned int entry = tx_q->cur_tx; + struct dma_desc *tx_desc; + dma_addr_t dma_addr; + bool set_ic; + + if (stmmac_tx_avail(priv, queue) < STMMAC_TX_THRESH(priv)) + return STMMAC_XDP_CONSUMED; + + if (likely(priv->extend_desc)) + tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); + else if (tx_q->tbs & STMMAC_TBS_AVAIL) + tx_desc = &tx_q->dma_entx[entry].basic; + else + tx_desc = tx_q->dma_tx + entry; + + if (dma_map) { + dma_addr = dma_map_single(priv->device, xdpf->data, + xdpf->len, DMA_TO_DEVICE); + if (dma_mapping_error(priv->device, dma_addr)) + return STMMAC_XDP_CONSUMED; + + tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_NDO; + } else { + struct page *page = virt_to_page(xdpf->data); + + dma_addr = page_pool_get_dma_addr(page) + sizeof(*xdpf) + + xdpf->headroom; + dma_sync_single_for_device(priv->device, dma_addr, + xdpf->len, DMA_BIDIRECTIONAL); + + tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_TX; + } + + tx_q->tx_skbuff_dma[entry].buf = dma_addr; + tx_q->tx_skbuff_dma[entry].map_as_page = false; + tx_q->tx_skbuff_dma[entry].len = xdpf->len; + tx_q->tx_skbuff_dma[entry].last_segment = true; + tx_q->tx_skbuff_dma[entry].is_jumbo = false; + + tx_q->xdpf[entry] = xdpf; + + stmmac_set_desc_addr(priv, tx_desc, dma_addr); + + stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len, + true, priv->mode, true, true, + xdpf->len); + + tx_q->tx_count_frames++; + + if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) + set_ic = true; + else + set_ic = false; + + if (set_ic) { + tx_q->tx_count_frames = 0; + stmmac_set_tx_ic(priv, tx_desc); + priv->xstats.tx_set_ic_bit++; + } + + stmmac_enable_dma_transmission(priv, priv->ioaddr); + + entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); + tx_q->cur_tx = entry; + + return STMMAC_XDP_TX; +} + +static int stmmac_xdp_get_tx_queue(struct stmmac_priv *priv, + int cpu) +{ + int index = cpu; + + if (unlikely(index < 0)) + index = 0; + + while (index >= priv->plat->tx_queues_to_use) + index -= priv->plat->tx_queues_to_use; + + return index; +} + +static int stmmac_xdp_xmit_back(struct stmmac_priv *priv, + struct xdp_buff *xdp) +{ + struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); + int cpu = smp_processor_id(); + struct netdev_queue *nq; + int queue; + int res; + + if (unlikely(!xdpf)) + return STMMAC_XDP_CONSUMED; + + queue = stmmac_xdp_get_tx_queue(priv, cpu); + nq = netdev_get_tx_queue(priv->dev, queue); + + __netif_tx_lock(nq, cpu); + /* Avoids TX time-out as we are sharing with slow path */ + txq_trans_cond_update(nq); + + res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, false); + if (res == STMMAC_XDP_TX) + stmmac_flush_tx_descriptors(priv, queue); + + __netif_tx_unlock(nq); + + return res; +} + +static int __stmmac_xdp_run_prog(struct stmmac_priv *priv, + struct bpf_prog *prog, + struct xdp_buff *xdp) +{ + u32 act; + int res; + + act = bpf_prog_run_xdp(prog, xdp); + switch (act) { + case XDP_PASS: + res = STMMAC_XDP_PASS; + break; + case XDP_TX: + res = stmmac_xdp_xmit_back(priv, xdp); + break; + case XDP_REDIRECT: + if (xdp_do_redirect(priv->dev, xdp, prog) < 0) + res = STMMAC_XDP_CONSUMED; + else + res = STMMAC_XDP_REDIRECT; + break; + default: + bpf_warn_invalid_xdp_action(priv->dev, prog, act); + fallthrough; + case XDP_ABORTED: + trace_xdp_exception(priv->dev, prog, act); + fallthrough; + case XDP_DROP: + res = STMMAC_XDP_CONSUMED; + break; + } + + return res; +} + +static struct sk_buff *stmmac_xdp_run_prog(struct stmmac_priv *priv, + struct xdp_buff *xdp) +{ + struct bpf_prog *prog; + int res; + + prog = READ_ONCE(priv->xdp_prog); + if (!prog) { + res = STMMAC_XDP_PASS; + goto out; + } + + res = __stmmac_xdp_run_prog(priv, prog, xdp); +out: + return ERR_PTR(-res); +} + +static void stmmac_finalize_xdp_rx(struct stmmac_priv *priv, + int xdp_status) +{ + int cpu = smp_processor_id(); + int queue; + + queue = stmmac_xdp_get_tx_queue(priv, cpu); + + if (xdp_status & STMMAC_XDP_TX) + stmmac_tx_timer_arm(priv, queue); + + if (xdp_status & STMMAC_XDP_REDIRECT) + xdp_do_flush(); +} + +static struct sk_buff *stmmac_construct_skb_zc(struct stmmac_channel *ch, + struct xdp_buff *xdp) +{ + unsigned int metasize = xdp->data - xdp->data_meta; + unsigned int datasize = xdp->data_end - xdp->data; + struct sk_buff *skb; + + skb = __napi_alloc_skb(&ch->rxtx_napi, + xdp->data_end - xdp->data_hard_start, + GFP_ATOMIC | __GFP_NOWARN); + if (unlikely(!skb)) + return NULL; + + skb_reserve(skb, xdp->data - xdp->data_hard_start); + memcpy(__skb_put(skb, datasize), xdp->data, datasize); + if (metasize) + skb_metadata_set(skb, metasize); + + return skb; +} + +static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue, + struct dma_desc *p, struct dma_desc *np, + struct xdp_buff *xdp) +{ + struct stmmac_channel *ch = &priv->channel[queue]; + unsigned int len = xdp->data_end - xdp->data; + enum pkt_hash_types hash_type; + int coe = priv->hw->rx_csum; + struct sk_buff *skb; + u32 hash; + + skb = stmmac_construct_skb_zc(ch, xdp); + if (!skb) { + priv->dev->stats.rx_dropped++; + return; + } + + stmmac_get_rx_hwtstamp(priv, p, np, skb); + stmmac_rx_vlan(priv->dev, skb); + skb->protocol = eth_type_trans(skb, priv->dev); + + if (unlikely(!coe)) + skb_checksum_none_assert(skb); + else + skb->ip_summed = CHECKSUM_UNNECESSARY; + + if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) + skb_set_hash(skb, hash, hash_type); + + skb_record_rx_queue(skb, queue); + napi_gro_receive(&ch->rxtx_napi, skb); + + priv->dev->stats.rx_packets++; + priv->dev->stats.rx_bytes += len; +} + +static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget) +{ + struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; + unsigned int entry = rx_q->dirty_rx; + struct dma_desc *rx_desc = NULL; + bool ret = true; + + budget = min(budget, stmmac_rx_dirty(priv, queue)); + + while (budget-- > 0 && entry != rx_q->cur_rx) { + struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; + dma_addr_t dma_addr; + bool use_rx_wd; + + if (!buf->xdp) { + buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); + if (!buf->xdp) { + ret = false; + break; + } + } + + if (priv->extend_desc) + rx_desc = (struct dma_desc *)(rx_q->dma_erx + entry); + else + rx_desc = rx_q->dma_rx + entry; + + dma_addr = xsk_buff_xdp_get_dma(buf->xdp); + stmmac_set_desc_addr(priv, rx_desc, dma_addr); + stmmac_set_desc_sec_addr(priv, rx_desc, 0, false); + stmmac_refill_desc3(priv, rx_q, rx_desc); + + rx_q->rx_count_frames++; + rx_q->rx_count_frames += priv->rx_coal_frames[queue]; + if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) + rx_q->rx_count_frames = 0; + + use_rx_wd = !priv->rx_coal_frames[queue]; + use_rx_wd |= rx_q->rx_count_frames > 0; + if (!priv->use_riwt) + use_rx_wd = false; + + dma_wmb(); + stmmac_set_rx_owner(priv, rx_desc, use_rx_wd); + + entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size); + } + + if (rx_desc) { + rx_q->dirty_rx = entry; + rx_q->rx_tail_addr = rx_q->dma_rx_phy + + (rx_q->dirty_rx * sizeof(struct dma_desc)); + stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); + } + + return ret; +} + +static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) +{ + struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; + unsigned int count = 0, error = 0, len = 0; + int dirty = stmmac_rx_dirty(priv, queue); + unsigned int next_entry = rx_q->cur_rx; + unsigned int desc_size; + struct bpf_prog *prog; + bool failure = false; + int xdp_status = 0; + int status = 0; + + if (netif_msg_rx_status(priv)) { + void *rx_head; + + netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); + if (priv->extend_desc) { + rx_head = (void *)rx_q->dma_erx; + desc_size = sizeof(struct dma_extended_desc); + } else { + rx_head = (void *)rx_q->dma_rx; + desc_size = sizeof(struct dma_desc); + } + + stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, + rx_q->dma_rx_phy, desc_size); + } + while (count < limit) { + struct stmmac_rx_buffer *buf; + unsigned int buf1_len = 0; + struct dma_desc *np, *p; + int entry; + int res; + + if (!count && rx_q->state_saved) { + error = rx_q->state.error; + len = rx_q->state.len; + } else { + rx_q->state_saved = false; + error = 0; + len = 0; + } + + if (count >= limit) + break; + +read_again: + buf1_len = 0; + entry = next_entry; + buf = &rx_q->buf_pool[entry]; + + if (dirty >= STMMAC_RX_FILL_BATCH) { + failure = failure || + !stmmac_rx_refill_zc(priv, queue, dirty); + dirty = 0; + } + + if (priv->extend_desc) + p = (struct dma_desc *)(rx_q->dma_erx + entry); + else + p = rx_q->dma_rx + entry; + + /* read the status of the incoming frame */ + status = stmmac_rx_status(priv, &priv->dev->stats, + &priv->xstats, p); + /* check if managed by the DMA otherwise go ahead */ + if (unlikely(status & dma_own)) + break; + + /* Prefetch the next RX descriptor */ + rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, + priv->dma_conf.dma_rx_size); + next_entry = rx_q->cur_rx; + + if (priv->extend_desc) + np = (struct dma_desc *)(rx_q->dma_erx + next_entry); + else + np = rx_q->dma_rx + next_entry; + + prefetch(np); + + /* Ensure a valid XSK buffer before proceed */ + if (!buf->xdp) + break; + + if (priv->extend_desc) + stmmac_rx_extended_status(priv, &priv->dev->stats, + &priv->xstats, + rx_q->dma_erx + entry); + if (unlikely(status == discard_frame)) { + xsk_buff_free(buf->xdp); + buf->xdp = NULL; + dirty++; + error = 1; + if (!priv->hwts_rx_en) + priv->dev->stats.rx_errors++; + } + + if (unlikely(error && (status & rx_not_ls))) + goto read_again; + if (unlikely(error)) { + count++; + continue; + } + + /* XSK pool expects RX frame 1:1 mapped to XSK buffer */ + if (likely(status & rx_not_ls)) { + xsk_buff_free(buf->xdp); + buf->xdp = NULL; + dirty++; + count++; + goto read_again; + } + + /* XDP ZC Frame only support primary buffers for now */ + buf1_len = stmmac_rx_buf1_len(priv, p, status, len); + len += buf1_len; + + /* ACS is disabled; strip manually. */ + if (likely(!(status & rx_not_ls))) { + buf1_len -= ETH_FCS_LEN; + len -= ETH_FCS_LEN; + } + + /* RX buffer is good and fit into a XSK pool buffer */ + buf->xdp->data_end = buf->xdp->data + buf1_len; + xsk_buff_dma_sync_for_cpu(buf->xdp, rx_q->xsk_pool); + + prog = READ_ONCE(priv->xdp_prog); + res = __stmmac_xdp_run_prog(priv, prog, buf->xdp); + + switch (res) { + case STMMAC_XDP_PASS: + stmmac_dispatch_skb_zc(priv, queue, p, np, buf->xdp); + xsk_buff_free(buf->xdp); + break; + case STMMAC_XDP_CONSUMED: + xsk_buff_free(buf->xdp); + priv->dev->stats.rx_dropped++; + break; + case STMMAC_XDP_TX: + case STMMAC_XDP_REDIRECT: + xdp_status |= res; + break; + } + + buf->xdp = NULL; + dirty++; + count++; + } + + if (status & rx_not_ls) { + rx_q->state_saved = true; + rx_q->state.error = error; + rx_q->state.len = len; + } + + stmmac_finalize_xdp_rx(priv, xdp_status); + + priv->xstats.rx_pkt_n += count; + priv->xstats.rxq_stats[queue].rx_pkt_n += count; + + if (xsk_uses_need_wakeup(rx_q->xsk_pool)) { + if (failure || stmmac_rx_dirty(priv, queue) > 0) + xsk_set_rx_need_wakeup(rx_q->xsk_pool); + else + xsk_clear_rx_need_wakeup(rx_q->xsk_pool); + + return (int)count; + } + + return failure ? limit : (int)count; +} + /** * stmmac_rx - manage the receive process * @priv: driver private structure @@ -3614,23 +5156,35 @@ static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, */ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) { - struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; + struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; struct stmmac_channel *ch = &priv->channel[queue]; unsigned int count = 0, error = 0, len = 0; int status = 0, coe = priv->hw->rx_csum; unsigned int next_entry = rx_q->cur_rx; + enum dma_data_direction dma_dir; + unsigned int desc_size; struct sk_buff *skb = NULL; + struct xdp_buff xdp; + int xdp_status = 0; + int buf_sz; + + dma_dir = page_pool_get_dma_dir(rx_q->page_pool); + buf_sz = DIV_ROUND_UP(priv->dma_conf.dma_buf_sz, PAGE_SIZE) * PAGE_SIZE; if (netif_msg_rx_status(priv)) { void *rx_head; netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); - if (priv->extend_desc) + if (priv->extend_desc) { rx_head = (void *)rx_q->dma_erx; - else + desc_size = sizeof(struct dma_extended_desc); + } else { rx_head = (void *)rx_q->dma_rx; + desc_size = sizeof(struct dma_desc); + } - stmmac_display_ring(priv, rx_head, DMA_RX_SIZE, true); + stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, + rx_q->dma_rx_phy, desc_size); } while (count < limit) { unsigned int buf1_len = 0, buf2_len = 0; @@ -3672,7 +5226,8 @@ read_again: if (unlikely(status & dma_own)) break; - rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, DMA_RX_SIZE); + rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, + priv->dma_conf.dma_rx_size); next_entry = rx_q->cur_rx; if (priv->extend_desc) @@ -3704,7 +5259,7 @@ read_again: /* Buffer is good. Go on. */ - prefetch(page_address(buf->page)); + prefetch(page_address(buf->page) + buf->page_offset); if (buf->sec_page) prefetch(page_address(buf->sec_page)); @@ -3713,25 +5268,73 @@ read_again: buf2_len = stmmac_rx_buf2_len(priv, p, status, len); len += buf2_len; - /* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3 - * Type frames (LLC/LLC-SNAP) - * - * llc_snap is never checked in GMAC >= 4, so this ACS - * feature is always disabled and packets need to be - * stripped manually. - */ - if (likely(!(status & rx_not_ls)) && - (likely(priv->synopsys_id >= DWMAC_CORE_4_00) || - unlikely(status != llc_snap))) { - if (buf2_len) + /* ACS is disabled; strip manually. */ + if (likely(!(status & rx_not_ls))) { + if (buf2_len) { buf2_len -= ETH_FCS_LEN; - else + len -= ETH_FCS_LEN; + } else if (buf1_len) { buf1_len -= ETH_FCS_LEN; + len -= ETH_FCS_LEN; + } + } - len -= ETH_FCS_LEN; + if (!skb) { + unsigned int pre_len, sync_len; + + dma_sync_single_for_cpu(priv->device, buf->addr, + buf1_len, dma_dir); + + xdp_init_buff(&xdp, buf_sz, &rx_q->xdp_rxq); + xdp_prepare_buff(&xdp, page_address(buf->page), + buf->page_offset, buf1_len, false); + + pre_len = xdp.data_end - xdp.data_hard_start - + buf->page_offset; + skb = stmmac_xdp_run_prog(priv, &xdp); + /* Due xdp_adjust_tail: DMA sync for_device + * cover max len CPU touch + */ + sync_len = xdp.data_end - xdp.data_hard_start - + buf->page_offset; + sync_len = max(sync_len, pre_len); + + /* For Not XDP_PASS verdict */ + if (IS_ERR(skb)) { + unsigned int xdp_res = -PTR_ERR(skb); + + if (xdp_res & STMMAC_XDP_CONSUMED) { + page_pool_put_page(rx_q->page_pool, + virt_to_head_page(xdp.data), + sync_len, true); + buf->page = NULL; + priv->dev->stats.rx_dropped++; + + /* Clear skb as it was set as + * status by XDP program. + */ + skb = NULL; + + if (unlikely((status & rx_not_ls))) + goto read_again; + + count++; + continue; + } else if (xdp_res & (STMMAC_XDP_TX | + STMMAC_XDP_REDIRECT)) { + xdp_status |= xdp_res; + buf->page = NULL; + skb = NULL; + count++; + continue; + } + } } if (!skb) { + /* XDP program may expand or reduce tail */ + buf1_len = xdp.data_end - xdp.data; + skb = napi_alloc_skb(&ch->rx_napi, buf1_len); if (!skb) { priv->dev->stats.rx_dropped++; @@ -3739,10 +5342,8 @@ read_again: goto drain_data; } - dma_sync_single_for_cpu(priv->device, buf->addr, - buf1_len, DMA_FROM_DEVICE); - skb_copy_to_linear_data(skb, page_address(buf->page), - buf1_len); + /* XDP program may adjust header */ + skb_copy_to_linear_data(skb, xdp.data, buf1_len); skb_put(skb, buf1_len); /* Data payload copied into SKB, page ready for recycle */ @@ -3750,10 +5351,10 @@ read_again: buf->page = NULL; } else if (buf1_len) { dma_sync_single_for_cpu(priv->device, buf->addr, - buf1_len, DMA_FROM_DEVICE); + buf1_len, dma_dir); skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, - buf->page, 0, buf1_len, - priv->dma_buf_sz); + buf->page, buf->page_offset, buf1_len, + priv->dma_conf.dma_buf_sz); /* Data payload appended into SKB */ page_pool_release_page(rx_q->page_pool, buf->page); @@ -3762,10 +5363,10 @@ read_again: if (buf2_len) { dma_sync_single_for_cpu(priv->device, buf->sec_addr, - buf2_len, DMA_FROM_DEVICE); + buf2_len, dma_dir); skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, buf->sec_page, 0, buf2_len, - priv->dma_buf_sz); + priv->dma_conf.dma_buf_sz); /* Data payload appended into SKB */ page_pool_release_page(rx_q->page_pool, buf->sec_page); @@ -3808,9 +5409,12 @@ drain_data: rx_q->state.len = len; } + stmmac_finalize_xdp_rx(priv, xdp_status); + stmmac_rx_refill(priv, queue); priv->xstats.rx_pkt_n += count; + priv->xstats.rxq_stats[queue].rx_pkt_n += count; return count; } @@ -3847,7 +5451,7 @@ static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget) priv->xstats.napi_poll++; - work_done = stmmac_tx_clean(priv, DMA_TX_SIZE, chan); + work_done = stmmac_tx_clean(priv, budget, chan); work_done = min(work_done, budget); if (work_done < budget && napi_complete_done(napi, work_done)) { @@ -3861,9 +5465,48 @@ static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget) return work_done; } +static int stmmac_napi_poll_rxtx(struct napi_struct *napi, int budget) +{ + struct stmmac_channel *ch = + container_of(napi, struct stmmac_channel, rxtx_napi); + struct stmmac_priv *priv = ch->priv_data; + int rx_done, tx_done, rxtx_done; + u32 chan = ch->index; + + priv->xstats.napi_poll++; + + tx_done = stmmac_tx_clean(priv, budget, chan); + tx_done = min(tx_done, budget); + + rx_done = stmmac_rx_zc(priv, budget, chan); + + rxtx_done = max(tx_done, rx_done); + + /* If either TX or RX work is not complete, return budget + * and keep pooling + */ + if (rxtx_done >= budget) + return budget; + + /* all work done, exit the polling mode */ + if (napi_complete_done(napi, rxtx_done)) { + unsigned long flags; + + spin_lock_irqsave(&ch->lock, flags); + /* Both RX and TX work done are compelte, + * so enable both RX & TX IRQs. + */ + stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); + spin_unlock_irqrestore(&ch->lock, flags); + } + + return min(rxtx_done, budget - 1); +} + /** * stmmac_tx_timeout * @dev : Pointer to net device structure + * @txqueue: the index of the hanging transmit queue * Description: this function is called when a packet transmission fails to * complete within a reasonable time. The driver will mark the error in the * netdev structure and arrange for the device to be reset to a sane state @@ -3907,15 +5550,18 @@ static int stmmac_change_mtu(struct net_device *dev, int new_mtu) { struct stmmac_priv *priv = netdev_priv(dev); int txfifosz = priv->plat->tx_fifo_size; + struct stmmac_dma_conf *dma_conf; + const int mtu = new_mtu; + int ret; if (txfifosz == 0) txfifosz = priv->dma_cap.tx_fifo_size; txfifosz /= priv->plat->tx_queues_to_use; - if (netif_running(dev)) { - netdev_err(priv->dev, "must be stopped to change its MTU\n"); - return -EBUSY; + if (stmmac_xdp_is_enabled(priv) && new_mtu > ETH_DATA_LEN) { + netdev_dbg(priv->dev, "Jumbo frames not supported for XDP\n"); + return -EINVAL; } new_mtu = STMMAC_ALIGN(new_mtu); @@ -3924,8 +5570,29 @@ static int stmmac_change_mtu(struct net_device *dev, int new_mtu) if ((txfifosz < new_mtu) || (new_mtu > BUF_SIZE_16KiB)) return -EINVAL; - dev->mtu = new_mtu; + if (netif_running(dev)) { + netdev_dbg(priv->dev, "restarting interface to change its MTU\n"); + /* Try to allocate the new DMA conf with the new mtu */ + dma_conf = stmmac_setup_dma_desc(priv, mtu); + if (IS_ERR(dma_conf)) { + netdev_err(priv->dev, "failed allocating new dma conf for new MTU %d\n", + mtu); + return PTR_ERR(dma_conf); + } + + stmmac_release(dev); + ret = __stmmac_open(dev, dma_conf); + kfree(dma_conf); + if (ret) { + netdev_err(priv->dev, "failed reopening the interface after MTU change\n"); + return ret; + } + + stmmac_set_rx_mode(dev); + } + + dev->mtu = mtu; netdev_update_features(dev); return 0; @@ -3965,8 +5632,6 @@ static int stmmac_set_features(struct net_device *netdev, netdev_features_t features) { struct stmmac_priv *priv = netdev_priv(netdev); - bool sph_en; - u32 chan; /* Keep the COE Type in case of csum is supporting */ if (features & NETIF_F_RXCSUM) @@ -3978,28 +5643,61 @@ static int stmmac_set_features(struct net_device *netdev, */ stmmac_rx_ipc(priv, priv->hw); - sph_en = (priv->hw->rx_csum > 0) && priv->sph; - for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++) - stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); + if (priv->sph_cap) { + bool sph_en = (priv->hw->rx_csum > 0) && priv->sph; + u32 chan; + + for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++) + stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); + } return 0; } -/** - * stmmac_interrupt - main ISR - * @irq: interrupt number. - * @dev_id: to pass the net device pointer. - * Description: this is the main driver interrupt service routine. - * It can call: - * o DMA service routine (to manage incoming frame reception and transmission - * status) - * o Core interrupts to manage: remote wake-up, management counter, LPI - * interrupts. - */ -static irqreturn_t stmmac_interrupt(int irq, void *dev_id) +static void stmmac_fpe_event_status(struct stmmac_priv *priv, int status) +{ + struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; + enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; + enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; + bool *hs_enable = &fpe_cfg->hs_enable; + + if (status == FPE_EVENT_UNKNOWN || !*hs_enable) + return; + + /* If LP has sent verify mPacket, LP is FPE capable */ + if ((status & FPE_EVENT_RVER) == FPE_EVENT_RVER) { + if (*lp_state < FPE_STATE_CAPABLE) + *lp_state = FPE_STATE_CAPABLE; + + /* If user has requested FPE enable, quickly response */ + if (*hs_enable) + stmmac_fpe_send_mpacket(priv, priv->ioaddr, + MPACKET_RESPONSE); + } + + /* If Local has sent verify mPacket, Local is FPE capable */ + if ((status & FPE_EVENT_TVER) == FPE_EVENT_TVER) { + if (*lo_state < FPE_STATE_CAPABLE) + *lo_state = FPE_STATE_CAPABLE; + } + + /* If LP has sent response mPacket, LP is entering FPE ON */ + if ((status & FPE_EVENT_RRSP) == FPE_EVENT_RRSP) + *lp_state = FPE_STATE_ENTERING_ON; + + /* If Local has sent response mPacket, Local is entering FPE ON */ + if ((status & FPE_EVENT_TRSP) == FPE_EVENT_TRSP) + *lo_state = FPE_STATE_ENTERING_ON; + + if (!test_bit(__FPE_REMOVING, &priv->fpe_task_state) && + !test_and_set_bit(__FPE_TASK_SCHED, &priv->fpe_task_state) && + priv->fpe_wq) { + queue_work(priv->fpe_wq, &priv->fpe_task); + } +} + +static void stmmac_common_interrupt(struct stmmac_priv *priv) { - struct net_device *dev = (struct net_device *)dev_id; - struct stmmac_priv *priv = netdev_priv(dev); u32 rx_cnt = priv->plat->rx_queues_to_use; u32 tx_cnt = priv->plat->tx_queues_to_use; u32 queues_count; @@ -4012,22 +5710,20 @@ static irqreturn_t stmmac_interrupt(int irq, void *dev_id) if (priv->irq_wake) pm_wakeup_event(priv->device, 0); - if (unlikely(!dev)) { - netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); - return IRQ_NONE; - } + if (priv->dma_cap.estsel) + stmmac_est_irq_status(priv, priv->ioaddr, priv->dev, + &priv->xstats, tx_cnt); - /* Check if adapter is up */ - if (test_bit(STMMAC_DOWN, &priv->state)) - return IRQ_HANDLED; - /* Check if a fatal error happened */ - if (stmmac_safety_feat_interrupt(priv)) - return IRQ_HANDLED; + if (priv->dma_cap.fpesel) { + int status = stmmac_fpe_irq_status(priv, priv->ioaddr, + priv->dev); + + stmmac_fpe_event_status(priv, status); + } /* To handle GMAC own interrupts */ if ((priv->plat->has_gmac) || xmac) { int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats); - int mtl_status; if (unlikely(status)) { /* For LPI we need to save the tx status */ @@ -4038,27 +5734,48 @@ static irqreturn_t stmmac_interrupt(int irq, void *dev_id) } for (queue = 0; queue < queues_count; queue++) { - struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; - - mtl_status = stmmac_host_mtl_irq_status(priv, priv->hw, - queue); - if (mtl_status != -EINVAL) - status |= mtl_status; - - if (status & CORE_IRQ_MTL_RX_OVERFLOW) - stmmac_set_rx_tail_ptr(priv, priv->ioaddr, - rx_q->rx_tail_addr, - queue); + status = stmmac_host_mtl_irq_status(priv, priv->hw, + queue); } /* PCS link status */ if (priv->hw->pcs) { if (priv->xstats.pcs_link) - netif_carrier_on(dev); + netif_carrier_on(priv->dev); else - netif_carrier_off(dev); + netif_carrier_off(priv->dev); } + + stmmac_timestamp_interrupt(priv, priv); } +} + +/** + * stmmac_interrupt - main ISR + * @irq: interrupt number. + * @dev_id: to pass the net device pointer. + * Description: this is the main driver interrupt service routine. + * It can call: + * o DMA service routine (to manage incoming frame reception and transmission + * status) + * o Core interrupts to manage: remote wake-up, management counter, LPI + * interrupts. + */ +static irqreturn_t stmmac_interrupt(int irq, void *dev_id) +{ + struct net_device *dev = (struct net_device *)dev_id; + struct stmmac_priv *priv = netdev_priv(dev); + + /* Check if adapter is up */ + if (test_bit(STMMAC_DOWN, &priv->state)) + return IRQ_HANDLED; + + /* Check if a fatal error happened */ + if (stmmac_safety_feat_interrupt(priv)) + return IRQ_HANDLED; + + /* To handle Common interrupts */ + stmmac_common_interrupt(priv); /* To handle DMA interrupts */ stmmac_dma_interrupt(priv); @@ -4066,15 +5783,126 @@ static irqreturn_t stmmac_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } +static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id) +{ + struct net_device *dev = (struct net_device *)dev_id; + struct stmmac_priv *priv = netdev_priv(dev); + + if (unlikely(!dev)) { + netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); + return IRQ_NONE; + } + + /* Check if adapter is up */ + if (test_bit(STMMAC_DOWN, &priv->state)) + return IRQ_HANDLED; + + /* To handle Common interrupts */ + stmmac_common_interrupt(priv); + + return IRQ_HANDLED; +} + +static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id) +{ + struct net_device *dev = (struct net_device *)dev_id; + struct stmmac_priv *priv = netdev_priv(dev); + + if (unlikely(!dev)) { + netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); + return IRQ_NONE; + } + + /* Check if adapter is up */ + if (test_bit(STMMAC_DOWN, &priv->state)) + return IRQ_HANDLED; + + /* Check if a fatal error happened */ + stmmac_safety_feat_interrupt(priv); + + return IRQ_HANDLED; +} + +static irqreturn_t stmmac_msi_intr_tx(int irq, void *data) +{ + struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)data; + struct stmmac_dma_conf *dma_conf; + int chan = tx_q->queue_index; + struct stmmac_priv *priv; + int status; + + dma_conf = container_of(tx_q, struct stmmac_dma_conf, tx_queue[chan]); + priv = container_of(dma_conf, struct stmmac_priv, dma_conf); + + if (unlikely(!data)) { + netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); + return IRQ_NONE; + } + + /* Check if adapter is up */ + if (test_bit(STMMAC_DOWN, &priv->state)) + return IRQ_HANDLED; + + status = stmmac_napi_check(priv, chan, DMA_DIR_TX); + + if (unlikely(status & tx_hard_error_bump_tc)) { + /* Try to bump up the dma threshold on this failure */ + stmmac_bump_dma_threshold(priv, chan); + } else if (unlikely(status == tx_hard_error)) { + stmmac_tx_err(priv, chan); + } + + return IRQ_HANDLED; +} + +static irqreturn_t stmmac_msi_intr_rx(int irq, void *data) +{ + struct stmmac_rx_queue *rx_q = (struct stmmac_rx_queue *)data; + struct stmmac_dma_conf *dma_conf; + int chan = rx_q->queue_index; + struct stmmac_priv *priv; + + dma_conf = container_of(rx_q, struct stmmac_dma_conf, rx_queue[chan]); + priv = container_of(dma_conf, struct stmmac_priv, dma_conf); + + if (unlikely(!data)) { + netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); + return IRQ_NONE; + } + + /* Check if adapter is up */ + if (test_bit(STMMAC_DOWN, &priv->state)) + return IRQ_HANDLED; + + stmmac_napi_check(priv, chan, DMA_DIR_RX); + + return IRQ_HANDLED; +} + #ifdef CONFIG_NET_POLL_CONTROLLER /* Polling receive - used by NETCONSOLE and other diagnostic tools * to allow network I/O with interrupts disabled. */ static void stmmac_poll_controller(struct net_device *dev) { - disable_irq(dev->irq); - stmmac_interrupt(dev->irq, dev); - enable_irq(dev->irq); + struct stmmac_priv *priv = netdev_priv(dev); + int i; + + /* If adapter is down, do nothing */ + if (test_bit(STMMAC_DOWN, &priv->state)) + return; + + if (priv->plat->multi_msi_en) { + for (i = 0; i < priv->plat->rx_queues_to_use; i++) + stmmac_msi_intr_rx(0, &priv->dma_conf.rx_queue[i]); + + for (i = 0; i < priv->plat->tx_queues_to_use; i++) + stmmac_msi_intr_tx(0, &priv->dma_conf.tx_queue[i]); + } else { + disable_irq(dev->irq); + stmmac_interrupt(dev->irq, dev); + enable_irq(dev->irq); + } } #endif @@ -4123,7 +5951,7 @@ static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data, if (!tc_cls_can_offload_and_chain0(priv->dev, type_data)) return ret; - stmmac_disable_all_queues(priv); + __stmmac_disable_all_queues(priv); switch (type) { case TC_SETUP_CLSU32: @@ -4187,12 +6015,19 @@ static int stmmac_set_mac_address(struct net_device *ndev, void *addr) struct stmmac_priv *priv = netdev_priv(ndev); int ret = 0; + ret = pm_runtime_resume_and_get(priv->device); + if (ret < 0) + return ret; + ret = eth_mac_addr(ndev, addr); if (ret) - return ret; + goto set_mac_error; stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0); +set_mac_error: + pm_runtime_put(priv->device); + return ret; } @@ -4200,24 +6035,27 @@ static int stmmac_set_mac_address(struct net_device *ndev, void *addr) static struct dentry *stmmac_fs_dir; static void sysfs_display_ring(void *head, int size, int extend_desc, - struct seq_file *seq) + struct seq_file *seq, dma_addr_t dma_phy_addr) { int i; struct dma_extended_desc *ep = (struct dma_extended_desc *)head; struct dma_desc *p = (struct dma_desc *)head; + dma_addr_t dma_addr; for (i = 0; i < size; i++) { if (extend_desc) { - seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n", - i, (unsigned int)virt_to_phys(ep), + dma_addr = dma_phy_addr + i * sizeof(*ep); + seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n", + i, &dma_addr, le32_to_cpu(ep->basic.des0), le32_to_cpu(ep->basic.des1), le32_to_cpu(ep->basic.des2), le32_to_cpu(ep->basic.des3)); ep++; } else { - seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n", - i, (unsigned int)virt_to_phys(p), + dma_addr = dma_phy_addr + i * sizeof(*p); + seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n", + i, &dma_addr, le32_to_cpu(p->des0), le32_to_cpu(p->des1), le32_to_cpu(p->des2), le32_to_cpu(p->des3)); p++; @@ -4238,34 +6076,34 @@ static int stmmac_rings_status_show(struct seq_file *seq, void *v) return 0; for (queue = 0; queue < rx_count; queue++) { - struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; + struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; seq_printf(seq, "RX Queue %d:\n", queue); if (priv->extend_desc) { seq_printf(seq, "Extended descriptor ring:\n"); sysfs_display_ring((void *)rx_q->dma_erx, - DMA_RX_SIZE, 1, seq); + priv->dma_conf.dma_rx_size, 1, seq, rx_q->dma_rx_phy); } else { seq_printf(seq, "Descriptor ring:\n"); sysfs_display_ring((void *)rx_q->dma_rx, - DMA_RX_SIZE, 0, seq); + priv->dma_conf.dma_rx_size, 0, seq, rx_q->dma_rx_phy); } } for (queue = 0; queue < tx_count; queue++) { - struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; + struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; seq_printf(seq, "TX Queue %d:\n", queue); if (priv->extend_desc) { seq_printf(seq, "Extended descriptor ring:\n"); sysfs_display_ring((void *)tx_q->dma_etx, - DMA_TX_SIZE, 1, seq); + priv->dma_conf.dma_tx_size, 1, seq, tx_q->dma_tx_phy); } else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) { seq_printf(seq, "Descriptor ring:\n"); sysfs_display_ring((void *)tx_q->dma_tx, - DMA_TX_SIZE, 0, seq); + priv->dma_conf.dma_tx_size, 0, seq, tx_q->dma_tx_phy); } } @@ -4494,19 +6332,388 @@ static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid return ret; } - return ret; + if (priv->hw->num_vlan) { + ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); + if (ret) + return ret; + } + + return 0; } static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid) { struct stmmac_priv *priv = netdev_priv(ndev); bool is_double = false; + int ret; + + ret = pm_runtime_resume_and_get(priv->device); + if (ret < 0) + return ret; if (be16_to_cpu(proto) == ETH_P_8021AD) is_double = true; clear_bit(vid, priv->active_vlans); - return stmmac_vlan_update(priv, is_double); + + if (priv->hw->num_vlan) { + ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); + if (ret) + goto del_vlan_error; + } + + ret = stmmac_vlan_update(priv, is_double); + +del_vlan_error: + pm_runtime_put(priv->device); + + return ret; +} + +static int stmmac_bpf(struct net_device *dev, struct netdev_bpf *bpf) +{ + struct stmmac_priv *priv = netdev_priv(dev); + + switch (bpf->command) { + case XDP_SETUP_PROG: + return stmmac_xdp_set_prog(priv, bpf->prog, bpf->extack); + case XDP_SETUP_XSK_POOL: + return stmmac_xdp_setup_pool(priv, bpf->xsk.pool, + bpf->xsk.queue_id); + default: + return -EOPNOTSUPP; + } +} + +static int stmmac_xdp_xmit(struct net_device *dev, int num_frames, + struct xdp_frame **frames, u32 flags) +{ + struct stmmac_priv *priv = netdev_priv(dev); + int cpu = smp_processor_id(); + struct netdev_queue *nq; + int i, nxmit = 0; + int queue; + + if (unlikely(test_bit(STMMAC_DOWN, &priv->state))) + return -ENETDOWN; + + if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) + return -EINVAL; + + queue = stmmac_xdp_get_tx_queue(priv, cpu); + nq = netdev_get_tx_queue(priv->dev, queue); + + __netif_tx_lock(nq, cpu); + /* Avoids TX time-out as we are sharing with slow path */ + txq_trans_cond_update(nq); + + for (i = 0; i < num_frames; i++) { + int res; + + res = stmmac_xdp_xmit_xdpf(priv, queue, frames[i], true); + if (res == STMMAC_XDP_CONSUMED) + break; + + nxmit++; + } + + if (flags & XDP_XMIT_FLUSH) { + stmmac_flush_tx_descriptors(priv, queue); + stmmac_tx_timer_arm(priv, queue); + } + + __netif_tx_unlock(nq); + + return nxmit; +} + +void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue) +{ + struct stmmac_channel *ch = &priv->channel[queue]; + unsigned long flags; + + spin_lock_irqsave(&ch->lock, flags); + stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 1, 0); + spin_unlock_irqrestore(&ch->lock, flags); + + stmmac_stop_rx_dma(priv, queue); + __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); +} + +void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue) +{ + struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; + struct stmmac_channel *ch = &priv->channel[queue]; + unsigned long flags; + u32 buf_size; + int ret; + + ret = __alloc_dma_rx_desc_resources(priv, &priv->dma_conf, queue); + if (ret) { + netdev_err(priv->dev, "Failed to alloc RX desc.\n"); + return; + } + + ret = __init_dma_rx_desc_rings(priv, &priv->dma_conf, queue, GFP_KERNEL); + if (ret) { + __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); + netdev_err(priv->dev, "Failed to init RX desc.\n"); + return; + } + + stmmac_reset_rx_queue(priv, queue); + stmmac_clear_rx_descriptors(priv, &priv->dma_conf, queue); + + stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, + rx_q->dma_rx_phy, rx_q->queue_index); + + rx_q->rx_tail_addr = rx_q->dma_rx_phy + (rx_q->buf_alloc_num * + sizeof(struct dma_desc)); + stmmac_set_rx_tail_ptr(priv, priv->ioaddr, + rx_q->rx_tail_addr, rx_q->queue_index); + + if (rx_q->xsk_pool && rx_q->buf_alloc_num) { + buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); + stmmac_set_dma_bfsize(priv, priv->ioaddr, + buf_size, + rx_q->queue_index); + } else { + stmmac_set_dma_bfsize(priv, priv->ioaddr, + priv->dma_conf.dma_buf_sz, + rx_q->queue_index); + } + + stmmac_start_rx_dma(priv, queue); + + spin_lock_irqsave(&ch->lock, flags); + stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 1, 0); + spin_unlock_irqrestore(&ch->lock, flags); +} + +void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue) +{ + struct stmmac_channel *ch = &priv->channel[queue]; + unsigned long flags; + + spin_lock_irqsave(&ch->lock, flags); + stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 0, 1); + spin_unlock_irqrestore(&ch->lock, flags); + + stmmac_stop_tx_dma(priv, queue); + __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); +} + +void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue) +{ + struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; + struct stmmac_channel *ch = &priv->channel[queue]; + unsigned long flags; + int ret; + + ret = __alloc_dma_tx_desc_resources(priv, &priv->dma_conf, queue); + if (ret) { + netdev_err(priv->dev, "Failed to alloc TX desc.\n"); + return; + } + + ret = __init_dma_tx_desc_rings(priv, &priv->dma_conf, queue); + if (ret) { + __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); + netdev_err(priv->dev, "Failed to init TX desc.\n"); + return; + } + + stmmac_reset_tx_queue(priv, queue); + stmmac_clear_tx_descriptors(priv, &priv->dma_conf, queue); + + stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, + tx_q->dma_tx_phy, tx_q->queue_index); + + if (tx_q->tbs & STMMAC_TBS_AVAIL) + stmmac_enable_tbs(priv, priv->ioaddr, 1, tx_q->queue_index); + + tx_q->tx_tail_addr = tx_q->dma_tx_phy; + stmmac_set_tx_tail_ptr(priv, priv->ioaddr, + tx_q->tx_tail_addr, tx_q->queue_index); + + stmmac_start_tx_dma(priv, queue); + + spin_lock_irqsave(&ch->lock, flags); + stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 0, 1); + spin_unlock_irqrestore(&ch->lock, flags); +} + +void stmmac_xdp_release(struct net_device *dev) +{ + struct stmmac_priv *priv = netdev_priv(dev); + u32 chan; + + /* Disable NAPI process */ + stmmac_disable_all_queues(priv); + + for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) + hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); + + /* Free the IRQ lines */ + stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); + + /* Stop TX/RX DMA channels */ + stmmac_stop_all_dma(priv); + + /* Release and free the Rx/Tx resources */ + free_dma_desc_resources(priv, &priv->dma_conf); + + /* Disable the MAC Rx/Tx */ + stmmac_mac_set(priv, priv->ioaddr, false); + + /* set trans_start so we don't get spurious + * watchdogs during reset + */ + netif_trans_update(dev); + netif_carrier_off(dev); +} + +int stmmac_xdp_open(struct net_device *dev) +{ + struct stmmac_priv *priv = netdev_priv(dev); + u32 rx_cnt = priv->plat->rx_queues_to_use; + u32 tx_cnt = priv->plat->tx_queues_to_use; + u32 dma_csr_ch = max(rx_cnt, tx_cnt); + struct stmmac_rx_queue *rx_q; + struct stmmac_tx_queue *tx_q; + u32 buf_size; + bool sph_en; + u32 chan; + int ret; + + ret = alloc_dma_desc_resources(priv, &priv->dma_conf); + if (ret < 0) { + netdev_err(dev, "%s: DMA descriptors allocation failed\n", + __func__); + goto dma_desc_error; + } + + ret = init_dma_desc_rings(dev, &priv->dma_conf, GFP_KERNEL); + if (ret < 0) { + netdev_err(dev, "%s: DMA descriptors initialization failed\n", + __func__); + goto init_error; + } + + /* DMA CSR Channel configuration */ + for (chan = 0; chan < dma_csr_ch; chan++) { + stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); + stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); + } + + /* Adjust Split header */ + sph_en = (priv->hw->rx_csum > 0) && priv->sph; + + /* DMA RX Channel Configuration */ + for (chan = 0; chan < rx_cnt; chan++) { + rx_q = &priv->dma_conf.rx_queue[chan]; + + stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, + rx_q->dma_rx_phy, chan); + + rx_q->rx_tail_addr = rx_q->dma_rx_phy + + (rx_q->buf_alloc_num * + sizeof(struct dma_desc)); + stmmac_set_rx_tail_ptr(priv, priv->ioaddr, + rx_q->rx_tail_addr, chan); + + if (rx_q->xsk_pool && rx_q->buf_alloc_num) { + buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); + stmmac_set_dma_bfsize(priv, priv->ioaddr, + buf_size, + rx_q->queue_index); + } else { + stmmac_set_dma_bfsize(priv, priv->ioaddr, + priv->dma_conf.dma_buf_sz, + rx_q->queue_index); + } + + stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); + } + + /* DMA TX Channel Configuration */ + for (chan = 0; chan < tx_cnt; chan++) { + tx_q = &priv->dma_conf.tx_queue[chan]; + + stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, + tx_q->dma_tx_phy, chan); + + tx_q->tx_tail_addr = tx_q->dma_tx_phy; + stmmac_set_tx_tail_ptr(priv, priv->ioaddr, + tx_q->tx_tail_addr, chan); + + hrtimer_init(&tx_q->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + tx_q->txtimer.function = stmmac_tx_timer; + } + + /* Enable the MAC Rx/Tx */ + stmmac_mac_set(priv, priv->ioaddr, true); + + /* Start Rx & Tx DMA Channels */ + stmmac_start_all_dma(priv); + + ret = stmmac_request_irq(dev); + if (ret) + goto irq_error; + + /* Enable NAPI process*/ + stmmac_enable_all_queues(priv); + netif_carrier_on(dev); + netif_tx_start_all_queues(dev); + stmmac_enable_all_dma_irq(priv); + + return 0; + +irq_error: + for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) + hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); + + stmmac_hw_teardown(dev); +init_error: + free_dma_desc_resources(priv, &priv->dma_conf); +dma_desc_error: + return ret; +} + +int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags) +{ + struct stmmac_priv *priv = netdev_priv(dev); + struct stmmac_rx_queue *rx_q; + struct stmmac_tx_queue *tx_q; + struct stmmac_channel *ch; + + if (test_bit(STMMAC_DOWN, &priv->state) || + !netif_carrier_ok(priv->dev)) + return -ENETDOWN; + + if (!stmmac_xdp_is_enabled(priv)) + return -EINVAL; + + if (queue >= priv->plat->rx_queues_to_use || + queue >= priv->plat->tx_queues_to_use) + return -EINVAL; + + rx_q = &priv->dma_conf.rx_queue[queue]; + tx_q = &priv->dma_conf.tx_queue[queue]; + ch = &priv->channel[queue]; + + if (!rx_q->xsk_pool && !tx_q->xsk_pool) + return -EINVAL; + + if (!napi_if_scheduled_mark_missed(&ch->rxtx_napi)) { + /* EQoS does not have per-DMA channel SW interrupt, + * so we schedule RX Napi straight-away. + */ + if (likely(napi_schedule_prep(&ch->rxtx_napi))) + __napi_schedule(&ch->rxtx_napi); + } + + return 0; } static const struct net_device_ops stmmac_netdev_ops = { @@ -4518,7 +6725,7 @@ static const struct net_device_ops stmmac_netdev_ops = { .ndo_set_features = stmmac_set_features, .ndo_set_rx_mode = stmmac_set_rx_mode, .ndo_tx_timeout = stmmac_tx_timeout, - .ndo_do_ioctl = stmmac_ioctl, + .ndo_eth_ioctl = stmmac_ioctl, .ndo_setup_tc = stmmac_setup_tc, .ndo_select_queue = stmmac_select_queue, #ifdef CONFIG_NET_POLL_CONTROLLER @@ -4527,6 +6734,9 @@ static const struct net_device_ops stmmac_netdev_ops = { .ndo_set_mac_address = stmmac_set_mac_address, .ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid, .ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid, + .ndo_bpf = stmmac_bpf, + .ndo_xdp_xmit = stmmac_xdp_xmit, + .ndo_xsk_wakeup = stmmac_xsk_wakeup, }; static void stmmac_reset_subtask(struct stmmac_priv *priv) @@ -4593,7 +6803,8 @@ static int stmmac_hw_init(struct stmmac_priv *priv) * register (if supported). */ priv->plat->enh_desc = priv->dma_cap.enh_desc; - priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up; + priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up && + !priv->plat->use_phy_wol; priv->hw->pmt = priv->plat->pmt; if (priv->dma_cap.hash_tb_sz) { priv->hw->multicast_filter_bins = @@ -4637,6 +6848,9 @@ static int stmmac_hw_init(struct stmmac_priv *priv) if (priv->dma_cap.tsoen) dev_info(priv->device, "TSO supported\n"); + priv->hw->vlan_fail_q_en = priv->plat->vlan_fail_q_en; + priv->hw->vlan_fail_q = priv->plat->vlan_fail_q; + /* Run HW quirks, if any */ if (priv->hwif_quirks) { ret = priv->hwif_quirks(priv); @@ -4659,6 +6873,156 @@ static int stmmac_hw_init(struct stmmac_priv *priv) return 0; } +static void stmmac_napi_add(struct net_device *dev) +{ + struct stmmac_priv *priv = netdev_priv(dev); + u32 queue, maxq; + + maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); + + for (queue = 0; queue < maxq; queue++) { + struct stmmac_channel *ch = &priv->channel[queue]; + + ch->priv_data = priv; + ch->index = queue; + spin_lock_init(&ch->lock); + + if (queue < priv->plat->rx_queues_to_use) { + netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx); + } + if (queue < priv->plat->tx_queues_to_use) { + netif_napi_add_tx(dev, &ch->tx_napi, + stmmac_napi_poll_tx); + } + if (queue < priv->plat->rx_queues_to_use && + queue < priv->plat->tx_queues_to_use) { + netif_napi_add(dev, &ch->rxtx_napi, + stmmac_napi_poll_rxtx); + } + } +} + +static void stmmac_napi_del(struct net_device *dev) +{ + struct stmmac_priv *priv = netdev_priv(dev); + u32 queue, maxq; + + maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); + + for (queue = 0; queue < maxq; queue++) { + struct stmmac_channel *ch = &priv->channel[queue]; + + if (queue < priv->plat->rx_queues_to_use) + netif_napi_del(&ch->rx_napi); + if (queue < priv->plat->tx_queues_to_use) + netif_napi_del(&ch->tx_napi); + if (queue < priv->plat->rx_queues_to_use && + queue < priv->plat->tx_queues_to_use) { + netif_napi_del(&ch->rxtx_napi); + } + } +} + +int stmmac_reinit_queues(struct net_device *dev, u32 rx_cnt, u32 tx_cnt) +{ + struct stmmac_priv *priv = netdev_priv(dev); + int ret = 0; + + if (netif_running(dev)) + stmmac_release(dev); + + stmmac_napi_del(dev); + + priv->plat->rx_queues_to_use = rx_cnt; + priv->plat->tx_queues_to_use = tx_cnt; + + stmmac_napi_add(dev); + + if (netif_running(dev)) + ret = stmmac_open(dev); + + return ret; +} + +int stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size) +{ + struct stmmac_priv *priv = netdev_priv(dev); + int ret = 0; + + if (netif_running(dev)) + stmmac_release(dev); + + priv->dma_conf.dma_rx_size = rx_size; + priv->dma_conf.dma_tx_size = tx_size; + + if (netif_running(dev)) + ret = stmmac_open(dev); + + return ret; +} + +#define SEND_VERIFY_MPAKCET_FMT "Send Verify mPacket lo_state=%d lp_state=%d\n" +static void stmmac_fpe_lp_task(struct work_struct *work) +{ + struct stmmac_priv *priv = container_of(work, struct stmmac_priv, + fpe_task); + struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; + enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; + enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; + bool *hs_enable = &fpe_cfg->hs_enable; + bool *enable = &fpe_cfg->enable; + int retries = 20; + + while (retries-- > 0) { + /* Bail out immediately if FPE handshake is OFF */ + if (*lo_state == FPE_STATE_OFF || !*hs_enable) + break; + + if (*lo_state == FPE_STATE_ENTERING_ON && + *lp_state == FPE_STATE_ENTERING_ON) { + stmmac_fpe_configure(priv, priv->ioaddr, + priv->plat->tx_queues_to_use, + priv->plat->rx_queues_to_use, + *enable); + + netdev_info(priv->dev, "configured FPE\n"); + + *lo_state = FPE_STATE_ON; + *lp_state = FPE_STATE_ON; + netdev_info(priv->dev, "!!! BOTH FPE stations ON\n"); + break; + } + + if ((*lo_state == FPE_STATE_CAPABLE || + *lo_state == FPE_STATE_ENTERING_ON) && + *lp_state != FPE_STATE_ON) { + netdev_info(priv->dev, SEND_VERIFY_MPAKCET_FMT, + *lo_state, *lp_state); + stmmac_fpe_send_mpacket(priv, priv->ioaddr, + MPACKET_VERIFY); + } + /* Sleep then retry */ + msleep(500); + } + + clear_bit(__FPE_TASK_SCHED, &priv->fpe_task_state); +} + +void stmmac_fpe_handshake(struct stmmac_priv *priv, bool enable) +{ + if (priv->plat->fpe_cfg->hs_enable != enable) { + if (enable) { + stmmac_fpe_send_mpacket(priv, priv->ioaddr, + MPACKET_VERIFY); + } else { + priv->plat->fpe_cfg->lo_fpe_state = FPE_STATE_OFF; + priv->plat->fpe_cfg->lp_fpe_state = FPE_STATE_OFF; + } + + priv->plat->fpe_cfg->hs_enable = enable; + } +} + /** * stmmac_dvr_probe * @device: device pointer @@ -4675,7 +7039,7 @@ int stmmac_dvr_probe(struct device *device, { struct net_device *ndev = NULL; struct stmmac_priv *priv; - u32 queue, rxq, maxq; + u32 rxq; int i, ret = 0; ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv), @@ -4694,19 +7058,30 @@ int stmmac_dvr_probe(struct device *device, priv->plat = plat_dat; priv->ioaddr = res->addr; priv->dev->base_addr = (unsigned long)res->addr; + priv->plat->dma_cfg->multi_msi_en = priv->plat->multi_msi_en; priv->dev->irq = res->irq; priv->wol_irq = res->wol_irq; priv->lpi_irq = res->lpi_irq; + priv->sfty_ce_irq = res->sfty_ce_irq; + priv->sfty_ue_irq = res->sfty_ue_irq; + for (i = 0; i < MTL_MAX_RX_QUEUES; i++) + priv->rx_irq[i] = res->rx_irq[i]; + for (i = 0; i < MTL_MAX_TX_QUEUES; i++) + priv->tx_irq[i] = res->tx_irq[i]; - if (!IS_ERR_OR_NULL(res->mac)) - memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN); + if (!is_zero_ether_addr(res->mac)) + eth_hw_addr_set(priv->dev, res->mac); dev_set_drvdata(device, priv->dev); /* Verify driver arguments */ stmmac_verify_args(); + priv->af_xdp_zc_qps = bitmap_zalloc(MTL_MAX_TX_QUEUES, GFP_KERNEL); + if (!priv->af_xdp_zc_qps) + return -ENOMEM; + /* Allocate workqueue */ priv->wq = create_singlethread_workqueue("stmmac_wq"); if (!priv->wq) { @@ -4716,6 +7091,9 @@ int stmmac_dvr_probe(struct device *device, INIT_WORK(&priv->service_task, stmmac_service_task); + /* Initialize Link Partner FPE workqueue */ + INIT_WORK(&priv->fpe_task, stmmac_fpe_lp_task); + /* Override with kernel parameters if supplied XXX CRS XXX * this needs to have multiple instances */ @@ -4732,16 +7110,22 @@ int stmmac_dvr_probe(struct device *device, reset_control_reset(priv->plat->stmmac_rst); } + ret = reset_control_deassert(priv->plat->stmmac_ahb_rst); + if (ret == -ENOTSUPP) + dev_err(priv->device, "unable to bring out of ahb reset: %pe\n", + ERR_PTR(ret)); + /* Init MAC and get the capabilities */ ret = stmmac_hw_init(priv); if (ret) goto error_hw_init; - stmmac_check_ether_addr(priv); + /* Only DWMAC core version 5.20 onwards supports HW descriptor prefetch. + */ + if (priv->synopsys_id < DWMAC_CORE_5_20) + priv->plat->dma_cfg->dche = false; - /* Configure real RX and TX queues */ - netif_set_real_num_rx_queues(ndev, priv->plat->rx_queues_to_use); - netif_set_real_num_tx_queues(ndev, priv->plat->tx_queues_to_use); + stmmac_check_ether_addr(priv); ndev->netdev_ops = &stmmac_netdev_ops; @@ -4761,12 +7145,21 @@ int stmmac_dvr_probe(struct device *device, dev_info(priv->device, "TSO feature enabled\n"); } - if (priv->dma_cap.sphen) { + if (priv->dma_cap.sphen && !priv->plat->sph_disable) { ndev->hw_features |= NETIF_F_GRO; - priv->sph = true; + priv->sph_cap = true; + priv->sph = priv->sph_cap; dev_info(priv->device, "SPH feature enabled\n"); } + /* The current IP register MAC_HW_Feature1[ADDR64] only define + * 32/40/64 bit width, but some SOC support others like i.MX8MP + * support 34 bits but it map to 40 bits width in MAC_HW_Feature1[ADDR64]. + * So overwrite dma_cap.addr64 according to HW real design. + */ + if (priv->plat->addr64) + priv->dma_cap.addr64 = priv->plat->addr64; + if (priv->dma_cap.addr64) { ret = dma_set_mask_and_coherent(device, DMA_BIT_MASK(priv->dma_cap.addr64)); @@ -4840,25 +7233,7 @@ int stmmac_dvr_probe(struct device *device, priv->flow_ctrl = FLOW_AUTO; /* RX/TX pause on */ /* Setup channels NAPI */ - maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); - - for (queue = 0; queue < maxq; queue++) { - struct stmmac_channel *ch = &priv->channel[queue]; - - spin_lock_init(&ch->lock); - ch->priv_data = priv; - ch->index = queue; - - if (queue < priv->plat->rx_queues_to_use) { - netif_napi_add(ndev, &ch->rx_napi, stmmac_napi_poll_rx, - NAPI_POLL_WEIGHT); - } - if (queue < priv->plat->tx_queues_to_use) { - netif_tx_napi_add(ndev, &ch->tx_napi, - stmmac_napi_poll_tx, - NAPI_POLL_WEIGHT); - } - } + stmmac_napi_add(ndev); mutex_init(&priv->lock); @@ -4875,18 +7250,32 @@ int stmmac_dvr_probe(struct device *device, stmmac_check_pcs_mode(priv); + pm_runtime_get_noresume(device); + pm_runtime_set_active(device); + if (!pm_runtime_enabled(device)) + pm_runtime_enable(device); + if (priv->hw->pcs != STMMAC_PCS_TBI && priv->hw->pcs != STMMAC_PCS_RTBI) { /* MDIO bus Registration */ ret = stmmac_mdio_register(ndev); if (ret < 0) { - dev_err(priv->device, - "%s: MDIO bus (id: %d) registration failed", - __func__, priv->plat->bus_id); + dev_err_probe(priv->device, ret, + "%s: MDIO bus (id: %d) registration failed\n", + __func__, priv->plat->bus_id); goto error_mdio_register; } } + if (priv->plat->speed_mode_2500) + priv->plat->speed_mode_2500(ndev, priv->plat->bsp_priv); + + if (priv->plat->mdio_bus_data && priv->plat->mdio_bus_data->has_xpcs) { + ret = stmmac_xpcs_setup(priv->mii); + if (ret) + goto error_xpcs_setup; + } + ret = stmmac_phy_setup(priv); if (ret) { netdev_err(ndev, "failed to setup phy (%d)\n", ret); @@ -4904,25 +7293,28 @@ int stmmac_dvr_probe(struct device *device, stmmac_init_fs(ndev); #endif + if (priv->plat->dump_debug_regs) + priv->plat->dump_debug_regs(priv->plat->bsp_priv); + + /* Let pm_runtime_put() disable the clocks. + * If CONFIG_PM is not enabled, the clocks will stay powered. + */ + pm_runtime_put(device); + return ret; error_netdev_register: phylink_destroy(priv->phylink); +error_xpcs_setup: error_phy_setup: if (priv->hw->pcs != STMMAC_PCS_TBI && priv->hw->pcs != STMMAC_PCS_RTBI) stmmac_mdio_unregister(ndev); error_mdio_register: - for (queue = 0; queue < maxq; queue++) { - struct stmmac_channel *ch = &priv->channel[queue]; - - if (queue < priv->plat->rx_queues_to_use) - netif_napi_del(&ch->rx_napi); - if (queue < priv->plat->tx_queues_to_use) - netif_napi_del(&ch->tx_napi); - } + stmmac_napi_del(ndev); error_hw_init: destroy_workqueue(priv->wq); + bitmap_free(priv->af_xdp_zc_qps); return ret; } @@ -4941,24 +7333,35 @@ int stmmac_dvr_remove(struct device *dev) netdev_info(priv->dev, "%s: removing driver", __func__); - stmmac_stop_all_dma(priv); + pm_runtime_get_sync(dev); + stmmac_stop_all_dma(priv); stmmac_mac_set(priv, priv->ioaddr, false); netif_carrier_off(ndev); unregister_netdev(ndev); + + /* Serdes power down needs to happen after VLAN filter + * is deleted that is triggered by unregister_netdev(). + */ + if (priv->plat->serdes_powerdown) + priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv); + #ifdef CONFIG_DEBUG_FS stmmac_exit_fs(ndev); #endif phylink_destroy(priv->phylink); if (priv->plat->stmmac_rst) reset_control_assert(priv->plat->stmmac_rst); - clk_disable_unprepare(priv->plat->pclk); - clk_disable_unprepare(priv->plat->stmmac_clk); + reset_control_assert(priv->plat->stmmac_ahb_rst); if (priv->hw->pcs != STMMAC_PCS_TBI && priv->hw->pcs != STMMAC_PCS_RTBI) stmmac_mdio_unregister(ndev); destroy_workqueue(priv->wq); mutex_destroy(&priv->lock); + bitmap_free(priv->af_xdp_zc_qps); + + pm_runtime_disable(dev); + pm_runtime_put_noidle(dev); return 0; } @@ -4980,50 +7383,84 @@ int stmmac_suspend(struct device *dev) if (!ndev || !netif_running(ndev)) return 0; - phylink_mac_change(priv->phylink, false); - mutex_lock(&priv->lock); netif_device_detach(ndev); - stmmac_stop_all_queues(priv); stmmac_disable_all_queues(priv); for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) - del_timer_sync(&priv->tx_queue[chan].txtimer); + hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); + + if (priv->eee_enabled) { + priv->tx_path_in_lpi_mode = false; + del_timer_sync(&priv->eee_ctrl_timer); + } /* Stop TX/RX DMA */ stmmac_stop_all_dma(priv); + if (priv->plat->serdes_powerdown) + priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv); + /* Enable Power down mode by programming the PMT regs */ - if (device_may_wakeup(priv->device)) { + if (device_may_wakeup(priv->device) && priv->plat->pmt) { stmmac_pmt(priv, priv->hw, priv->wolopts); priv->irq_wake = 1; } else { - mutex_unlock(&priv->lock); - rtnl_lock(); - phylink_stop(priv->phylink); - rtnl_unlock(); - mutex_lock(&priv->lock); - stmmac_mac_set(priv, priv->ioaddr, false); pinctrl_pm_select_sleep_state(priv->device); - /* Disable clock in case of PWM is off */ - if (priv->plat->clk_ptp_ref) - clk_disable_unprepare(priv->plat->clk_ptp_ref); - clk_disable_unprepare(priv->plat->pclk); - clk_disable_unprepare(priv->plat->stmmac_clk); } + mutex_unlock(&priv->lock); + rtnl_lock(); + if (device_may_wakeup(priv->device) && priv->plat->pmt) { + phylink_suspend(priv->phylink, true); + } else { + if (device_may_wakeup(priv->device)) + phylink_speed_down(priv->phylink, false); + phylink_suspend(priv->phylink, false); + } + rtnl_unlock(); + + if (priv->dma_cap.fpesel) { + /* Disable FPE */ + stmmac_fpe_configure(priv, priv->ioaddr, + priv->plat->tx_queues_to_use, + priv->plat->rx_queues_to_use, false); + + stmmac_fpe_handshake(priv, false); + stmmac_fpe_stop_wq(priv); + } + priv->speed = SPEED_UNKNOWN; return 0; } EXPORT_SYMBOL_GPL(stmmac_suspend); +static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue) +{ + struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; + + rx_q->cur_rx = 0; + rx_q->dirty_rx = 0; +} + +static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue) +{ + struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; + + tx_q->cur_tx = 0; + tx_q->dirty_tx = 0; + tx_q->mss = 0; + + netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); +} + /** * stmmac_reset_queues_param - reset queue parameters - * @dev: device pointer + * @priv: device pointer */ static void stmmac_reset_queues_param(struct stmmac_priv *priv) { @@ -5031,20 +7468,11 @@ static void stmmac_reset_queues_param(struct stmmac_priv *priv) u32 tx_cnt = priv->plat->tx_queues_to_use; u32 queue; - for (queue = 0; queue < rx_cnt; queue++) { - struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; - - rx_q->cur_rx = 0; - rx_q->dirty_rx = 0; - } - - for (queue = 0; queue < tx_cnt; queue++) { - struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; + for (queue = 0; queue < rx_cnt; queue++) + stmmac_reset_rx_queue(priv, queue); - tx_q->cur_tx = 0; - tx_q->dirty_tx = 0; - tx_q->mss = 0; - } + for (queue = 0; queue < tx_cnt; queue++) + stmmac_reset_tx_queue(priv, queue); } /** @@ -5057,6 +7485,7 @@ int stmmac_resume(struct device *dev) { struct net_device *ndev = dev_get_drvdata(dev); struct stmmac_priv *priv = netdev_priv(ndev); + int ret; if (!netif_running(ndev)) return 0; @@ -5067,48 +7496,57 @@ int stmmac_resume(struct device *dev) * this bit because it can generate problems while resuming * from another devices (e.g. serial console). */ - if (device_may_wakeup(priv->device)) { + if (device_may_wakeup(priv->device) && priv->plat->pmt) { mutex_lock(&priv->lock); stmmac_pmt(priv, priv->hw, 0); mutex_unlock(&priv->lock); priv->irq_wake = 0; } else { pinctrl_pm_select_default_state(priv->device); - /* enable the clk previously disabled */ - clk_prepare_enable(priv->plat->stmmac_clk); - clk_prepare_enable(priv->plat->pclk); - if (priv->plat->clk_ptp_ref) - clk_prepare_enable(priv->plat->clk_ptp_ref); /* reset the phy so that it's ready */ if (priv->mii) stmmac_mdio_reset(priv->mii); } - netif_device_attach(ndev); + if (priv->plat->serdes_powerup) { + ret = priv->plat->serdes_powerup(ndev, + priv->plat->bsp_priv); + if (ret < 0) + return ret; + } + + rtnl_lock(); + if (device_may_wakeup(priv->device) && priv->plat->pmt) { + phylink_resume(priv->phylink); + } else { + phylink_resume(priv->phylink); + if (device_may_wakeup(priv->device)) + phylink_speed_up(priv->phylink); + } + rtnl_unlock(); + + rtnl_lock(); mutex_lock(&priv->lock); stmmac_reset_queues_param(priv); - stmmac_clear_descriptors(priv); + stmmac_free_tx_skbufs(priv); + stmmac_clear_descriptors(priv, &priv->dma_conf); stmmac_hw_setup(ndev, false); stmmac_init_coalesce(priv); stmmac_set_rx_mode(ndev); - stmmac_enable_all_queues(priv); + stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw); - stmmac_start_all_queues(priv); + stmmac_enable_all_queues(priv); + stmmac_enable_all_dma_irq(priv); mutex_unlock(&priv->lock); + rtnl_unlock(); - if (!device_may_wakeup(priv->device)) { - rtnl_lock(); - phylink_start(priv->phylink); - rtnl_unlock(); - } - - phylink_mac_change(priv->phylink, true); + netif_device_attach(ndev); return 0; } @@ -5120,7 +7558,7 @@ static int __init stmmac_cmdline_opt(char *str) char *opt; if (!str || !*str) - return -EINVAL; + return 1; while ((opt = strsep(&str, ",")) != NULL) { if (!strncmp(opt, "debug:", 6)) { if (kstrtoint(opt + 6, 0, &debug)) @@ -5151,11 +7589,11 @@ static int __init stmmac_cmdline_opt(char *str) goto err; } } - return 0; + return 1; err: pr_err("%s: ERROR broken module parameter conversion", __func__); - return -EINVAL; + return 1; } __setup("stmmaceth=", stmmac_cmdline_opt); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c index cfe5d8b73142..5f177ea80725 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c @@ -15,6 +15,7 @@ #include <linux/iopoll.h> #include <linux/mii.h> #include <linux/of_mdio.h> +#include <linux/pm_runtime.h> #include <linux/phy.h> #include <linux/property.h> #include <linux/slab.h> @@ -87,21 +88,27 @@ static int stmmac_xgmac2_mdio_read(struct mii_bus *bus, int phyaddr, int phyreg) u32 tmp, addr, value = MII_XGMAC_BUSY; int ret; + ret = pm_runtime_resume_and_get(priv->device); + if (ret < 0) + return ret; + /* Wait until any existing MII operation is complete */ if (readl_poll_timeout(priv->ioaddr + mii_data, tmp, - !(tmp & MII_XGMAC_BUSY), 100, 10000)) - return -EBUSY; + !(tmp & MII_XGMAC_BUSY), 100, 10000)) { + ret = -EBUSY; + goto err_disable_clks; + } if (phyreg & MII_ADDR_C45) { phyreg &= ~MII_ADDR_C45; ret = stmmac_xgmac2_c45_format(priv, phyaddr, phyreg, &addr); if (ret) - return ret; + goto err_disable_clks; } else { ret = stmmac_xgmac2_c22_format(priv, phyaddr, phyreg, &addr); if (ret) - return ret; + goto err_disable_clks; value |= MII_XGMAC_SADDR; } @@ -112,8 +119,10 @@ static int stmmac_xgmac2_mdio_read(struct mii_bus *bus, int phyaddr, int phyreg) /* Wait until any existing MII operation is complete */ if (readl_poll_timeout(priv->ioaddr + mii_data, tmp, - !(tmp & MII_XGMAC_BUSY), 100, 10000)) - return -EBUSY; + !(tmp & MII_XGMAC_BUSY), 100, 10000)) { + ret = -EBUSY; + goto err_disable_clks; + } /* Set the MII address register to read */ writel(addr, priv->ioaddr + mii_address); @@ -121,11 +130,18 @@ static int stmmac_xgmac2_mdio_read(struct mii_bus *bus, int phyaddr, int phyreg) /* Wait until any existing MII operation is complete */ if (readl_poll_timeout(priv->ioaddr + mii_data, tmp, - !(tmp & MII_XGMAC_BUSY), 100, 10000)) - return -EBUSY; + !(tmp & MII_XGMAC_BUSY), 100, 10000)) { + ret = -EBUSY; + goto err_disable_clks; + } /* Read the data from the MII data register */ - return readl(priv->ioaddr + mii_data) & GENMASK(15, 0); + ret = (int)readl(priv->ioaddr + mii_data) & GENMASK(15, 0); + +err_disable_clks: + pm_runtime_put(priv->device); + + return ret; } static int stmmac_xgmac2_mdio_write(struct mii_bus *bus, int phyaddr, @@ -138,21 +154,27 @@ static int stmmac_xgmac2_mdio_write(struct mii_bus *bus, int phyaddr, u32 addr, tmp, value = MII_XGMAC_BUSY; int ret; + ret = pm_runtime_resume_and_get(priv->device); + if (ret < 0) + return ret; + /* Wait until any existing MII operation is complete */ if (readl_poll_timeout(priv->ioaddr + mii_data, tmp, - !(tmp & MII_XGMAC_BUSY), 100, 10000)) - return -EBUSY; + !(tmp & MII_XGMAC_BUSY), 100, 10000)) { + ret = -EBUSY; + goto err_disable_clks; + } if (phyreg & MII_ADDR_C45) { phyreg &= ~MII_ADDR_C45; ret = stmmac_xgmac2_c45_format(priv, phyaddr, phyreg, &addr); if (ret) - return ret; + goto err_disable_clks; } else { ret = stmmac_xgmac2_c22_format(priv, phyaddr, phyreg, &addr); if (ret) - return ret; + goto err_disable_clks; value |= MII_XGMAC_SADDR; } @@ -164,16 +186,23 @@ static int stmmac_xgmac2_mdio_write(struct mii_bus *bus, int phyaddr, /* Wait until any existing MII operation is complete */ if (readl_poll_timeout(priv->ioaddr + mii_data, tmp, - !(tmp & MII_XGMAC_BUSY), 100, 10000)) - return -EBUSY; + !(tmp & MII_XGMAC_BUSY), 100, 10000)) { + ret = -EBUSY; + goto err_disable_clks; + } /* Set the MII address register to write */ writel(addr, priv->ioaddr + mii_address); writel(value, priv->ioaddr + mii_data); /* Wait until any existing MII operation is complete */ - return readl_poll_timeout(priv->ioaddr + mii_data, tmp, - !(tmp & MII_XGMAC_BUSY), 100, 10000); + ret = readl_poll_timeout(priv->ioaddr + mii_data, tmp, + !(tmp & MII_XGMAC_BUSY), 100, 10000); + +err_disable_clks: + pm_runtime_put(priv->device); + + return ret; } /** @@ -196,6 +225,10 @@ static int stmmac_mdio_read(struct mii_bus *bus, int phyaddr, int phyreg) int data = 0; u32 v; + data = pm_runtime_resume_and_get(priv->device); + if (data < 0) + return data; + value |= (phyaddr << priv->hw->mii.addr_shift) & priv->hw->mii.addr_mask; value |= (phyreg << priv->hw->mii.reg_shift) & priv->hw->mii.reg_mask; @@ -216,19 +249,26 @@ static int stmmac_mdio_read(struct mii_bus *bus, int phyaddr, int phyreg) } if (readl_poll_timeout(priv->ioaddr + mii_address, v, !(v & MII_BUSY), - 100, 10000)) - return -EBUSY; + 100, 10000)) { + data = -EBUSY; + goto err_disable_clks; + } writel(data, priv->ioaddr + mii_data); writel(value, priv->ioaddr + mii_address); if (readl_poll_timeout(priv->ioaddr + mii_address, v, !(v & MII_BUSY), - 100, 10000)) - return -EBUSY; + 100, 10000)) { + data = -EBUSY; + goto err_disable_clks; + } /* Read the data from the MII data register */ data = (int)readl(priv->ioaddr + mii_data) & MII_DATA_MASK; +err_disable_clks: + pm_runtime_put(priv->device); + return data; } @@ -247,10 +287,14 @@ static int stmmac_mdio_write(struct mii_bus *bus, int phyaddr, int phyreg, struct stmmac_priv *priv = netdev_priv(ndev); unsigned int mii_address = priv->hw->mii.addr; unsigned int mii_data = priv->hw->mii.data; + int ret, data = phydata; u32 value = MII_BUSY; - int data = phydata; u32 v; + ret = pm_runtime_resume_and_get(priv->device); + if (ret < 0) + return ret; + value |= (phyaddr << priv->hw->mii.addr_shift) & priv->hw->mii.addr_mask; value |= (phyreg << priv->hw->mii.reg_shift) & priv->hw->mii.reg_mask; @@ -275,16 +319,23 @@ static int stmmac_mdio_write(struct mii_bus *bus, int phyaddr, int phyreg, /* Wait until any existing MII operation is complete */ if (readl_poll_timeout(priv->ioaddr + mii_address, v, !(v & MII_BUSY), - 100, 10000)) - return -EBUSY; + 100, 10000)) { + ret = -EBUSY; + goto err_disable_clks; + } /* Set the MII address register to write */ writel(data, priv->ioaddr + mii_data); writel(value, priv->ioaddr + mii_address); /* Wait until any existing MII operation is complete */ - return readl_poll_timeout(priv->ioaddr + mii_address, v, !(v & MII_BUSY), - 100, 10000); + ret = readl_poll_timeout(priv->ioaddr + mii_address, v, !(v & MII_BUSY), + 100, 10000); + +err_disable_clks: + pm_runtime_put(priv->device); + + return ret; } /** @@ -338,6 +389,41 @@ int stmmac_mdio_reset(struct mii_bus *bus) return 0; } +int stmmac_xpcs_setup(struct mii_bus *bus) +{ + struct net_device *ndev = bus->priv; + struct mdio_device *mdiodev; + struct stmmac_priv *priv; + struct dw_xpcs *xpcs; + int mode, addr; + + priv = netdev_priv(ndev); + mode = priv->plat->phy_interface; + + /* Try to probe the XPCS by scanning all addresses. */ + for (addr = 0; addr < PHY_MAX_ADDR; addr++) { + mdiodev = mdio_device_create(bus, addr); + if (IS_ERR(mdiodev)) + continue; + + xpcs = xpcs_create(mdiodev, mode); + if (IS_ERR_OR_NULL(xpcs)) { + mdio_device_free(mdiodev); + continue; + } + + priv->hw->xpcs = xpcs; + break; + } + + if (!priv->hw->xpcs) { + dev_warn(priv->device, "No xPCS found\n"); + return -ENODEV; + } + + return 0; +} + /** * stmmac_mdio_register * @ndev: net device structure @@ -348,9 +434,11 @@ int stmmac_mdio_register(struct net_device *ndev) int err = 0; struct mii_bus *new_bus; struct stmmac_priv *priv = netdev_priv(ndev); + struct fwnode_handle *fwnode = of_fwnode_handle(priv->plat->phylink_node); struct stmmac_mdio_bus_data *mdio_bus_data = priv->plat->mdio_bus_data; struct device_node *mdio_node = priv->plat->mdio_node; struct device *dev = ndev->dev.parent; + struct fwnode_handle *fixed_node; int addr, found, max_addr; if (!mdio_bus_data) @@ -365,6 +453,9 @@ int stmmac_mdio_register(struct net_device *ndev) new_bus->name = "stmmac"; + if (priv->plat->has_gmac4) + new_bus->probe_capabilities = MDIOBUS_C22_C45; + if (priv->plat->has_xgmac) { new_bus->read = &stmmac_xgmac2_mdio_read; new_bus->write = &stmmac_xgmac2_mdio_write; @@ -393,7 +484,7 @@ int stmmac_mdio_register(struct net_device *ndev) err = of_mdiobus_register(new_bus, mdio_node); if (err != 0) { - dev_err(dev, "Cannot register the MDIO bus\n"); + dev_err_probe(dev, err, "Cannot register the MDIO bus\n"); goto bus_register_fail; } @@ -401,6 +492,18 @@ int stmmac_mdio_register(struct net_device *ndev) if (priv->plat->has_xgmac) stmmac_xgmac2_mdio_read(new_bus, 0, MII_ADDR_C45); + /* If fixed-link is set, skip PHY scanning */ + if (!fwnode) + fwnode = dev_fwnode(priv->device); + + if (fwnode) { + fixed_node = fwnode_get_named_child_node(fwnode, "fixed-link"); + if (fixed_node) { + fwnode_handle_put(fixed_node); + goto bus_register_done; + } + } + if (priv->plat->phy_node || mdio_node) goto bus_register_done; @@ -435,9 +538,8 @@ int stmmac_mdio_register(struct net_device *ndev) if (!found && !mdio_node) { dev_warn(dev, "No PHY found\n"); - mdiobus_unregister(new_bus); - mdiobus_free(new_bus); - return -ENODEV; + err = -ENODEV; + goto no_phy_found; } bus_register_done: @@ -445,6 +547,8 @@ bus_register_done: return 0; +no_phy_found: + mdiobus_unregister(new_bus); bus_register_fail: mdiobus_free(new_bus); return err; @@ -462,6 +566,11 @@ int stmmac_mdio_unregister(struct net_device *ndev) if (!priv->mii) return 0; + if (priv->hw->xpcs) { + mdio_device_free(priv->hw->xpcs->mdiodev); + xpcs_destroy(priv->hw->xpcs); + } + mdiobus_unregister(priv->mii); priv->mii->priv = NULL; mdiobus_free(priv->mii); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c index fe2c9fa6a71c..644bb54f5f02 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c @@ -15,49 +15,10 @@ #include "stmmac.h" -/* - * This struct is used to associate PCI Function of MAC controller on a board, - * discovered via DMI, with the address of PHY connected to the MAC. The - * negative value of the address means that MAC controller is not connected - * with PHY. - */ -struct stmmac_pci_func_data { - unsigned int func; - int phy_addr; -}; - -struct stmmac_pci_dmi_data { - const struct stmmac_pci_func_data *func; - size_t nfuncs; -}; - struct stmmac_pci_info { int (*setup)(struct pci_dev *pdev, struct plat_stmmacenet_data *plat); }; -static int stmmac_pci_find_phy_addr(struct pci_dev *pdev, - const struct dmi_system_id *dmi_list) -{ - const struct stmmac_pci_func_data *func_data; - const struct stmmac_pci_dmi_data *dmi_data; - const struct dmi_system_id *dmi_id; - int func = PCI_FUNC(pdev->devfn); - size_t n; - - dmi_id = dmi_first_match(dmi_list); - if (!dmi_id) - return -ENODEV; - - dmi_data = dmi_id->driver_data; - func_data = dmi_data->func; - - for (n = 0; n < dmi_data->nfuncs; n++, func_data++) - if (func_data->func == func) - return func_data->phy_addr; - - return -ENODEV; -} - static void common_default_data(struct plat_stmmacenet_data *plat) { plat->clk_csr = 2; /* clk_csr_i = 20-35MHz & MDC = clk_csr_i/16 */ @@ -108,272 +69,6 @@ static const struct stmmac_pci_info stmmac_pci_info = { .setup = stmmac_default_data, }; -static int intel_mgbe_common_data(struct pci_dev *pdev, - struct plat_stmmacenet_data *plat) -{ - int i; - - plat->clk_csr = 5; - plat->has_gmac = 0; - plat->has_gmac4 = 1; - plat->force_sf_dma_mode = 0; - plat->tso_en = 1; - - plat->rx_sched_algorithm = MTL_RX_ALGORITHM_SP; - - for (i = 0; i < plat->rx_queues_to_use; i++) { - plat->rx_queues_cfg[i].mode_to_use = MTL_QUEUE_DCB; - plat->rx_queues_cfg[i].chan = i; - - /* Disable Priority config by default */ - plat->rx_queues_cfg[i].use_prio = false; - - /* Disable RX queues routing by default */ - plat->rx_queues_cfg[i].pkt_route = 0x0; - } - - for (i = 0; i < plat->tx_queues_to_use; i++) { - plat->tx_queues_cfg[i].mode_to_use = MTL_QUEUE_DCB; - - /* Disable Priority config by default */ - plat->tx_queues_cfg[i].use_prio = false; - } - - /* FIFO size is 4096 bytes for 1 tx/rx queue */ - plat->tx_fifo_size = plat->tx_queues_to_use * 4096; - plat->rx_fifo_size = plat->rx_queues_to_use * 4096; - - plat->tx_sched_algorithm = MTL_TX_ALGORITHM_WRR; - plat->tx_queues_cfg[0].weight = 0x09; - plat->tx_queues_cfg[1].weight = 0x0A; - plat->tx_queues_cfg[2].weight = 0x0B; - plat->tx_queues_cfg[3].weight = 0x0C; - plat->tx_queues_cfg[4].weight = 0x0D; - plat->tx_queues_cfg[5].weight = 0x0E; - plat->tx_queues_cfg[6].weight = 0x0F; - plat->tx_queues_cfg[7].weight = 0x10; - - plat->dma_cfg->pbl = 32; - plat->dma_cfg->pblx8 = true; - plat->dma_cfg->fixed_burst = 0; - plat->dma_cfg->mixed_burst = 0; - plat->dma_cfg->aal = 0; - - plat->axi = devm_kzalloc(&pdev->dev, sizeof(*plat->axi), - GFP_KERNEL); - if (!plat->axi) - return -ENOMEM; - - plat->axi->axi_lpi_en = 0; - plat->axi->axi_xit_frm = 0; - plat->axi->axi_wr_osr_lmt = 1; - plat->axi->axi_rd_osr_lmt = 1; - plat->axi->axi_blen[0] = 4; - plat->axi->axi_blen[1] = 8; - plat->axi->axi_blen[2] = 16; - - plat->ptp_max_adj = plat->clk_ptp_rate; - - /* Set system clock */ - plat->stmmac_clk = clk_register_fixed_rate(&pdev->dev, - "stmmac-clk", NULL, 0, - plat->clk_ptp_rate); - - if (IS_ERR(plat->stmmac_clk)) { - dev_warn(&pdev->dev, "Fail to register stmmac-clk\n"); - plat->stmmac_clk = NULL; - } - clk_prepare_enable(plat->stmmac_clk); - - /* Set default value for multicast hash bins */ - plat->multicast_filter_bins = HASH_TABLE_SIZE; - - /* Set default value for unicast filter entries */ - plat->unicast_filter_entries = 1; - - /* Set the maxmtu to a default of JUMBO_LEN */ - plat->maxmtu = JUMBO_LEN; - - return 0; -} - -static int ehl_common_data(struct pci_dev *pdev, - struct plat_stmmacenet_data *plat) -{ - int ret; - - plat->rx_queues_to_use = 8; - plat->tx_queues_to_use = 8; - plat->clk_ptp_rate = 200000000; - ret = intel_mgbe_common_data(pdev, plat); - if (ret) - return ret; - - return 0; -} - -static int ehl_sgmii_data(struct pci_dev *pdev, - struct plat_stmmacenet_data *plat) -{ - plat->bus_id = 1; - plat->phy_addr = 0; - plat->phy_interface = PHY_INTERFACE_MODE_SGMII; - - return ehl_common_data(pdev, plat); -} - -static struct stmmac_pci_info ehl_sgmii1g_pci_info = { - .setup = ehl_sgmii_data, -}; - -static int ehl_rgmii_data(struct pci_dev *pdev, - struct plat_stmmacenet_data *plat) -{ - plat->bus_id = 1; - plat->phy_addr = 0; - plat->phy_interface = PHY_INTERFACE_MODE_RGMII; - - return ehl_common_data(pdev, plat); -} - -static struct stmmac_pci_info ehl_rgmii1g_pci_info = { - .setup = ehl_rgmii_data, -}; - -static int tgl_common_data(struct pci_dev *pdev, - struct plat_stmmacenet_data *plat) -{ - int ret; - - plat->rx_queues_to_use = 6; - plat->tx_queues_to_use = 4; - plat->clk_ptp_rate = 200000000; - ret = intel_mgbe_common_data(pdev, plat); - if (ret) - return ret; - - return 0; -} - -static int tgl_sgmii_data(struct pci_dev *pdev, - struct plat_stmmacenet_data *plat) -{ - plat->bus_id = 1; - plat->phy_addr = 0; - plat->phy_interface = PHY_INTERFACE_MODE_SGMII; - return tgl_common_data(pdev, plat); -} - -static struct stmmac_pci_info tgl_sgmii1g_pci_info = { - .setup = tgl_sgmii_data, -}; - -static const struct stmmac_pci_func_data galileo_stmmac_func_data[] = { - { - .func = 6, - .phy_addr = 1, - }, -}; - -static const struct stmmac_pci_dmi_data galileo_stmmac_dmi_data = { - .func = galileo_stmmac_func_data, - .nfuncs = ARRAY_SIZE(galileo_stmmac_func_data), -}; - -static const struct stmmac_pci_func_data iot2040_stmmac_func_data[] = { - { - .func = 6, - .phy_addr = 1, - }, - { - .func = 7, - .phy_addr = 1, - }, -}; - -static const struct stmmac_pci_dmi_data iot2040_stmmac_dmi_data = { - .func = iot2040_stmmac_func_data, - .nfuncs = ARRAY_SIZE(iot2040_stmmac_func_data), -}; - -static const struct dmi_system_id quark_pci_dmi[] = { - { - .matches = { - DMI_EXACT_MATCH(DMI_BOARD_NAME, "Galileo"), - }, - .driver_data = (void *)&galileo_stmmac_dmi_data, - }, - { - .matches = { - DMI_EXACT_MATCH(DMI_BOARD_NAME, "GalileoGen2"), - }, - .driver_data = (void *)&galileo_stmmac_dmi_data, - }, - /* - * There are 2 types of SIMATIC IOT2000: IOT2020 and IOT2040. - * The asset tag "6ES7647-0AA00-0YA2" is only for IOT2020 which - * has only one pci network device while other asset tags are - * for IOT2040 which has two. - */ - { - .matches = { - DMI_EXACT_MATCH(DMI_BOARD_NAME, "SIMATIC IOT2000"), - DMI_EXACT_MATCH(DMI_BOARD_ASSET_TAG, - "6ES7647-0AA00-0YA2"), - }, - .driver_data = (void *)&galileo_stmmac_dmi_data, - }, - { - .matches = { - DMI_EXACT_MATCH(DMI_BOARD_NAME, "SIMATIC IOT2000"), - }, - .driver_data = (void *)&iot2040_stmmac_dmi_data, - }, - {} -}; - -static int quark_default_data(struct pci_dev *pdev, - struct plat_stmmacenet_data *plat) -{ - int ret; - - /* Set common default data first */ - common_default_data(plat); - - /* - * Refuse to load the driver and register net device if MAC controller - * does not connect to any PHY interface. - */ - ret = stmmac_pci_find_phy_addr(pdev, quark_pci_dmi); - if (ret < 0) { - /* Return error to the caller on DMI enabled boards. */ - if (dmi_get_system_info(DMI_BOARD_NAME)) - return ret; - - /* - * Galileo boards with old firmware don't support DMI. We always - * use 1 here as PHY address, so at least the first found MAC - * controller would be probed. - */ - ret = 1; - } - - plat->bus_id = pci_dev_id(pdev); - plat->phy_addr = ret; - plat->phy_interface = PHY_INTERFACE_MODE_RMII; - - plat->dma_cfg->pbl = 16; - plat->dma_cfg->pblx8 = true; - plat->dma_cfg->fixed_burst = 1; - /* AXI (TODO) */ - - return 0; -} - -static const struct stmmac_pci_info quark_pci_info = { - .setup = quark_default_data, -}; - static int snps_gmac5_default_data(struct pci_dev *pdev, struct plat_stmmacenet_data *plat) { @@ -479,8 +174,14 @@ static int stmmac_pci_probe(struct pci_dev *pdev, if (!plat->dma_cfg) return -ENOMEM; + plat->safety_feat_cfg = devm_kzalloc(&pdev->dev, + sizeof(*plat->safety_feat_cfg), + GFP_KERNEL); + if (!plat->safety_feat_cfg) + return -ENOMEM; + /* Enable pci device */ - ret = pci_enable_device(pdev); + ret = pcim_enable_device(pdev); if (ret) { dev_err(&pdev->dev, "%s: ERROR: failed to enable device\n", __func__); @@ -503,13 +204,21 @@ static int stmmac_pci_probe(struct pci_dev *pdev, if (ret) return ret; - pci_enable_msi(pdev); - memset(&res, 0, sizeof(res)); res.addr = pcim_iomap_table(pdev)[i]; res.wol_irq = pdev->irq; res.irq = pdev->irq; + plat->safety_feat_cfg->tsoee = 1; + plat->safety_feat_cfg->mrxpee = 1; + plat->safety_feat_cfg->mestee = 1; + plat->safety_feat_cfg->mrxee = 1; + plat->safety_feat_cfg->mtxee = 1; + plat->safety_feat_cfg->epsi = 1; + plat->safety_feat_cfg->edpp = 1; + plat->safety_feat_cfg->prtyen = 1; + plat->safety_feat_cfg->tmouten = 1; + return stmmac_dvr_probe(&pdev->dev, plat, &res); } @@ -522,23 +231,16 @@ static int stmmac_pci_probe(struct pci_dev *pdev, */ static void stmmac_pci_remove(struct pci_dev *pdev) { - struct net_device *ndev = dev_get_drvdata(&pdev->dev); - struct stmmac_priv *priv = netdev_priv(ndev); int i; stmmac_dvr_remove(&pdev->dev); - if (priv->plat->stmmac_clk) - clk_unregister_fixed_rate(priv->plat->stmmac_clk); - for (i = 0; i < PCI_STD_NUM_BARS; i++) { if (pci_resource_len(pdev, i) == 0) continue; pcim_iounmap_regions(pdev, BIT(i)); break; } - - pci_disable_device(pdev); } static int __maybe_unused stmmac_pci_suspend(struct device *dev) @@ -579,28 +281,15 @@ static int __maybe_unused stmmac_pci_resume(struct device *dev) static SIMPLE_DEV_PM_OPS(stmmac_pm_ops, stmmac_pci_suspend, stmmac_pci_resume); /* synthetic ID, no official vendor */ -#define PCI_VENDOR_ID_STMMAC 0x700 - -#define STMMAC_QUARK_ID 0x0937 -#define STMMAC_DEVICE_ID 0x1108 -#define STMMAC_EHL_RGMII1G_ID 0x4b30 -#define STMMAC_EHL_SGMII1G_ID 0x4b31 -#define STMMAC_TGL_SGMII1G_ID 0xa0ac -#define STMMAC_GMAC5_ID 0x7102 - -#define STMMAC_DEVICE(vendor_id, dev_id, info) { \ - PCI_VDEVICE(vendor_id, dev_id), \ - .driver_data = (kernel_ulong_t)&info \ - } +#define PCI_VENDOR_ID_STMMAC 0x0700 + +#define PCI_DEVICE_ID_STMMAC_STMMAC 0x1108 +#define PCI_DEVICE_ID_SYNOPSYS_GMAC5_ID 0x7102 static const struct pci_device_id stmmac_id_table[] = { - STMMAC_DEVICE(STMMAC, STMMAC_DEVICE_ID, stmmac_pci_info), - STMMAC_DEVICE(STMICRO, PCI_DEVICE_ID_STMICRO_MAC, stmmac_pci_info), - STMMAC_DEVICE(INTEL, STMMAC_QUARK_ID, quark_pci_info), - STMMAC_DEVICE(INTEL, STMMAC_EHL_RGMII1G_ID, ehl_rgmii1g_pci_info), - STMMAC_DEVICE(INTEL, STMMAC_EHL_SGMII1G_ID, ehl_sgmii1g_pci_info), - STMMAC_DEVICE(INTEL, STMMAC_TGL_SGMII1G_ID, tgl_sgmii1g_pci_info), - STMMAC_DEVICE(SYNOPSYS, STMMAC_GMAC5_ID, snps_gmac5_pci_info), + { PCI_DEVICE_DATA(STMMAC, STMMAC, &stmmac_pci_info) }, + { PCI_DEVICE_DATA(STMICRO, MAC, &stmmac_pci_info) }, + { PCI_DEVICE_DATA(SYNOPSYS, GMAC5_ID, &snps_gmac5_pci_info) }, {} }; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c index d10ac54bf385..50f6b4a14be4 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c @@ -9,6 +9,7 @@ *******************************************************************************/ #include <linux/platform_device.h> +#include <linux/pm_runtime.h> #include <linux/module.h> #include <linux/io.h> #include <linux/of.h> @@ -125,6 +126,7 @@ static struct stmmac_axi *stmmac_axi_setup(struct platform_device *pdev) /** * stmmac_mtl_setup - parse DT parameters for multiple queues configuration * @pdev: platform device + * @plat: enet data */ static int stmmac_mtl_setup(struct platform_device *pdev, struct plat_stmmacenet_data *plat) @@ -229,8 +231,6 @@ static int stmmac_mtl_setup(struct platform_device *pdev, plat->tx_sched_algorithm = MTL_TX_ALGORITHM_WFQ; else if (of_property_read_bool(tx_node, "snps,tx-sched-dwrr")) plat->tx_sched_algorithm = MTL_TX_ALGORITHM_DWRR; - else if (of_property_read_bool(tx_node, "snps,tx-sched-sp")) - plat->tx_sched_algorithm = MTL_TX_ALGORITHM_SP; else plat->tx_sched_algorithm = MTL_TX_ALGORITHM_SP; @@ -360,7 +360,7 @@ static int stmmac_dt_phy(struct plat_stmmacenet_data *plat, /** * stmmac_of_get_mac_mode - retrieves the interface of the MAC - * @np - device-tree node + * @np: - device-tree node * Description: * Similar to `of_get_phy_mode()`, this function will retrieve (from * the device-tree) the interface mode on the MAC side. This assumes @@ -393,29 +393,32 @@ static int stmmac_of_get_mac_mode(struct device_node *np) * set some private fields that will be used by the main at runtime. */ struct plat_stmmacenet_data * -stmmac_probe_config_dt(struct platform_device *pdev, const char **mac) +stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac) { struct device_node *np = pdev->dev.of_node; struct plat_stmmacenet_data *plat; struct stmmac_dma_cfg *dma_cfg; + int phy_mode; + void *ret; int rc; plat = devm_kzalloc(&pdev->dev, sizeof(*plat), GFP_KERNEL); if (!plat) return ERR_PTR(-ENOMEM); - *mac = of_get_mac_address(np); - if (IS_ERR(*mac)) { - if (PTR_ERR(*mac) == -EPROBE_DEFER) - return ERR_CAST(*mac); + rc = of_get_mac_address(np, mac); + if (rc) { + if (rc == -EPROBE_DEFER) + return ERR_PTR(rc); - *mac = NULL; + eth_zero_addr(mac); } - plat->phy_interface = device_get_phy_mode(&pdev->dev); - if (plat->phy_interface < 0) - return ERR_PTR(plat->phy_interface); + phy_mode = device_get_phy_mode(&pdev->dev); + if (phy_mode < 0) + return ERR_PTR(phy_mode); + plat->phy_interface = phy_mode; plat->interface = stmmac_of_get_mac_mode(np); if (plat->interface < 0) plat->interface = plat->phy_interface; @@ -428,8 +431,7 @@ stmmac_probe_config_dt(struct platform_device *pdev, const char **mac) plat->phylink_node = np; /* Get max speed of operation from device tree */ - if (of_property_read_u32(np, "max-speed", &plat->max_speed)) - plat->max_speed = -1; + of_property_read_u32(np, "max-speed", &plat->max_speed); plat->bus_id = of_alias_get_id(np, "ethernet"); if (plat->bus_id < 0) @@ -438,11 +440,12 @@ stmmac_probe_config_dt(struct platform_device *pdev, const char **mac) /* Default to phy auto-detection */ plat->phy_addr = -1; - /* Default to get clk_csr from stmmac_clk_crs_set(), + /* Default to get clk_csr from stmmac_clk_csr_set(), * or get clk_csr from device tree. */ plat->clk_csr = -1; - of_property_read_u32(np, "clk_csr", &plat->clk_csr); + if (of_property_read_u32(np, "snps,clk-csr", &plat->clk_csr)) + of_property_read_u32(np, "clk_csr", &plat->clk_csr); /* "snps,phy-addr" is not a standard property. Mark it as deprecated * and warn of its use. Remove this when phy node support is added. @@ -505,9 +508,18 @@ stmmac_probe_config_dt(struct platform_device *pdev, const char **mac) plat->pmt = 1; } + if (of_device_is_compatible(np, "snps,dwmac-3.40a")) { + plat->has_gmac = 1; + plat->enh_desc = 1; + plat->tx_coe = 1; + plat->bugged_jumbo = 1; + plat->pmt = 1; + } + if (of_device_is_compatible(np, "snps,dwmac-4.00") || of_device_is_compatible(np, "snps,dwmac-4.10a") || - of_device_is_compatible(np, "snps,dwmac-4.20a")) { + of_device_is_compatible(np, "snps,dwmac-4.20a") || + of_device_is_compatible(np, "snps,dwmac-5.10a")) { plat->has_gmac4 = 1; plat->has_gmac = 0; plat->pmt = 1; @@ -574,12 +586,10 @@ stmmac_probe_config_dt(struct platform_device *pdev, const char **mac) clk_prepare_enable(plat->stmmac_clk); } - plat->pclk = devm_clk_get(&pdev->dev, "pclk"); + plat->pclk = devm_clk_get_optional(&pdev->dev, "pclk"); if (IS_ERR(plat->pclk)) { - if (PTR_ERR(plat->pclk) == -EPROBE_DEFER) - goto error_pclk_get; - - plat->pclk = NULL; + ret = plat->pclk; + goto error_pclk_get; } clk_prepare_enable(plat->pclk); @@ -588,20 +598,24 @@ stmmac_probe_config_dt(struct platform_device *pdev, const char **mac) if (IS_ERR(plat->clk_ptp_ref)) { plat->clk_ptp_rate = clk_get_rate(plat->stmmac_clk); plat->clk_ptp_ref = NULL; - dev_warn(&pdev->dev, "PTP uses main clock\n"); + dev_info(&pdev->dev, "PTP uses main clock\n"); } else { plat->clk_ptp_rate = clk_get_rate(plat->clk_ptp_ref); dev_dbg(&pdev->dev, "PTP rate %d\n", plat->clk_ptp_rate); } - plat->stmmac_rst = devm_reset_control_get(&pdev->dev, - STMMAC_RESOURCE_NAME); + plat->stmmac_rst = devm_reset_control_get_optional(&pdev->dev, + STMMAC_RESOURCE_NAME); if (IS_ERR(plat->stmmac_rst)) { - if (PTR_ERR(plat->stmmac_rst) == -EPROBE_DEFER) - goto error_hw_init; + ret = plat->stmmac_rst; + goto error_hw_init; + } - dev_info(&pdev->dev, "no reset control found\n"); - plat->stmmac_rst = NULL; + plat->stmmac_ahb_rst = devm_reset_control_get_optional_shared( + &pdev->dev, "ahb"); + if (IS_ERR(plat->stmmac_ahb_rst)) { + ret = plat->stmmac_ahb_rst; + goto error_hw_init; } return plat; @@ -611,7 +625,7 @@ error_hw_init: error_pclk_get: clk_disable_unprepare(plat->stmmac_clk); - return ERR_PTR(-EPROBE_DEFER); + return ret; } /** @@ -624,12 +638,14 @@ error_pclk_get: void stmmac_remove_config_dt(struct platform_device *pdev, struct plat_stmmacenet_data *plat) { + clk_disable_unprepare(plat->stmmac_clk); + clk_disable_unprepare(plat->pclk); of_node_put(plat->phy_node); of_node_put(plat->mdio_node); } #else struct plat_stmmacenet_data * -stmmac_probe_config_dt(struct platform_device *pdev, const char **mac) +stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac) { return ERR_PTR(-EINVAL); } @@ -645,8 +661,6 @@ EXPORT_SYMBOL_GPL(stmmac_remove_config_dt); int stmmac_get_platform_resources(struct platform_device *pdev, struct stmmac_resources *stmmac_res) { - struct resource *res; - memset(stmmac_res, 0, sizeof(*stmmac_res)); /* Get IRQ information early to have an ability to ask for deferred @@ -663,19 +677,24 @@ int stmmac_get_platform_resources(struct platform_device *pdev, * In case the wake up interrupt is not passed from the platform * so the driver will continue to use the mac irq (ndev->irq) */ - stmmac_res->wol_irq = platform_get_irq_byname(pdev, "eth_wake_irq"); + stmmac_res->wol_irq = + platform_get_irq_byname_optional(pdev, "eth_wake_irq"); if (stmmac_res->wol_irq < 0) { if (stmmac_res->wol_irq == -EPROBE_DEFER) return -EPROBE_DEFER; + dev_info(&pdev->dev, "IRQ eth_wake_irq not found\n"); stmmac_res->wol_irq = stmmac_res->irq; } - stmmac_res->lpi_irq = platform_get_irq_byname(pdev, "eth_lpi"); - if (stmmac_res->lpi_irq == -EPROBE_DEFER) - return -EPROBE_DEFER; + stmmac_res->lpi_irq = + platform_get_irq_byname_optional(pdev, "eth_lpi"); + if (stmmac_res->lpi_irq < 0) { + if (stmmac_res->lpi_irq == -EPROBE_DEFER) + return -EPROBE_DEFER; + dev_info(&pdev->dev, "IRQ eth_lpi not found\n"); + } - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - stmmac_res->addr = devm_ioremap_resource(&pdev->dev, res); + stmmac_res->addr = devm_platform_ioremap_resource(pdev, 0); return PTR_ERR_OR_ZERO(stmmac_res->addr); } @@ -703,7 +722,6 @@ int stmmac_pltfr_remove(struct platform_device *pdev) } EXPORT_SYMBOL_GPL(stmmac_pltfr_remove); -#ifdef CONFIG_PM_SLEEP /** * stmmac_pltfr_suspend * @dev: device pointer @@ -711,7 +729,7 @@ EXPORT_SYMBOL_GPL(stmmac_pltfr_remove); * call the main suspend function and then, if required, on some platform, it * can call an exit helper. */ -static int stmmac_pltfr_suspend(struct device *dev) +static int __maybe_unused stmmac_pltfr_suspend(struct device *dev) { int ret; struct net_device *ndev = dev_get_drvdata(dev); @@ -732,7 +750,7 @@ static int stmmac_pltfr_suspend(struct device *dev) * the main resume function, on some platforms, it can call own init helper * if required. */ -static int stmmac_pltfr_resume(struct device *dev) +static int __maybe_unused stmmac_pltfr_resume(struct device *dev) { struct net_device *ndev = dev_get_drvdata(dev); struct stmmac_priv *priv = netdev_priv(ndev); @@ -743,10 +761,78 @@ static int stmmac_pltfr_resume(struct device *dev) return stmmac_resume(dev); } -#endif /* CONFIG_PM_SLEEP */ -SIMPLE_DEV_PM_OPS(stmmac_pltfr_pm_ops, stmmac_pltfr_suspend, - stmmac_pltfr_resume); +static int __maybe_unused stmmac_runtime_suspend(struct device *dev) +{ + struct net_device *ndev = dev_get_drvdata(dev); + struct stmmac_priv *priv = netdev_priv(ndev); + + stmmac_bus_clks_config(priv, false); + + return 0; +} + +static int __maybe_unused stmmac_runtime_resume(struct device *dev) +{ + struct net_device *ndev = dev_get_drvdata(dev); + struct stmmac_priv *priv = netdev_priv(ndev); + + return stmmac_bus_clks_config(priv, true); +} + +static int __maybe_unused stmmac_pltfr_noirq_suspend(struct device *dev) +{ + struct net_device *ndev = dev_get_drvdata(dev); + struct stmmac_priv *priv = netdev_priv(ndev); + int ret; + + if (!netif_running(ndev)) + return 0; + + if (!device_may_wakeup(priv->device) || !priv->plat->pmt) { + /* Disable clock in case of PWM is off */ + clk_disable_unprepare(priv->plat->clk_ptp_ref); + + ret = pm_runtime_force_suspend(dev); + if (ret) + return ret; + } + + return 0; +} + +static int __maybe_unused stmmac_pltfr_noirq_resume(struct device *dev) +{ + struct net_device *ndev = dev_get_drvdata(dev); + struct stmmac_priv *priv = netdev_priv(ndev); + int ret; + + if (!netif_running(ndev)) + return 0; + + if (!device_may_wakeup(priv->device) || !priv->plat->pmt) { + /* enable the clk previously disabled */ + ret = pm_runtime_force_resume(dev); + if (ret) + return ret; + + ret = clk_prepare_enable(priv->plat->clk_ptp_ref); + if (ret < 0) { + netdev_warn(priv->dev, + "failed to enable PTP reference clock: %pe\n", + ERR_PTR(ret)); + return ret; + } + } + + return 0; +} + +const struct dev_pm_ops stmmac_pltfr_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(stmmac_pltfr_suspend, stmmac_pltfr_resume) + SET_RUNTIME_PM_OPS(stmmac_runtime_suspend, stmmac_runtime_resume, NULL) + SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(stmmac_pltfr_noirq_suspend, stmmac_pltfr_noirq_resume) +}; EXPORT_SYMBOL_GPL(stmmac_pltfr_pm_ops); MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet platform support"); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.h b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.h index 3a4663b7b460..3fff3f59d73d 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.h +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.h @@ -12,7 +12,7 @@ #include "stmmac.h" struct plat_stmmacenet_data * -stmmac_probe_config_dt(struct platform_device *pdev, const char **mac); +stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac); void stmmac_remove_config_dt(struct platform_device *pdev, struct plat_stmmacenet_data *plat); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c index 0989e2bb6ee3..4d11980dcd64 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c @@ -9,6 +9,7 @@ *******************************************************************************/ #include "stmmac.h" #include "stmmac_ptp.h" +#include "dwmac4.h" /** * stmmac_adjust_freq @@ -38,9 +39,9 @@ static int stmmac_adjust_freq(struct ptp_clock_info *ptp, s32 ppb) diff = div_u64(adj, 1000000000ULL); addend = neg_adj ? (addend - diff) : (addend + diff); - spin_lock_irqsave(&priv->ptp_lock, flags); + write_lock_irqsave(&priv->ptp_lock, flags); stmmac_config_addend(priv, priv->ptpaddr, addend); - spin_unlock_irqrestore(&priv->ptp_lock, flags); + write_unlock_irqrestore(&priv->ptp_lock, flags); return 0; } @@ -61,7 +62,8 @@ static int stmmac_adjust_time(struct ptp_clock_info *ptp, s64 delta) u32 sec, nsec; u32 quotient, reminder; int neg_adj = 0; - bool xmac; + bool xmac, est_rst = false; + int ret; xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; @@ -74,9 +76,47 @@ static int stmmac_adjust_time(struct ptp_clock_info *ptp, s64 delta) sec = quotient; nsec = reminder; - spin_lock_irqsave(&priv->ptp_lock, flags); + /* If EST is enabled, disabled it before adjust ptp time. */ + if (priv->plat->est && priv->plat->est->enable) { + est_rst = true; + mutex_lock(&priv->plat->est->lock); + priv->plat->est->enable = false; + stmmac_est_configure(priv, priv->ioaddr, priv->plat->est, + priv->plat->clk_ptp_rate); + mutex_unlock(&priv->plat->est->lock); + } + + write_lock_irqsave(&priv->ptp_lock, flags); stmmac_adjust_systime(priv, priv->ptpaddr, sec, nsec, neg_adj, xmac); - spin_unlock_irqrestore(&priv->ptp_lock, flags); + write_unlock_irqrestore(&priv->ptp_lock, flags); + + /* Caculate new basetime and re-configured EST after PTP time adjust. */ + if (est_rst) { + struct timespec64 current_time, time; + ktime_t current_time_ns, basetime; + u64 cycle_time; + + mutex_lock(&priv->plat->est->lock); + priv->ptp_clock_ops.gettime64(&priv->ptp_clock_ops, ¤t_time); + current_time_ns = timespec64_to_ktime(current_time); + time.tv_nsec = priv->plat->est->btr_reserve[0]; + time.tv_sec = priv->plat->est->btr_reserve[1]; + basetime = timespec64_to_ktime(time); + cycle_time = (u64)priv->plat->est->ctr[1] * NSEC_PER_SEC + + priv->plat->est->ctr[0]; + time = stmmac_calc_tas_basetime(basetime, + current_time_ns, + cycle_time); + + priv->plat->est->btr[0] = (u32)time.tv_nsec; + priv->plat->est->btr[1] = (u32)time.tv_sec; + priv->plat->est->enable = true; + ret = stmmac_est_configure(priv, priv->ioaddr, priv->plat->est, + priv->plat->clk_ptp_rate); + mutex_unlock(&priv->plat->est->lock); + if (ret) + netdev_err(priv->dev, "failed to configure EST\n"); + } return 0; } @@ -97,9 +137,9 @@ static int stmmac_get_time(struct ptp_clock_info *ptp, struct timespec64 *ts) unsigned long flags; u64 ns = 0; - spin_lock_irqsave(&priv->ptp_lock, flags); + read_lock_irqsave(&priv->ptp_lock, flags); stmmac_get_systime(priv, priv->ptpaddr, &ns); - spin_unlock_irqrestore(&priv->ptp_lock, flags); + read_unlock_irqrestore(&priv->ptp_lock, flags); *ts = ns_to_timespec64(ns); @@ -122,9 +162,9 @@ static int stmmac_set_time(struct ptp_clock_info *ptp, container_of(ptp, struct stmmac_priv, ptp_clock_ops); unsigned long flags; - spin_lock_irqsave(&priv->ptp_lock, flags); + write_lock_irqsave(&priv->ptp_lock, flags); stmmac_init_systime(priv, priv->ptpaddr, ts->tv_sec, ts->tv_nsec); - spin_unlock_irqrestore(&priv->ptp_lock, flags); + write_unlock_irqrestore(&priv->ptp_lock, flags); return 0; } @@ -134,9 +174,11 @@ static int stmmac_enable(struct ptp_clock_info *ptp, { struct stmmac_priv *priv = container_of(ptp, struct stmmac_priv, ptp_clock_ops); + void __iomem *ptpaddr = priv->ptpaddr; struct stmmac_pps_cfg *cfg; int ret = -EOPNOTSUPP; unsigned long flags; + u32 acr_value; switch (rq->type) { case PTP_CLK_REQ_PEROUT: @@ -151,13 +193,35 @@ static int stmmac_enable(struct ptp_clock_info *ptp, cfg->period.tv_sec = rq->perout.period.sec; cfg->period.tv_nsec = rq->perout.period.nsec; - spin_lock_irqsave(&priv->ptp_lock, flags); + write_lock_irqsave(&priv->ptp_lock, flags); ret = stmmac_flex_pps_config(priv, priv->ioaddr, rq->perout.index, cfg, on, priv->sub_second_inc, priv->systime_flags); - spin_unlock_irqrestore(&priv->ptp_lock, flags); + write_unlock_irqrestore(&priv->ptp_lock, flags); break; + case PTP_CLK_REQ_EXTTS: + priv->plat->ext_snapshot_en = on; + mutex_lock(&priv->aux_ts_lock); + acr_value = readl(ptpaddr + PTP_ACR); + acr_value &= ~PTP_ACR_MASK; + if (on) { + /* Enable External snapshot trigger */ + acr_value |= priv->plat->ext_snapshot_num; + acr_value |= PTP_ACR_ATSFC; + netdev_dbg(priv->dev, "Auxiliary Snapshot %d enabled.\n", + priv->plat->ext_snapshot_num >> + PTP_ACR_ATSEN_SHIFT); + } else { + netdev_dbg(priv->dev, "Auxiliary Snapshot %d disabled.\n", + priv->plat->ext_snapshot_num >> + PTP_ACR_ATSEN_SHIFT); + } + writel(acr_value, ptpaddr + PTP_ACR); + mutex_unlock(&priv->aux_ts_lock); + ret = 0; + break; + default: break; } @@ -165,13 +229,43 @@ static int stmmac_enable(struct ptp_clock_info *ptp, return ret; } +/** + * stmmac_get_syncdevicetime + * @device: current device time + * @system: system counter value read synchronously with device time + * @ctx: context provided by timekeeping code + * Description: Read device and system clock simultaneously and return the + * corrected clock values in ns. + **/ +static int stmmac_get_syncdevicetime(ktime_t *device, + struct system_counterval_t *system, + void *ctx) +{ + struct stmmac_priv *priv = (struct stmmac_priv *)ctx; + + if (priv->plat->crosststamp) + return priv->plat->crosststamp(device, system, ctx); + else + return -EOPNOTSUPP; +} + +static int stmmac_getcrosststamp(struct ptp_clock_info *ptp, + struct system_device_crosststamp *xtstamp) +{ + struct stmmac_priv *priv = + container_of(ptp, struct stmmac_priv, ptp_clock_ops); + + return get_device_system_crosststamp(stmmac_get_syncdevicetime, + priv, NULL, xtstamp); +} + /* structure describing a PTP hardware clock */ static struct ptp_clock_info stmmac_ptp_clock_ops = { .owner = THIS_MODULE, .name = "stmmac ptp", .max_adj = 62500000, .n_alarm = 0, - .n_ext_ts = 0, + .n_ext_ts = 0, /* will be overwritten in stmmac_ptp_register */ .n_per_out = 0, /* will be overwritten in stmmac_ptp_register */ .n_pins = 0, .pps = 0, @@ -180,6 +274,7 @@ static struct ptp_clock_info stmmac_ptp_clock_ops = { .gettime64 = stmmac_get_time, .settime64 = stmmac_set_time, .enable = stmmac_enable, + .getcrosststamp = stmmac_getcrosststamp, }; /** @@ -201,9 +296,16 @@ void stmmac_ptp_register(struct stmmac_priv *priv) if (priv->plat->ptp_max_adj) stmmac_ptp_clock_ops.max_adj = priv->plat->ptp_max_adj; + /* Calculate the clock domain crossing (CDC) error if necessary */ + priv->plat->cdc_error_adj = 0; + if (priv->plat->has_gmac4 && priv->plat->clk_ptp_rate) + priv->plat->cdc_error_adj = (2 * NSEC_PER_SEC) / priv->plat->clk_ptp_rate; + stmmac_ptp_clock_ops.n_per_out = priv->dma_cap.pps_out_num; + stmmac_ptp_clock_ops.n_ext_ts = priv->dma_cap.aux_snapshot_n; - spin_lock_init(&priv->ptp_lock); + rwlock_init(&priv->ptp_lock); + mutex_init(&priv->aux_ts_lock); priv->ptp_clock_ops = stmmac_ptp_clock_ops; priv->ptp_clock = ptp_clock_register(&priv->ptp_clock_ops, @@ -229,4 +331,6 @@ void stmmac_ptp_unregister(struct stmmac_priv *priv) pr_debug("Removed PTP HW clock successfully on %s\n", priv->dev->name); } + + mutex_destroy(&priv->aux_ts_lock); } diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.h b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.h index 7abb1d47e7da..53172a439810 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.h +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.h @@ -23,6 +23,9 @@ #define PTP_STSUR 0x10 /* System Time – Seconds Update Reg */ #define PTP_STNSUR 0x14 /* System Time – Nanoseconds Update Reg */ #define PTP_TAR 0x18 /* Timestamp Addend Reg */ +#define PTP_ACR 0x40 /* Auxiliary Control Reg */ +#define PTP_ATNR 0x48 /* Auxiliary Timestamp - Nanoseconds Reg */ +#define PTP_ATSR 0x4c /* Auxiliary Timestamp - Seconds Reg */ #define PTP_STNSUR_ADDSUB_SHIFT 31 #define PTP_DIGITAL_ROLLOVER_MODE 0x3B9ACA00 /* 10e9-1 ns */ @@ -64,4 +67,25 @@ #define PTP_SSIR_SSINC_MASK 0xff #define GMAC4_PTP_SSIR_SSINC_SHIFT 16 +/* Auxiliary Control defines */ +#define PTP_ACR_ATSFC BIT(0) /* Auxiliary Snapshot FIFO Clear */ +#define PTP_ACR_ATSEN0 BIT(4) /* Auxiliary Snapshot 0 Enable */ +#define PTP_ACR_ATSEN1 BIT(5) /* Auxiliary Snapshot 1 Enable */ +#define PTP_ACR_ATSEN2 BIT(6) /* Auxiliary Snapshot 2 Enable */ +#define PTP_ACR_ATSEN3 BIT(7) /* Auxiliary Snapshot 3 Enable */ +#define PTP_ACR_ATSEN_SHIFT 5 /* Auxiliary Snapshot shift */ +#define PTP_ACR_MASK GENMASK(7, 4) /* Aux Snapshot Mask */ +#define PMC_ART_VALUE0 0x01 /* PMC_ART[15:0] timer value */ +#define PMC_ART_VALUE1 0x02 /* PMC_ART[31:16] timer value */ +#define PMC_ART_VALUE2 0x03 /* PMC_ART[47:32] timer value */ +#define PMC_ART_VALUE3 0x04 /* PMC_ART[63:48] timer value */ +#define GMAC4_ART_TIME_SHIFT 16 /* ART TIME 16-bits shift */ + +enum aux_snapshot { + AUX_SNAPSHOT0 = 0x10, + AUX_SNAPSHOT1 = 0x20, + AUX_SNAPSHOT2 = 0x40, + AUX_SNAPSHOT3 = 0x80, +}; + #endif /* __STMMAC_PTP_H__ */ diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c index 2aba2673d6c3..49af7e78b7f5 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c @@ -36,7 +36,7 @@ struct stmmac_packet_attrs { int vlan_id_in; int vlan_id_out; unsigned char *src; - unsigned char *dst; + const unsigned char *dst; u32 ip_src; u32 ip_dst; int tcp; @@ -249,8 +249,8 @@ static int stmmac_test_loopback_validate(struct sk_buff *skb, struct net_device *orig_ndev) { struct stmmac_test_priv *tpriv = pt->af_packet_priv; + const unsigned char *dst = tpriv->packet->dst; unsigned char *src = tpriv->packet->src; - unsigned char *dst = tpriv->packet->dst; struct stmmachdr *shdr; struct ethhdr *ehdr; struct udphdr *uhdr; @@ -380,7 +380,7 @@ static int stmmac_test_phy_loopback(struct stmmac_priv *priv) int ret; if (!priv->dev->phydev) - return -EBUSY; + return -EOPNOTSUPP; ret = phy_loopback(priv->dev->phydev, true); if (ret) @@ -795,8 +795,8 @@ static int stmmac_test_flowctrl(struct stmmac_priv *priv) struct stmmac_channel *ch = &priv->channel[i]; u32 tail; - tail = priv->rx_queue[i].dma_rx_phy + - (DMA_RX_SIZE * sizeof(struct dma_desc)); + tail = priv->dma_conf.rx_queue[i].dma_rx_phy + + (priv->dma_conf.dma_rx_size * sizeof(struct dma_desc)); stmmac_set_rx_tail_ptr(priv, priv->ioaddr, tail, i); stmmac_start_rx(priv, priv->ioaddr, i); @@ -1084,8 +1084,9 @@ static int stmmac_test_rxp(struct stmmac_priv *priv) unsigned char addr[ETH_ALEN] = {0xde, 0xad, 0xbe, 0xef, 0x00, 0x00}; struct tc_cls_u32_offload cls_u32 = { }; struct stmmac_packet_attrs attr = { }; - struct tc_action **actions, *act; + struct tc_action **actions; struct tc_u32_sel *sel; + struct tcf_gact *gact; struct tcf_exts *exts; int ret, i, nk = 1; @@ -1094,7 +1095,7 @@ static int stmmac_test_rxp(struct stmmac_priv *priv) if (!priv->dma_cap.frpsel) return -EOPNOTSUPP; - sel = kzalloc(sizeof(*sel) + nk * sizeof(struct tc_u32_key), GFP_KERNEL); + sel = kzalloc(struct_size(sel, keys, nk), GFP_KERNEL); if (!sel) return -ENOMEM; @@ -1104,14 +1105,14 @@ static int stmmac_test_rxp(struct stmmac_priv *priv) goto cleanup_sel; } - actions = kzalloc(nk * sizeof(*actions), GFP_KERNEL); + actions = kcalloc(nk, sizeof(*actions), GFP_KERNEL); if (!actions) { ret = -ENOMEM; goto cleanup_exts; } - act = kzalloc(nk * sizeof(*act), GFP_KERNEL); - if (!act) { + gact = kcalloc(nk, sizeof(*gact), GFP_KERNEL); + if (!gact) { ret = -ENOMEM; goto cleanup_actions; } @@ -1126,9 +1127,7 @@ static int stmmac_test_rxp(struct stmmac_priv *priv) exts->nr_actions = nk; exts->actions = actions; for (i = 0; i < nk; i++) { - struct tcf_gact *gact = to_gact(&act[i]); - - actions[i] = &act[i]; + actions[i] = (struct tc_action *)&gact[i]; gact->tcf_action = TC_ACT_SHOT; } @@ -1152,7 +1151,7 @@ static int stmmac_test_rxp(struct stmmac_priv *priv) stmmac_tc_setup_cls_u32(priv, priv, &cls_u32); cleanup_act: - kfree(act); + kfree(gact); cleanup_actions: kfree(actions); cleanup_exts: @@ -1387,6 +1386,7 @@ static int __stmmac_test_l3filt(struct stmmac_priv *priv, u32 dst, u32 src, cls->rule = rule; rule->action.entries[0].id = FLOW_ACTION_DROP; + rule->action.entries[0].hw_stats = FLOW_ACTION_HW_STATS_ANY; rule->action.num_entries = 1; attr.dst = priv->dev->dev_addr; @@ -1515,6 +1515,7 @@ static int __stmmac_test_l4filt(struct stmmac_priv *priv, u32 dst, u32 src, cls->rule = rule; rule->action.entries[0].id = FLOW_ACTION_DROP; + rule->action.entries[0].hw_stats = FLOW_ACTION_HW_STATS_ANY; rule->action.num_entries = 1; attr.dst = priv->dev->dev_addr; @@ -1679,7 +1680,7 @@ cleanup: static int __stmmac_test_jumbo(struct stmmac_priv *priv, u16 queue) { struct stmmac_packet_attrs attr = { }; - int size = priv->dma_buf_sz; + int size = priv->dma_conf.dma_buf_sz; attr.dst = priv->dev->dev_addr; attr.max_size = size - ETH_FCS_LEN; @@ -1762,7 +1763,7 @@ static int stmmac_test_tbs(struct stmmac_priv *priv) /* Find first TBS enabled Queue, if any */ for (i = 0; i < priv->plat->tx_queues_to_use; i++) - if (priv->tx_queue[i].tbs & STMMAC_TBS_AVAIL) + if (priv->dma_conf.tx_queue[i].tbs & STMMAC_TBS_AVAIL) break; if (i >= priv->plat->tx_queues_to_use) @@ -1775,9 +1776,9 @@ static int stmmac_test_tbs(struct stmmac_priv *priv) if (ret) return ret; - spin_lock_irqsave(&priv->ptp_lock, flags); + read_lock_irqsave(&priv->ptp_lock, flags); stmmac_get_systime(priv, priv->ptpaddr, &curr_time); - spin_unlock_irqrestore(&priv->ptp_lock, flags); + read_unlock_irqrestore(&priv->ptp_lock, flags); if (!curr_time) { ret = -EOPNOTSUPP; @@ -1797,9 +1798,9 @@ static int stmmac_test_tbs(struct stmmac_priv *priv) goto fail_disable; /* Check if expected time has elapsed */ - spin_lock_irqsave(&priv->ptp_lock, flags); + read_lock_irqsave(&priv->ptp_lock, flags); stmmac_get_systime(priv, priv->ptpaddr, &curr_time); - spin_unlock_irqrestore(&priv->ptp_lock, flags); + read_unlock_irqrestore(&priv->ptp_lock, flags); if ((curr_time - start_time) < STMMAC_TBS_LT_OFFSET) ret = -EINVAL; @@ -1983,7 +1984,7 @@ void stmmac_selftest_run(struct net_device *dev, ret = phy_loopback(dev->phydev, true); if (!ret) break; - /* Fallthrough */ + fallthrough; case STMMAC_LOOPBACK_MAC: ret = stmmac_set_mac_loopback(priv, priv->ioaddr, true); break; @@ -2016,7 +2017,7 @@ void stmmac_selftest_run(struct net_device *dev, ret = phy_loopback(dev->phydev, false); if (!ret) break; - /* Fallthrough */ + fallthrough; case STMMAC_LOOPBACK_MAC: stmmac_set_mac_loopback(priv, priv->ioaddr, false); break; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c index 7a01dee2f9a8..773e415cc2de 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c @@ -209,17 +209,11 @@ err_unfill: static int tc_delete_knode(struct stmmac_priv *priv, struct tc_cls_u32_offload *cls) { - int ret; - /* Set entry and fragments as not used */ tc_unfill_entry(priv, cls); - ret = stmmac_rxp_config(priv, priv->hw->pcsr, priv->tc_entries, - priv->tc_entries_max); - if (ret) - return ret; - - return 0; + return stmmac_rxp_config(priv, priv->hw->pcsr, priv->tc_entries, + priv->tc_entries_max); } static int tc_setup_cls_u32(struct stmmac_priv *priv, @@ -228,7 +222,7 @@ static int tc_setup_cls_u32(struct stmmac_priv *priv, switch (cls->command) { case TC_CLSU32_REPLACE_KNODE: tc_unfill_entry(priv, cls); - /* Fall through */ + fallthrough; case TC_CLSU32_NEW_KNODE: return tc_config_knode(priv, cls); case TC_CLSU32_DELETE_KNODE: @@ -238,11 +232,35 @@ static int tc_setup_cls_u32(struct stmmac_priv *priv, } } +static int tc_rfs_init(struct stmmac_priv *priv) +{ + int i; + + priv->rfs_entries_max[STMMAC_RFS_T_VLAN] = 8; + priv->rfs_entries_max[STMMAC_RFS_T_LLDP] = 1; + priv->rfs_entries_max[STMMAC_RFS_T_1588] = 1; + + for (i = 0; i < STMMAC_RFS_T_MAX; i++) + priv->rfs_entries_total += priv->rfs_entries_max[i]; + + priv->rfs_entries = devm_kcalloc(priv->device, + priv->rfs_entries_total, + sizeof(*priv->rfs_entries), + GFP_KERNEL); + if (!priv->rfs_entries) + return -ENOMEM; + + dev_info(priv->device, "Enabled RFS Flow TC (entries=%d)\n", + priv->rfs_entries_total); + + return 0; +} + static int tc_init(struct stmmac_priv *priv) { struct dma_features *dma_cap = &priv->dma_cap; unsigned int count; - int i; + int ret, i; if (dma_cap->l3l4fnum) { priv->flow_entries_max = dma_cap->l3l4fnum; @@ -256,10 +274,24 @@ static int tc_init(struct stmmac_priv *priv) for (i = 0; i < priv->flow_entries_max; i++) priv->flow_entries[i].idx = i; - dev_info(priv->device, "Enabled Flow TC (entries=%d)\n", + dev_info(priv->device, "Enabled L3L4 Flow TC (entries=%d)\n", priv->flow_entries_max); } + ret = tc_rfs_init(priv); + if (ret) + return -ENOMEM; + + if (!priv->plat->fpe_cfg) { + priv->plat->fpe_cfg = devm_kzalloc(priv->device, + sizeof(*priv->plat->fpe_cfg), + GFP_KERNEL); + if (!priv->plat->fpe_cfg) + return -ENOMEM; + } else { + memset(priv->plat->fpe_cfg, 0, sizeof(*priv->plat->fpe_cfg)); + } + /* Fail silently as we can still use remaining features, e.g. CBS */ if (!dma_cap->frpsel) return 0; @@ -303,6 +335,7 @@ static int tc_init(struct stmmac_priv *priv) dev_info(priv->device, "Enabling HW TC (entries=%d, max_off=%d)\n", priv->tc_entries_max, priv->tc_off_max); + return 0; } @@ -322,6 +355,32 @@ static int tc_setup_cbs(struct stmmac_priv *priv, if (!priv->dma_cap.av) return -EOPNOTSUPP; + /* Port Transmit Rate and Speed Divider */ + switch (priv->speed) { + case SPEED_10000: + ptr = 32; + speed_div = 10000000; + break; + case SPEED_5000: + ptr = 32; + speed_div = 5000000; + break; + case SPEED_2500: + ptr = 8; + speed_div = 2500000; + break; + case SPEED_1000: + ptr = 8; + speed_div = 1000000; + break; + case SPEED_100: + ptr = 4; + speed_div = 100000; + break; + default: + return -EOPNOTSUPP; + } + mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use; if (mode_to_use == MTL_QUEUE_DCB && qopt->enable) { ret = stmmac_dma_qmode(priv, priv->ioaddr, queue, MTL_QUEUE_AVB); @@ -330,12 +389,13 @@ static int tc_setup_cbs(struct stmmac_priv *priv, priv->plat->tx_queues_cfg[queue].mode_to_use = MTL_QUEUE_AVB; } else if (!qopt->enable) { - return stmmac_dma_qmode(priv, priv->ioaddr, queue, MTL_QUEUE_DCB); - } + ret = stmmac_dma_qmode(priv, priv->ioaddr, queue, + MTL_QUEUE_DCB); + if (ret) + return ret; - /* Port Transmit Rate and Speed Divider */ - ptr = (priv->speed == SPEED_100) ? 4 : 8; - speed_div = (priv->speed == SPEED_100) ? 100000 : 1000000; + priv->plat->tx_queues_cfg[queue].mode_to_use = MTL_QUEUE_DCB; + } /* Final adjustments for HW */ value = div_s64(qopt->idleslope * 1024ll * ptr, speed_div); @@ -367,7 +427,8 @@ static int tc_setup_cbs(struct stmmac_priv *priv, static int tc_parse_flow_actions(struct stmmac_priv *priv, struct flow_action *action, - struct stmmac_flow_entry *entry) + struct stmmac_flow_entry *entry, + struct netlink_ext_ack *extack) { struct flow_action_entry *act; int i; @@ -375,6 +436,9 @@ static int tc_parse_flow_actions(struct stmmac_priv *priv, if (!flow_action_has_entries(action)) return -EINVAL; + if (!flow_action_basic_hw_stats_check(action, extack)) + return -EOPNOTSUPP; + flow_action_for_each(i, act, action) { switch (act->id) { case FLOW_ACTION_DROP: @@ -389,6 +453,8 @@ static int tc_parse_flow_actions(struct stmmac_priv *priv, return 0; } +#define ETHER_TYPE_FULL_MASK cpu_to_be16(~0) + static int tc_add_basic_flow(struct stmmac_priv *priv, struct flow_cls_offload *cls, struct stmmac_flow_entry *entry) @@ -402,6 +468,7 @@ static int tc_add_basic_flow(struct stmmac_priv *priv, return -EINVAL; flow_rule_match_basic(rule, &match); + entry->ip_proto = match.key->ip_proto; return 0; } @@ -530,16 +597,15 @@ static int tc_add_flow(struct stmmac_priv *priv, return -ENOENT; } - ret = tc_parse_flow_actions(priv, &rule->action, entry); + ret = tc_parse_flow_actions(priv, &rule->action, entry, + cls->common.extack); if (ret) return ret; for (i = 0; i < ARRAY_SIZE(tc_flow_parsers); i++) { ret = tc_flow_parsers[i].fn(priv, cls, entry); - if (!ret) { + if (!ret) entry->in_use = true; - continue; - } } if (!entry->in_use) @@ -572,6 +638,237 @@ static int tc_del_flow(struct stmmac_priv *priv, return ret; } +static struct stmmac_rfs_entry *tc_find_rfs(struct stmmac_priv *priv, + struct flow_cls_offload *cls, + bool get_free) +{ + int i; + + for (i = 0; i < priv->rfs_entries_total; i++) { + struct stmmac_rfs_entry *entry = &priv->rfs_entries[i]; + + if (entry->cookie == cls->cookie) + return entry; + if (get_free && entry->in_use == false) + return entry; + } + + return NULL; +} + +#define VLAN_PRIO_FULL_MASK (0x07) + +static int tc_add_vlan_flow(struct stmmac_priv *priv, + struct flow_cls_offload *cls) +{ + struct stmmac_rfs_entry *entry = tc_find_rfs(priv, cls, false); + struct flow_rule *rule = flow_cls_offload_flow_rule(cls); + struct flow_dissector *dissector = rule->match.dissector; + int tc = tc_classid_to_hwtc(priv->dev, cls->classid); + struct flow_match_vlan match; + + if (!entry) { + entry = tc_find_rfs(priv, cls, true); + if (!entry) + return -ENOENT; + } + + if (priv->rfs_entries_cnt[STMMAC_RFS_T_VLAN] >= + priv->rfs_entries_max[STMMAC_RFS_T_VLAN]) + return -ENOENT; + + /* Nothing to do here */ + if (!dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_VLAN)) + return -EINVAL; + + if (tc < 0) { + netdev_err(priv->dev, "Invalid traffic class\n"); + return -EINVAL; + } + + flow_rule_match_vlan(rule, &match); + + if (match.mask->vlan_priority) { + u32 prio; + + if (match.mask->vlan_priority != VLAN_PRIO_FULL_MASK) { + netdev_err(priv->dev, "Only full mask is supported for VLAN priority"); + return -EINVAL; + } + + prio = BIT(match.key->vlan_priority); + stmmac_rx_queue_prio(priv, priv->hw, prio, tc); + + entry->in_use = true; + entry->cookie = cls->cookie; + entry->tc = tc; + entry->type = STMMAC_RFS_T_VLAN; + priv->rfs_entries_cnt[STMMAC_RFS_T_VLAN]++; + } + + return 0; +} + +static int tc_del_vlan_flow(struct stmmac_priv *priv, + struct flow_cls_offload *cls) +{ + struct stmmac_rfs_entry *entry = tc_find_rfs(priv, cls, false); + + if (!entry || !entry->in_use || entry->type != STMMAC_RFS_T_VLAN) + return -ENOENT; + + stmmac_rx_queue_prio(priv, priv->hw, 0, entry->tc); + + entry->in_use = false; + entry->cookie = 0; + entry->tc = 0; + entry->type = 0; + + priv->rfs_entries_cnt[STMMAC_RFS_T_VLAN]--; + + return 0; +} + +static int tc_add_ethtype_flow(struct stmmac_priv *priv, + struct flow_cls_offload *cls) +{ + struct stmmac_rfs_entry *entry = tc_find_rfs(priv, cls, false); + struct flow_rule *rule = flow_cls_offload_flow_rule(cls); + struct flow_dissector *dissector = rule->match.dissector; + int tc = tc_classid_to_hwtc(priv->dev, cls->classid); + struct flow_match_basic match; + + if (!entry) { + entry = tc_find_rfs(priv, cls, true); + if (!entry) + return -ENOENT; + } + + /* Nothing to do here */ + if (!dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_BASIC)) + return -EINVAL; + + if (tc < 0) { + netdev_err(priv->dev, "Invalid traffic class\n"); + return -EINVAL; + } + + flow_rule_match_basic(rule, &match); + + if (match.mask->n_proto) { + u16 etype = ntohs(match.key->n_proto); + + if (match.mask->n_proto != ETHER_TYPE_FULL_MASK) { + netdev_err(priv->dev, "Only full mask is supported for EthType filter"); + return -EINVAL; + } + switch (etype) { + case ETH_P_LLDP: + if (priv->rfs_entries_cnt[STMMAC_RFS_T_LLDP] >= + priv->rfs_entries_max[STMMAC_RFS_T_LLDP]) + return -ENOENT; + + entry->type = STMMAC_RFS_T_LLDP; + priv->rfs_entries_cnt[STMMAC_RFS_T_LLDP]++; + + stmmac_rx_queue_routing(priv, priv->hw, + PACKET_DCBCPQ, tc); + break; + case ETH_P_1588: + if (priv->rfs_entries_cnt[STMMAC_RFS_T_1588] >= + priv->rfs_entries_max[STMMAC_RFS_T_1588]) + return -ENOENT; + + entry->type = STMMAC_RFS_T_1588; + priv->rfs_entries_cnt[STMMAC_RFS_T_1588]++; + + stmmac_rx_queue_routing(priv, priv->hw, + PACKET_PTPQ, tc); + break; + default: + netdev_err(priv->dev, "EthType(0x%x) is not supported", etype); + return -EINVAL; + } + + entry->in_use = true; + entry->cookie = cls->cookie; + entry->tc = tc; + entry->etype = etype; + + return 0; + } + + return -EINVAL; +} + +static int tc_del_ethtype_flow(struct stmmac_priv *priv, + struct flow_cls_offload *cls) +{ + struct stmmac_rfs_entry *entry = tc_find_rfs(priv, cls, false); + + if (!entry || !entry->in_use || + entry->type < STMMAC_RFS_T_LLDP || + entry->type > STMMAC_RFS_T_1588) + return -ENOENT; + + switch (entry->etype) { + case ETH_P_LLDP: + stmmac_rx_queue_routing(priv, priv->hw, + PACKET_DCBCPQ, 0); + priv->rfs_entries_cnt[STMMAC_RFS_T_LLDP]--; + break; + case ETH_P_1588: + stmmac_rx_queue_routing(priv, priv->hw, + PACKET_PTPQ, 0); + priv->rfs_entries_cnt[STMMAC_RFS_T_1588]--; + break; + default: + netdev_err(priv->dev, "EthType(0x%x) is not supported", + entry->etype); + return -EINVAL; + } + + entry->in_use = false; + entry->cookie = 0; + entry->tc = 0; + entry->etype = 0; + entry->type = 0; + + return 0; +} + +static int tc_add_flow_cls(struct stmmac_priv *priv, + struct flow_cls_offload *cls) +{ + int ret; + + ret = tc_add_flow(priv, cls); + if (!ret) + return ret; + + ret = tc_add_ethtype_flow(priv, cls); + if (!ret) + return ret; + + return tc_add_vlan_flow(priv, cls); +} + +static int tc_del_flow_cls(struct stmmac_priv *priv, + struct flow_cls_offload *cls) +{ + int ret; + + ret = tc_del_flow(priv, cls); + if (!ret) + return ret; + + ret = tc_del_ethtype_flow(priv, cls); + if (!ret) + return ret; + + return tc_del_vlan_flow(priv, cls); +} + static int tc_setup_cls(struct stmmac_priv *priv, struct flow_cls_offload *cls) { @@ -583,10 +880,10 @@ static int tc_setup_cls(struct stmmac_priv *priv, switch (cls->command) { case FLOW_CLS_REPLACE: - ret = tc_add_flow(priv, cls); + ret = tc_add_flow_cls(priv, cls); break; case FLOW_CLS_DESTROY: - ret = tc_del_flow(priv, cls); + ret = tc_del_flow_cls(priv, cls); break; default: return -EOPNOTSUPP; @@ -595,12 +892,36 @@ static int tc_setup_cls(struct stmmac_priv *priv, return ret; } +struct timespec64 stmmac_calc_tas_basetime(ktime_t old_base_time, + ktime_t current_time, + u64 cycle_time) +{ + struct timespec64 time; + + if (ktime_after(old_base_time, current_time)) { + time = ktime_to_timespec64(old_base_time); + } else { + s64 n; + ktime_t base_time; + + n = div64_s64(ktime_sub_ns(current_time, old_base_time), + cycle_time); + base_time = ktime_add_ns(old_base_time, + (n + 1) * cycle_time); + + time = ktime_to_timespec64(base_time); + } + + return time; +} + static int tc_setup_taprio(struct stmmac_priv *priv, struct tc_taprio_qopt_offload *qopt) { u32 size, wid = priv->dma_cap.estwid, dep = priv->dma_cap.estdep; struct plat_stmmacenet_data *plat = priv->plat; - struct timespec64 time; + struct timespec64 time, current_time, qopt_time; + ktime_t current_time_ns; bool fpe = false; int i, ret = 0; u64 ctr; @@ -646,8 +967,6 @@ static int tc_setup_taprio(struct stmmac_priv *priv, goto disable; if (qopt->num_entries >= dep) return -EINVAL; - if (!qopt->base_time) - return -ERANGE; if (!qopt->cycle_time) return -ERANGE; @@ -656,14 +975,18 @@ static int tc_setup_taprio(struct stmmac_priv *priv, GFP_KERNEL); if (!plat->est) return -ENOMEM; + + mutex_init(&priv->plat->est->lock); } else { memset(plat->est, 0, sizeof(*plat->est)); } size = qopt->num_entries; + mutex_lock(&priv->plat->est->lock); priv->plat->est->gcl_size = size; priv->plat->est->enable = qopt->enable; + mutex_unlock(&priv->plat->est->lock); for (i = 0; i < size; i++) { s64 delta_ns = qopt->entries[i].interval; @@ -694,40 +1017,70 @@ static int tc_setup_taprio(struct stmmac_priv *priv, priv->plat->est->gcl[i] = delta_ns | (gates << wid); } + mutex_lock(&priv->plat->est->lock); /* Adjust for real system time */ - time = ktime_to_timespec64(qopt->base_time); + priv->ptp_clock_ops.gettime64(&priv->ptp_clock_ops, ¤t_time); + current_time_ns = timespec64_to_ktime(current_time); + time = stmmac_calc_tas_basetime(qopt->base_time, current_time_ns, + qopt->cycle_time); + priv->plat->est->btr[0] = (u32)time.tv_nsec; priv->plat->est->btr[1] = (u32)time.tv_sec; + qopt_time = ktime_to_timespec64(qopt->base_time); + priv->plat->est->btr_reserve[0] = (u32)qopt_time.tv_nsec; + priv->plat->est->btr_reserve[1] = (u32)qopt_time.tv_sec; + ctr = qopt->cycle_time; priv->plat->est->ctr[0] = do_div(ctr, NSEC_PER_SEC); priv->plat->est->ctr[1] = (u32)ctr; - if (fpe && !priv->dma_cap.fpesel) + if (fpe && !priv->dma_cap.fpesel) { + mutex_unlock(&priv->plat->est->lock); return -EOPNOTSUPP; - - ret = stmmac_fpe_configure(priv, priv->ioaddr, - priv->plat->tx_queues_to_use, - priv->plat->rx_queues_to_use, fpe); - if (ret && fpe) { - netdev_err(priv->dev, "failed to enable Frame Preemption\n"); - return ret; } + /* Actual FPE register configuration will be done after FPE handshake + * is success. + */ + priv->plat->fpe_cfg->enable = fpe; + ret = stmmac_est_configure(priv, priv->ioaddr, priv->plat->est, priv->plat->clk_ptp_rate); + mutex_unlock(&priv->plat->est->lock); if (ret) { netdev_err(priv->dev, "failed to configure EST\n"); goto disable; } netdev_info(priv->dev, "configured EST\n"); + + if (fpe) { + stmmac_fpe_handshake(priv, true); + netdev_info(priv->dev, "start FPE handshake\n"); + } + return 0; disable: - priv->plat->est->enable = false; - stmmac_est_configure(priv, priv->ioaddr, priv->plat->est, - priv->plat->clk_ptp_rate); + if (priv->plat->est) { + mutex_lock(&priv->plat->est->lock); + priv->plat->est->enable = false; + stmmac_est_configure(priv, priv->ioaddr, priv->plat->est, + priv->plat->clk_ptp_rate); + mutex_unlock(&priv->plat->est->lock); + } + + priv->plat->fpe_cfg->enable = false; + stmmac_fpe_configure(priv, priv->ioaddr, + priv->plat->tx_queues_to_use, + priv->plat->rx_queues_to_use, + false); + netdev_info(priv->dev, "disabled FPE\n"); + + stmmac_fpe_handshake(priv, false); + netdev_info(priv->dev, "stop FPE handshake\n"); + return ret; } @@ -738,13 +1091,13 @@ static int tc_setup_etf(struct stmmac_priv *priv, return -EOPNOTSUPP; if (qopt->queue >= priv->plat->tx_queues_to_use) return -EINVAL; - if (!(priv->tx_queue[qopt->queue].tbs & STMMAC_TBS_AVAIL)) + if (!(priv->dma_conf.tx_queue[qopt->queue].tbs & STMMAC_TBS_AVAIL)) return -EINVAL; if (qopt->enable) - priv->tx_queue[qopt->queue].tbs |= STMMAC_TBS_EN; + priv->dma_conf.tx_queue[qopt->queue].tbs |= STMMAC_TBS_EN; else - priv->tx_queue[qopt->queue].tbs &= ~STMMAC_TBS_EN; + priv->dma_conf.tx_queue[qopt->queue].tbs &= ~STMMAC_TBS_EN; netdev_info(priv->dev, "%s ETF for Queue %d\n", qopt->enable ? "enabled" : "disabled", qopt->queue); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_xdp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_xdp.c new file mode 100644 index 000000000000..9d4d8c3dad0a --- /dev/null +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_xdp.c @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2021, Intel Corporation. */ + +#include <net/xdp_sock_drv.h> + +#include "stmmac.h" +#include "stmmac_xdp.h" + +static int stmmac_xdp_enable_pool(struct stmmac_priv *priv, + struct xsk_buff_pool *pool, u16 queue) +{ + struct stmmac_channel *ch = &priv->channel[queue]; + bool need_update; + u32 frame_size; + int err; + + if (queue >= priv->plat->rx_queues_to_use || + queue >= priv->plat->tx_queues_to_use) + return -EINVAL; + + frame_size = xsk_pool_get_rx_frame_size(pool); + /* XDP ZC does not span multiple frame, make sure XSK pool buffer + * size can at least store Q-in-Q frame. + */ + if (frame_size < ETH_FRAME_LEN + VLAN_HLEN * 2) + return -EOPNOTSUPP; + + err = xsk_pool_dma_map(pool, priv->device, STMMAC_RX_DMA_ATTR); + if (err) { + netdev_err(priv->dev, "Failed to map xsk pool\n"); + return err; + } + + need_update = netif_running(priv->dev) && stmmac_xdp_is_enabled(priv); + + if (need_update) { + napi_disable(&ch->rx_napi); + napi_disable(&ch->tx_napi); + stmmac_disable_rx_queue(priv, queue); + stmmac_disable_tx_queue(priv, queue); + } + + set_bit(queue, priv->af_xdp_zc_qps); + + if (need_update) { + stmmac_enable_rx_queue(priv, queue); + stmmac_enable_tx_queue(priv, queue); + napi_enable(&ch->rxtx_napi); + + err = stmmac_xsk_wakeup(priv->dev, queue, XDP_WAKEUP_RX); + if (err) + return err; + } + + return 0; +} + +static int stmmac_xdp_disable_pool(struct stmmac_priv *priv, u16 queue) +{ + struct stmmac_channel *ch = &priv->channel[queue]; + struct xsk_buff_pool *pool; + bool need_update; + + if (queue >= priv->plat->rx_queues_to_use || + queue >= priv->plat->tx_queues_to_use) + return -EINVAL; + + pool = xsk_get_pool_from_qid(priv->dev, queue); + if (!pool) + return -EINVAL; + + need_update = netif_running(priv->dev) && stmmac_xdp_is_enabled(priv); + + if (need_update) { + napi_disable(&ch->rxtx_napi); + stmmac_disable_rx_queue(priv, queue); + stmmac_disable_tx_queue(priv, queue); + synchronize_rcu(); + } + + xsk_pool_dma_unmap(pool, STMMAC_RX_DMA_ATTR); + + clear_bit(queue, priv->af_xdp_zc_qps); + + if (need_update) { + stmmac_enable_rx_queue(priv, queue); + stmmac_enable_tx_queue(priv, queue); + napi_enable(&ch->rx_napi); + napi_enable(&ch->tx_napi); + } + + return 0; +} + +int stmmac_xdp_setup_pool(struct stmmac_priv *priv, struct xsk_buff_pool *pool, + u16 queue) +{ + return pool ? stmmac_xdp_enable_pool(priv, pool, queue) : + stmmac_xdp_disable_pool(priv, queue); +} + +int stmmac_xdp_set_prog(struct stmmac_priv *priv, struct bpf_prog *prog, + struct netlink_ext_ack *extack) +{ + struct net_device *dev = priv->dev; + struct bpf_prog *old_prog; + bool need_update; + bool if_running; + + if_running = netif_running(dev); + + if (prog && dev->mtu > ETH_DATA_LEN) { + /* For now, the driver doesn't support XDP functionality with + * jumbo frames so we return error. + */ + NL_SET_ERR_MSG_MOD(extack, "Jumbo frames not supported"); + return -EOPNOTSUPP; + } + + need_update = !!priv->xdp_prog != !!prog; + if (if_running && need_update) + stmmac_xdp_release(dev); + + old_prog = xchg(&priv->xdp_prog, prog); + if (old_prog) + bpf_prog_put(old_prog); + + /* Disable RX SPH for XDP operation */ + priv->sph = priv->sph_cap && !stmmac_xdp_is_enabled(priv); + + if (if_running && need_update) + stmmac_xdp_open(dev); + + return 0; +} diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_xdp.h b/drivers/net/ethernet/stmicro/stmmac/stmmac_xdp.h new file mode 100644 index 000000000000..896dc987d4ef --- /dev/null +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_xdp.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2021, Intel Corporation. */ + +#ifndef _STMMAC_XDP_H_ +#define _STMMAC_XDP_H_ + +#define STMMAC_MAX_RX_BUF_SIZE(num) (((num) * PAGE_SIZE) - XDP_PACKET_HEADROOM) +#define STMMAC_RX_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING) + +int stmmac_xdp_setup_pool(struct stmmac_priv *priv, struct xsk_buff_pool *pool, + u16 queue); +int stmmac_xdp_set_prog(struct stmmac_priv *priv, struct bpf_prog *prog, + struct netlink_ext_ack *extack); + +#endif /* _STMMAC_XDP_H_ */ |