diff options
| author |   Maarten Lankhorst <maarten.lankhorst@linux.intel.com> | 2020-08-25 11:00:02 +0200 |
|---|---|---|
| committer | Maarten Lankhorst <maarten.lankhorst@linux.intel.com> | 2020-08-25 11:00:02 +0200 |
| commit | 2d9ad4cfaf4d32a64a4ed556e5bcab9121215026 (patch) | |
| tree | 3572e6cd05effa4e2943cee817defb2b9a72afd1 /drivers/net/ethernet/mellanox | |
| parent | drm/modeset-lock: Take the modeset BKL for legacy drivers (diff) | |
| parent | Linux 5.9-rc2 (diff) | |
| download | wireguard-linux-2d9ad4cfaf4d32a64a4ed556e5bcab9121215026.tar.xz wireguard-linux-2d9ad4cfaf4d32a64a4ed556e5bcab9121215026.zip | |
Merge tag 'v5.9-rc2' into drm-misc-fixes
Backmerge requested by Tomi for a fix to omap inconsistent
locking state issue, and because we need at least v5.9-rc2 now.
Signed-off-by: Maarten Lankhorst <maarten.lankhorst@linux.intel.com>
Diffstat (limited to 'drivers/net/ethernet/mellanox')
137 files changed, 9888 insertions, 3742 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c index 5bd3cd37d50f..106513f772c3 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c @@ -1816,7 +1816,7 @@ int mlx4_en_start_port(struct net_device *dev) queue_work(mdev->workqueue, &priv->rx_mode_task); if (priv->mdev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) - udp_tunnel_get_rx_info(dev); + udp_tunnel_nic_reset_ntf(dev); priv->port_up = true; @@ -2628,89 +2628,32 @@ static int mlx4_en_get_phys_port_id(struct net_device *dev, return 0; } -static void mlx4_en_add_vxlan_offloads(struct work_struct *work) +static int mlx4_udp_tunnel_sync(struct net_device *dev, unsigned int table) { + struct mlx4_en_priv *priv = netdev_priv(dev); + struct udp_tunnel_info ti; int ret; - struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, - vxlan_add_task); - ret = mlx4_config_vxlan_port(priv->mdev->dev, priv->vxlan_port); - if (ret) - goto out; - - ret = mlx4_SET_PORT_VXLAN(priv->mdev->dev, priv->port, - VXLAN_STEER_BY_OUTER_MAC, 1); -out: - if (ret) { - en_err(priv, "failed setting L2 tunnel configuration ret %d\n", ret); - return; - } -} + udp_tunnel_nic_get_port(dev, table, 0, &ti); + priv->vxlan_port = ti.port; -static void mlx4_en_del_vxlan_offloads(struct work_struct *work) -{ - int ret; - struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, - vxlan_del_task); - ret = mlx4_SET_PORT_VXLAN(priv->mdev->dev, priv->port, - VXLAN_STEER_BY_OUTER_MAC, 0); + ret = mlx4_config_vxlan_port(priv->mdev->dev, priv->vxlan_port); if (ret) - en_err(priv, "failed setting L2 tunnel configuration ret %d\n", ret); - - priv->vxlan_port = 0; -} - -static void mlx4_en_add_vxlan_port(struct net_device *dev, - struct udp_tunnel_info *ti) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - __be16 port = ti->port; - __be16 current_port; - - if (ti->type != UDP_TUNNEL_TYPE_VXLAN) - return; - - if (ti->sa_family != AF_INET) - return; - - if (priv->mdev->dev->caps.tunnel_offload_mode != MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) - return; - - current_port = priv->vxlan_port; - if (current_port && current_port != port) { - en_warn(priv, "vxlan port %d configured, can't add port %d\n", - ntohs(current_port), ntohs(port)); - return; - } + return ret; - priv->vxlan_port = port; - queue_work(priv->mdev->workqueue, &priv->vxlan_add_task); + return mlx4_SET_PORT_VXLAN(priv->mdev->dev, priv->port, + VXLAN_STEER_BY_OUTER_MAC, + !!priv->vxlan_port); } -static void mlx4_en_del_vxlan_port(struct net_device *dev, - struct udp_tunnel_info *ti) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - __be16 port = ti->port; - __be16 current_port; - - if (ti->type != UDP_TUNNEL_TYPE_VXLAN) - return; - - if (ti->sa_family != AF_INET) - return; - - if (priv->mdev->dev->caps.tunnel_offload_mode != MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) - return; - - current_port = priv->vxlan_port; - if (current_port != port) { - en_dbg(DRV, priv, "vxlan port %d isn't configured, ignoring\n", ntohs(port)); - return; - } - - queue_work(priv->mdev->workqueue, &priv->vxlan_del_task); -} +static const struct udp_tunnel_nic_info mlx4_udp_tunnels = { + .sync_table = mlx4_udp_tunnel_sync, + .flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP | + UDP_TUNNEL_NIC_INFO_IPV4_ONLY, + .tables = { + { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, }, + }, +}; static netdev_features_t mlx4_en_features_check(struct sk_buff *skb, struct net_device *dev, @@ -2859,35 +2802,11 @@ unlock_out: return err; } -static u32 mlx4_xdp_query(struct net_device *dev) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - const struct bpf_prog *xdp_prog; - u32 prog_id = 0; - - if (!priv->tx_ring_num[TX_XDP]) - return prog_id; - - mutex_lock(&mdev->state_lock); - xdp_prog = rcu_dereference_protected( - priv->rx_ring[0]->xdp_prog, - lockdep_is_held(&mdev->state_lock)); - if (xdp_prog) - prog_id = xdp_prog->aux->id; - mutex_unlock(&mdev->state_lock); - - return prog_id; -} - static int mlx4_xdp(struct net_device *dev, struct netdev_bpf *xdp) { switch (xdp->command) { case XDP_SETUP_PROG: return mlx4_xdp_set(dev, xdp->prog); - case XDP_QUERY_PROG: - xdp->prog_id = mlx4_xdp_query(dev); - return 0; default: return -EINVAL; } @@ -2914,8 +2833,8 @@ static const struct net_device_ops mlx4_netdev_ops = { .ndo_rx_flow_steer = mlx4_en_filter_rfs, #endif .ndo_get_phys_port_id = mlx4_en_get_phys_port_id, - .ndo_udp_tunnel_add = mlx4_en_add_vxlan_port, - .ndo_udp_tunnel_del = mlx4_en_del_vxlan_port, + .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, + .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_features_check = mlx4_en_features_check, .ndo_set_tx_maxrate = mlx4_en_set_tx_maxrate, .ndo_bpf = mlx4_xdp, @@ -2948,8 +2867,8 @@ static const struct net_device_ops mlx4_netdev_ops_master = { .ndo_rx_flow_steer = mlx4_en_filter_rfs, #endif .ndo_get_phys_port_id = mlx4_en_get_phys_port_id, - .ndo_udp_tunnel_add = mlx4_en_add_vxlan_port, - .ndo_udp_tunnel_del = mlx4_en_del_vxlan_port, + .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, + .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_features_check = mlx4_en_features_check, .ndo_set_tx_maxrate = mlx4_en_set_tx_maxrate, .ndo_bpf = mlx4_xdp, @@ -3250,8 +3169,6 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, INIT_WORK(&priv->linkstate_task, mlx4_en_linkstate); INIT_DELAYED_WORK(&priv->stats_task, mlx4_en_do_get_stats); INIT_DELAYED_WORK(&priv->service_task, mlx4_en_service_task); - INIT_WORK(&priv->vxlan_add_task, mlx4_en_add_vxlan_offloads); - INIT_WORK(&priv->vxlan_del_task, mlx4_en_del_vxlan_offloads); #ifdef CONFIG_RFS_ACCEL INIT_LIST_HEAD(&priv->filters); spin_lock_init(&priv->filters_lock); @@ -3406,6 +3323,8 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_PARTIAL; + + dev->udp_tunnel_nic_info = &mlx4_udp_tunnels; } dev->vlan_features = dev->hw_features; diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c index 8a10285b0e10..b50c567ef508 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c @@ -806,10 +806,10 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud goto xdp_drop_no_cnt; /* Drop on xmit failure */ default: bpf_warn_invalid_xdp_action(act); - /* fall through */ + fallthrough; case XDP_ABORTED: trace_xdp_exception(dev, xdp_prog, act); - /* fall through */ + fallthrough; case XDP_DROP: ring->xdp_drop++; xdp_drop_no_cnt: diff --git a/drivers/net/ethernet/mellanox/mlx4/eq.c b/drivers/net/ethernet/mellanox/mlx4/eq.c index c790a5fcea73..ae305c2e9225 100644 --- a/drivers/net/ethernet/mellanox/mlx4/eq.c +++ b/drivers/net/ethernet/mellanox/mlx4/eq.c @@ -558,7 +558,7 @@ static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq) mlx4_dbg(dev, "%s: MLX4_EVENT_TYPE_SRQ_LIMIT. srq_no=0x%x, eq 0x%x\n", __func__, be32_to_cpu(eqe->event.srq.srqn), eq->eqn); - /* fall through */ + fallthrough; case MLX4_EVENT_TYPE_SRQ_CATAS_ERROR: if (mlx4_is_master(dev)) { /* forward only to slave owning the SRQ */ diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c index 3d9aa7da95e9..258c7a96f269 100644 --- a/drivers/net/ethernet/mellanox/mlx4/main.c +++ b/drivers/net/ethernet/mellanox/mlx4/main.c @@ -4356,12 +4356,14 @@ end: static void mlx4_shutdown(struct pci_dev *pdev) { struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev); + struct mlx4_dev *dev = persist->dev; mlx4_info(persist->dev, "mlx4_shutdown was called\n"); mutex_lock(&persist->interface_state_mutex); if (persist->interface_state & MLX4_INTERFACE_STATE_UP) mlx4_unload_one(pdev); mutex_unlock(&persist->interface_state_mutex); + mlx4_pci_disable_device(dev); } static const struct pci_error_handlers mlx4_err_handler = { @@ -4370,8 +4372,9 @@ static const struct pci_error_handlers mlx4_err_handler = { .resume = mlx4_pci_resume, }; -static int mlx4_suspend(struct pci_dev *pdev, pm_message_t state) +static int __maybe_unused mlx4_suspend(struct device *dev_d) { + struct pci_dev *pdev = to_pci_dev(dev_d); struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev); struct mlx4_dev *dev = persist->dev; @@ -4384,8 +4387,9 @@ static int mlx4_suspend(struct pci_dev *pdev, pm_message_t state) return 0; } -static int mlx4_resume(struct pci_dev *pdev) +static int __maybe_unused mlx4_resume(struct device *dev_d) { + struct pci_dev *pdev = to_pci_dev(dev_d); struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev); struct mlx4_dev *dev = persist->dev; struct mlx4_priv *priv = mlx4_priv(dev); @@ -4414,14 +4418,15 @@ static int mlx4_resume(struct pci_dev *pdev) return ret; } +static SIMPLE_DEV_PM_OPS(mlx4_pm_ops, mlx4_suspend, mlx4_resume); + static struct pci_driver mlx4_driver = { .name = DRV_NAME, .id_table = mlx4_pci_table, .probe = mlx4_init_one, .shutdown = mlx4_shutdown, .remove = mlx4_remove_one, - .suspend = mlx4_suspend, - .resume = mlx4_resume, + .driver.pm = &mlx4_pm_ops, .err_handler = &mlx4_err_handler, }; diff --git a/drivers/net/ethernet/mellanox/mlx4/mcg.c b/drivers/net/ethernet/mellanox/mlx4/mcg.c index 9486caecfbdc..f1b4ad9c66d2 100644 --- a/drivers/net/ethernet/mellanox/mlx4/mcg.c +++ b/drivers/net/ethernet/mellanox/mlx4/mcg.c @@ -1412,7 +1412,7 @@ int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], case MLX4_STEERING_MODE_A0: if (prot == MLX4_PROT_ETH) return 0; - /* fall through */ + fallthrough; case MLX4_STEERING_MODE_B0: if (prot == MLX4_PROT_ETH) @@ -1442,7 +1442,7 @@ int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], case MLX4_STEERING_MODE_A0: if (prot == MLX4_PROT_ETH) return 0; - /* fall through */ + fallthrough; case MLX4_STEERING_MODE_B0: if (prot == MLX4_PROT_ETH) diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h index 9f5603612960..a46efe37cfa9 100644 --- a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h +++ b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h @@ -599,8 +599,6 @@ struct mlx4_en_priv { struct work_struct linkstate_task; struct delayed_work stats_task; struct delayed_work service_task; - struct work_struct vxlan_add_task; - struct work_struct vxlan_del_task; struct mlx4_en_perf_stats pstats; struct mlx4_en_pkt_stats pkstats; struct mlx4_en_counter_stats pf_stats; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig index 4dfdbb82ea9d..99f1ec3b2575 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig +++ b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig @@ -134,12 +134,25 @@ config MLX5_FPGA_IPSEC mlx5_core driver will include the Innova FPGA core and allow building sandbox-specific client drivers. +config MLX5_IPSEC + bool "Mellanox Technologies IPsec Connect-X support" + depends on MLX5_CORE_EN + depends on XFRM_OFFLOAD + depends on INET_ESP_OFFLOAD || INET6_ESP_OFFLOAD + select MLX5_ACCEL + default n + help + Build IPsec support for the Connect-X family of network cards by Mellanox + Technologies. + Note: If you select this option, the mlx5_core driver will include + IPsec support for the Connect-X family. + config MLX5_EN_IPSEC bool "IPSec XFRM cryptography-offload accelaration" depends on MLX5_CORE_EN depends on XFRM_OFFLOAD depends on INET_ESP_OFFLOAD || INET6_ESP_OFFLOAD - depends on MLX5_FPGA_IPSEC + depends on MLX5_FPGA_IPSEC || MLX5_IPSEC default n help Build support for IPsec cryptography-offload accelaration in the NIC. @@ -150,7 +163,10 @@ config MLX5_FPGA_TLS bool "Mellanox Technologies TLS Innova support" depends on TLS_DEVICE depends on TLS=y || MLX5_CORE=m + depends on MLX5_CORE_EN depends on MLX5_FPGA + depends on XPS + select MLX5_EN_TLS default n help Build TLS support for the Innova family of network cards by Mellanox @@ -161,20 +177,19 @@ config MLX5_FPGA_TLS config MLX5_TLS bool "Mellanox Technologies TLS Connect-X support" - depends on MLX5_CORE_EN depends on TLS_DEVICE depends on TLS=y || MLX5_CORE=m + depends on MLX5_CORE_EN + depends on XPS select MLX5_ACCEL + select MLX5_EN_TLS default n help Build TLS support for the Connect-X family of network cards by Mellanox Technologies. config MLX5_EN_TLS - bool "TLS cryptography-offload accelaration" - depends on MLX5_CORE_EN - depends on MLX5_FPGA_TLS || MLX5_TLS - default y + bool help Build support for TLS cryptography-offload accelaration in the NIC. Note: Support for hardware with this capability needs to be selected diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index b61e47bc16e8..10e6886c96ba 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -35,7 +35,7 @@ mlx5_core-$(CONFIG_MLX5_EN_RXNFC) += en_fs_ethtool.o mlx5_core-$(CONFIG_MLX5_CORE_EN_DCB) += en_dcbnl.o en/port_buffer.o mlx5_core-$(CONFIG_PCI_HYPERV_INTERFACE) += en/hv_vhca_stats.o mlx5_core-$(CONFIG_MLX5_ESWITCH) += lag_mp.o lib/geneve.o lib/port_tun.o \ - en_rep.o en/rep/bond.o + en_rep.o en/rep/bond.o en/mod_hdr.o mlx5_core-$(CONFIG_MLX5_CLS_ACT) += en_tc.o en/rep/tc.o en/rep/neigh.o \ en/mapping.o esw/chains.o en/tc_tun.o \ en/tc_tun_vxlan.o en/tc_tun_gre.o en/tc_tun_geneve.o \ @@ -64,6 +64,7 @@ mlx5_core-$(CONFIG_MLX5_CORE_IPOIB) += ipoib/ipoib.o ipoib/ethtool.o ipoib/ipoib # # Accelerations & FPGA # +mlx5_core-$(CONFIG_MLX5_IPSEC) += accel/ipsec_offload.o mlx5_core-$(CONFIG_MLX5_FPGA_IPSEC) += fpga/ipsec.o mlx5_core-$(CONFIG_MLX5_FPGA_TLS) += fpga/tls.o mlx5_core-$(CONFIG_MLX5_ACCEL) += lib/crypto.o accel/tls.o accel/ipsec.o @@ -71,10 +72,11 @@ mlx5_core-$(CONFIG_MLX5_ACCEL) += lib/crypto.o accel/tls.o accel/ipsec.o mlx5_core-$(CONFIG_MLX5_FPGA) += fpga/cmd.o fpga/core.o fpga/conn.o fpga/sdk.o mlx5_core-$(CONFIG_MLX5_EN_IPSEC) += en_accel/ipsec.o en_accel/ipsec_rxtx.o \ - en_accel/ipsec_stats.o + en_accel/ipsec_stats.o en_accel/ipsec_fs.o mlx5_core-$(CONFIG_MLX5_EN_TLS) += en_accel/tls.o en_accel/tls_rxtx.o en_accel/tls_stats.o \ - en_accel/ktls.o en_accel/ktls_tx.o + en_accel/fs_tcp.o en_accel/ktls.o en_accel/ktls_txrx.o \ + en_accel/ktls_tx.o en_accel/ktls_rx.o mlx5_core-$(CONFIG_MLX5_SW_STEERING) += steering/dr_domain.o steering/dr_table.o \ steering/dr_matcher.o steering/dr_rule.o \ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec.c index 8a4985d8cbfe..09f5ce97af46 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec.c @@ -31,37 +31,88 @@ * */ -#ifdef CONFIG_MLX5_FPGA_IPSEC - #include <linux/mlx5/device.h> #include "accel/ipsec.h" #include "mlx5_core.h" #include "fpga/ipsec.h" +#include "accel/ipsec_offload.h" + +void mlx5_accel_ipsec_init(struct mlx5_core_dev *mdev) +{ + const struct mlx5_accel_ipsec_ops *ipsec_ops; + int err = 0; + + ipsec_ops = (mlx5_ipsec_offload_ops(mdev)) ? + mlx5_ipsec_offload_ops(mdev) : + mlx5_fpga_ipsec_ops(mdev); + + if (!ipsec_ops || !ipsec_ops->init) { + mlx5_core_dbg(mdev, "IPsec ops is not supported\n"); + return; + } + + err = ipsec_ops->init(mdev); + if (err) { + mlx5_core_warn_once(mdev, "Failed to start IPsec device, err = %d\n", err); + return; + } + + mdev->ipsec_ops = ipsec_ops; +} + +void mlx5_accel_ipsec_cleanup(struct mlx5_core_dev *mdev) +{ + const struct mlx5_accel_ipsec_ops *ipsec_ops = mdev->ipsec_ops; + + if (!ipsec_ops || !ipsec_ops->cleanup) + return; + + ipsec_ops->cleanup(mdev); +} u32 mlx5_accel_ipsec_device_caps(struct mlx5_core_dev *mdev) { - return mlx5_fpga_ipsec_device_caps(mdev); + const struct mlx5_accel_ipsec_ops *ipsec_ops = mdev->ipsec_ops; + + if (!ipsec_ops || !ipsec_ops->device_caps) + return 0; + + return ipsec_ops->device_caps(mdev); } EXPORT_SYMBOL_GPL(mlx5_accel_ipsec_device_caps); unsigned int mlx5_accel_ipsec_counters_count(struct mlx5_core_dev *mdev) { - return mlx5_fpga_ipsec_counters_count(mdev); + const struct mlx5_accel_ipsec_ops *ipsec_ops = mdev->ipsec_ops; + + if (!ipsec_ops || !ipsec_ops->counters_count) + return -EOPNOTSUPP; + + return ipsec_ops->counters_count(mdev); } int mlx5_accel_ipsec_counters_read(struct mlx5_core_dev *mdev, u64 *counters, unsigned int count) { - return mlx5_fpga_ipsec_counters_read(mdev, counters, count); + const struct mlx5_accel_ipsec_ops *ipsec_ops = mdev->ipsec_ops; + + if (!ipsec_ops || !ipsec_ops->counters_read) + return -EOPNOTSUPP; + + return ipsec_ops->counters_read(mdev, counters, count); } void *mlx5_accel_esp_create_hw_context(struct mlx5_core_dev *mdev, struct mlx5_accel_esp_xfrm *xfrm, u32 *sa_handle) { + const struct mlx5_accel_ipsec_ops *ipsec_ops = mdev->ipsec_ops; __be32 saddr[4] = {}, daddr[4] = {}; + if (!ipsec_ops || !ipsec_ops->create_hw_context) + return ERR_PTR(-EOPNOTSUPP); + if (!xfrm->attrs.is_ipv6) { saddr[3] = xfrm->attrs.saddr.a4; daddr[3] = xfrm->attrs.daddr.a4; @@ -70,29 +121,18 @@ void *mlx5_accel_esp_create_hw_context(struct mlx5_core_dev *mdev, memcpy(daddr, xfrm->attrs.daddr.a6, sizeof(daddr)); } - return mlx5_fpga_ipsec_create_sa_ctx(mdev, xfrm, saddr, - daddr, xfrm->attrs.spi, - xfrm->attrs.is_ipv6, sa_handle); + return ipsec_ops->create_hw_context(mdev, xfrm, saddr, daddr, xfrm->attrs.spi, + xfrm->attrs.is_ipv6, sa_handle); } -void mlx5_accel_esp_free_hw_context(void *context) +void mlx5_accel_esp_free_hw_context(struct mlx5_core_dev *mdev, void *context) { - mlx5_fpga_ipsec_delete_sa_ctx(context); -} + const struct mlx5_accel_ipsec_ops *ipsec_ops = mdev->ipsec_ops; -int mlx5_accel_ipsec_init(struct mlx5_core_dev *mdev) -{ - return mlx5_fpga_ipsec_init(mdev); -} - -void mlx5_accel_ipsec_build_fs_cmds(void) -{ - mlx5_fpga_ipsec_build_fs_cmds(); -} + if (!ipsec_ops || !ipsec_ops->free_hw_context) + return; -void mlx5_accel_ipsec_cleanup(struct mlx5_core_dev *mdev) -{ - mlx5_fpga_ipsec_cleanup(mdev); + ipsec_ops->free_hw_context(context); } struct mlx5_accel_esp_xfrm * @@ -100,9 +140,13 @@ mlx5_accel_esp_create_xfrm(struct mlx5_core_dev *mdev, const struct mlx5_accel_esp_xfrm_attrs *attrs, u32 flags) { + const struct mlx5_accel_ipsec_ops *ipsec_ops = mdev->ipsec_ops; struct mlx5_accel_esp_xfrm *xfrm; - xfrm = mlx5_fpga_esp_create_xfrm(mdev, attrs, flags); + if (!ipsec_ops || !ipsec_ops->esp_create_xfrm) + return ERR_PTR(-EOPNOTSUPP); + + xfrm = ipsec_ops->esp_create_xfrm(mdev, attrs, flags); if (IS_ERR(xfrm)) return xfrm; @@ -113,15 +157,23 @@ EXPORT_SYMBOL_GPL(mlx5_accel_esp_create_xfrm); void mlx5_accel_esp_destroy_xfrm(struct mlx5_accel_esp_xfrm *xfrm) { - mlx5_fpga_esp_destroy_xfrm(xfrm); + const struct mlx5_accel_ipsec_ops *ipsec_ops = xfrm->mdev->ipsec_ops; + + if (!ipsec_ops || !ipsec_ops->esp_destroy_xfrm) + return; + + ipsec_ops->esp_destroy_xfrm(xfrm); } EXPORT_SYMBOL_GPL(mlx5_accel_esp_destroy_xfrm); int mlx5_accel_esp_modify_xfrm(struct mlx5_accel_esp_xfrm *xfrm, const struct mlx5_accel_esp_xfrm_attrs *attrs) { - return mlx5_fpga_esp_modify_xfrm(xfrm, attrs); + const struct mlx5_accel_ipsec_ops *ipsec_ops = xfrm->mdev->ipsec_ops; + + if (!ipsec_ops || !ipsec_ops->esp_modify_xfrm) + return -EOPNOTSUPP; + + return ipsec_ops->esp_modify_xfrm(xfrm, attrs); } EXPORT_SYMBOL_GPL(mlx5_accel_esp_modify_xfrm); - -#endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec.h index e89747674712..fbb9c5415d53 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec.h @@ -37,7 +37,7 @@ #include <linux/mlx5/driver.h> #include <linux/mlx5/accel.h> -#ifdef CONFIG_MLX5_FPGA_IPSEC +#ifdef CONFIG_MLX5_ACCEL #define MLX5_IPSEC_DEV(mdev) (mlx5_accel_ipsec_device_caps(mdev) & \ MLX5_ACCEL_IPSEC_CAP_DEVICE) @@ -49,12 +49,30 @@ int mlx5_accel_ipsec_counters_read(struct mlx5_core_dev *mdev, u64 *counters, void *mlx5_accel_esp_create_hw_context(struct mlx5_core_dev *mdev, struct mlx5_accel_esp_xfrm *xfrm, u32 *sa_handle); -void mlx5_accel_esp_free_hw_context(void *context); +void mlx5_accel_esp_free_hw_context(struct mlx5_core_dev *mdev, void *context); -int mlx5_accel_ipsec_init(struct mlx5_core_dev *mdev); -void mlx5_accel_ipsec_build_fs_cmds(void); +void mlx5_accel_ipsec_init(struct mlx5_core_dev *mdev); void mlx5_accel_ipsec_cleanup(struct mlx5_core_dev *mdev); +struct mlx5_accel_ipsec_ops { + u32 (*device_caps)(struct mlx5_core_dev *mdev); + unsigned int (*counters_count)(struct mlx5_core_dev *mdev); + int (*counters_read)(struct mlx5_core_dev *mdev, u64 *counters, unsigned int count); + void* (*create_hw_context)(struct mlx5_core_dev *mdev, + struct mlx5_accel_esp_xfrm *xfrm, + const __be32 saddr[4], const __be32 daddr[4], + const __be32 spi, bool is_ipv6, u32 *sa_handle); + void (*free_hw_context)(void *context); + int (*init)(struct mlx5_core_dev *mdev); + void (*cleanup)(struct mlx5_core_dev *mdev); + struct mlx5_accel_esp_xfrm* (*esp_create_xfrm)(struct mlx5_core_dev *mdev, + const struct mlx5_accel_esp_xfrm_attrs *attrs, + u32 flags); + int (*esp_modify_xfrm)(struct mlx5_accel_esp_xfrm *xfrm, + const struct mlx5_accel_esp_xfrm_attrs *attrs); + void (*esp_destroy_xfrm)(struct mlx5_accel_esp_xfrm *xfrm); +}; + #else #define MLX5_IPSEC_DEV(mdev) false @@ -67,23 +85,12 @@ mlx5_accel_esp_create_hw_context(struct mlx5_core_dev *mdev, return NULL; } -static inline void mlx5_accel_esp_free_hw_context(void *context) -{ -} - -static inline int mlx5_accel_ipsec_init(struct mlx5_core_dev *mdev) -{ - return 0; -} +static inline void mlx5_accel_esp_free_hw_context(struct mlx5_core_dev *mdev, void *context) {} -static inline void mlx5_accel_ipsec_build_fs_cmds(void) -{ -} +static inline void mlx5_accel_ipsec_init(struct mlx5_core_dev *mdev) {} -static inline void mlx5_accel_ipsec_cleanup(struct mlx5_core_dev *mdev) -{ -} +static inline void mlx5_accel_ipsec_cleanup(struct mlx5_core_dev *mdev) {} -#endif +#endif /* CONFIG_MLX5_ACCEL */ #endif /* __MLX5_ACCEL_IPSEC_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec_offload.c b/drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec_offload.c new file mode 100644 index 000000000000..2f13a250aab3 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec_offload.c @@ -0,0 +1,385 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIBt +/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ + +#include "mlx5_core.h" +#include "ipsec_offload.h" +#include "lib/mlx5.h" +#include "en_accel/ipsec_fs.h" + +#define MLX5_IPSEC_DEV_BASIC_CAPS (MLX5_ACCEL_IPSEC_CAP_DEVICE | MLX5_ACCEL_IPSEC_CAP_IPV6 | \ + MLX5_ACCEL_IPSEC_CAP_LSO) + +struct mlx5_ipsec_sa_ctx { + struct rhash_head hash; + u32 enc_key_id; + u32 ipsec_obj_id; + /* hw ctx */ + struct mlx5_core_dev *dev; + struct mlx5_ipsec_esp_xfrm *mxfrm; +}; + +struct mlx5_ipsec_esp_xfrm { + /* reference counter of SA ctx */ + struct mlx5_ipsec_sa_ctx *sa_ctx; + struct mutex lock; /* protects mlx5_ipsec_esp_xfrm */ + struct mlx5_accel_esp_xfrm accel_xfrm; +}; + +static u32 mlx5_ipsec_offload_device_caps(struct mlx5_core_dev *mdev) +{ + u32 caps = MLX5_IPSEC_DEV_BASIC_CAPS; + + if (!mlx5_is_ipsec_device(mdev)) + return 0; + + if (!MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ipsec_encrypt) || + !MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ipsec_decrypt)) + return 0; + + if (MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_encrypt) && + MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_decrypt)) + caps |= MLX5_ACCEL_IPSEC_CAP_ESP; + + if (MLX5_CAP_IPSEC(mdev, ipsec_esn)) { + caps |= MLX5_ACCEL_IPSEC_CAP_ESN; + caps |= MLX5_ACCEL_IPSEC_CAP_TX_IV_IS_ESN; + } + + /* We can accommodate up to 2^24 different IPsec objects + * because we use up to 24 bit in flow table metadata + * to hold the IPsec Object unique handle. + */ + WARN_ON_ONCE(MLX5_CAP_IPSEC(mdev, log_max_ipsec_offload) > 24); + return caps; +} + +static int +mlx5_ipsec_offload_esp_validate_xfrm_attrs(struct mlx5_core_dev *mdev, + const struct mlx5_accel_esp_xfrm_attrs *attrs) +{ + if (attrs->replay_type != MLX5_ACCEL_ESP_REPLAY_NONE) { + mlx5_core_err(mdev, "Cannot offload xfrm states with anti replay (replay_type = %d)\n", + attrs->replay_type); + return -EOPNOTSUPP; + } + + if (attrs->keymat_type != MLX5_ACCEL_ESP_KEYMAT_AES_GCM) { + mlx5_core_err(mdev, "Only aes gcm keymat is supported (keymat_type = %d)\n", + attrs->keymat_type); + return -EOPNOTSUPP; + } + + if (attrs->keymat.aes_gcm.iv_algo != + MLX5_ACCEL_ESP_AES_GCM_IV_ALGO_SEQ) { + mlx5_core_err(mdev, "Only iv sequence algo is supported (iv_algo = %d)\n", + attrs->keymat.aes_gcm.iv_algo); + return -EOPNOTSUPP; + } + + if (attrs->keymat.aes_gcm.key_len != 128 && + attrs->keymat.aes_gcm.key_len != 256) { + mlx5_core_err(mdev, "Cannot offload xfrm states with key length other than 128/256 bit (key length = %d)\n", + attrs->keymat.aes_gcm.key_len); + return -EOPNOTSUPP; + } + + if ((attrs->flags & MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED) && + !MLX5_CAP_IPSEC(mdev, ipsec_esn)) { + mlx5_core_err(mdev, "Cannot offload xfrm states with ESN triggered\n"); + return -EOPNOTSUPP; + } + + return 0; +} + +static struct mlx5_accel_esp_xfrm * +mlx5_ipsec_offload_esp_create_xfrm(struct mlx5_core_dev *mdev, + const struct mlx5_accel_esp_xfrm_attrs *attrs, + u32 flags) +{ + struct mlx5_ipsec_esp_xfrm *mxfrm; + int err = 0; + + err = mlx5_ipsec_offload_esp_validate_xfrm_attrs(mdev, attrs); + if (err) + return ERR_PTR(err); + + mxfrm = kzalloc(sizeof(*mxfrm), GFP_KERNEL); + if (!mxfrm) + return ERR_PTR(-ENOMEM); + + mutex_init(&mxfrm->lock); + memcpy(&mxfrm->accel_xfrm.attrs, attrs, + sizeof(mxfrm->accel_xfrm.attrs)); + + return &mxfrm->accel_xfrm; +} + +static void mlx5_ipsec_offload_esp_destroy_xfrm(struct mlx5_accel_esp_xfrm *xfrm) +{ + struct mlx5_ipsec_esp_xfrm *mxfrm = container_of(xfrm, struct mlx5_ipsec_esp_xfrm, + accel_xfrm); + + /* assuming no sa_ctx are connected to this xfrm_ctx */ + WARN_ON(mxfrm->sa_ctx); + kfree(mxfrm); +} + +struct mlx5_ipsec_obj_attrs { + const struct aes_gcm_keymat *aes_gcm; + u32 accel_flags; + u32 esn_msb; + u32 enc_key_id; +}; + +static int mlx5_create_ipsec_obj(struct mlx5_core_dev *mdev, + struct mlx5_ipsec_obj_attrs *attrs, + u32 *ipsec_id) +{ + const struct aes_gcm_keymat *aes_gcm = attrs->aes_gcm; + u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; + u32 in[MLX5_ST_SZ_DW(create_ipsec_obj_in)] = {}; + void *obj, *salt_p, *salt_iv_p; + int err; + + obj = MLX5_ADDR_OF(create_ipsec_obj_in, in, ipsec_object); + + /* salt and seq_iv */ + salt_p = MLX5_ADDR_OF(ipsec_obj, obj, salt); + memcpy(salt_p, &aes_gcm->salt, sizeof(aes_gcm->salt)); + + switch (aes_gcm->icv_len) { + case 64: + MLX5_SET(ipsec_obj, obj, icv_length, + MLX5_IPSEC_OBJECT_ICV_LEN_8B); + break; + case 96: + MLX5_SET(ipsec_obj, obj, icv_length, + MLX5_IPSEC_OBJECT_ICV_LEN_12B); + break; + case 128: + MLX5_SET(ipsec_obj, obj, icv_length, + MLX5_IPSEC_OBJECT_ICV_LEN_16B); + break; + default: + return -EINVAL; + } + salt_iv_p = MLX5_ADDR_OF(ipsec_obj, obj, implicit_iv); + memcpy(salt_iv_p, &aes_gcm->seq_iv, sizeof(aes_gcm->seq_iv)); + /* esn */ + if (attrs->accel_flags & MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED) { + MLX5_SET(ipsec_obj, obj, esn_en, 1); + MLX5_SET(ipsec_obj, obj, esn_msb, attrs->esn_msb); + if (attrs->accel_flags & MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP) + MLX5_SET(ipsec_obj, obj, esn_overlap, 1); + } + + MLX5_SET(ipsec_obj, obj, dekn, attrs->enc_key_id); + + /* general object fields set */ + MLX5_SET(general_obj_in_cmd_hdr, in, opcode, + MLX5_CMD_OP_CREATE_GENERAL_OBJECT); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, + MLX5_GENERAL_OBJECT_TYPES_IPSEC); + + err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); + if (!err) + *ipsec_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); + + return err; +} + +static void mlx5_destroy_ipsec_obj(struct mlx5_core_dev *mdev, u32 ipsec_id) +{ + u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {}; + u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; + + MLX5_SET(general_obj_in_cmd_hdr, in, opcode, + MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, + MLX5_GENERAL_OBJECT_TYPES_IPSEC); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, ipsec_id); + + mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); +} + +static void *mlx5_ipsec_offload_create_sa_ctx(struct mlx5_core_dev *mdev, + struct mlx5_accel_esp_xfrm *accel_xfrm, + const __be32 saddr[4], const __be32 daddr[4], + const __be32 spi, bool is_ipv6, u32 *hw_handle) +{ + struct mlx5_accel_esp_xfrm_attrs *xfrm_attrs = &accel_xfrm->attrs; + struct aes_gcm_keymat *aes_gcm = &xfrm_attrs->keymat.aes_gcm; + struct mlx5_ipsec_obj_attrs ipsec_attrs = {}; + struct mlx5_ipsec_esp_xfrm *mxfrm; + struct mlx5_ipsec_sa_ctx *sa_ctx; + int err; + + /* alloc SA context */ + sa_ctx = kzalloc(sizeof(*sa_ctx), GFP_KERNEL); + if (!sa_ctx) + return ERR_PTR(-ENOMEM); + + sa_ctx->dev = mdev; + + mxfrm = container_of(accel_xfrm, struct mlx5_ipsec_esp_xfrm, accel_xfrm); + mutex_lock(&mxfrm->lock); + sa_ctx->mxfrm = mxfrm; + + /* key */ + err = mlx5_create_encryption_key(mdev, aes_gcm->aes_key, + aes_gcm->key_len / BITS_PER_BYTE, + MLX5_ACCEL_OBJ_IPSEC_KEY, + &sa_ctx->enc_key_id); + if (err) { + mlx5_core_dbg(mdev, "Failed to create encryption key (err = %d)\n", err); + goto err_sa_ctx; + } + + ipsec_attrs.aes_gcm = aes_gcm; + ipsec_attrs.accel_flags = accel_xfrm->attrs.flags; + ipsec_attrs.esn_msb = accel_xfrm->attrs.esn; + ipsec_attrs.enc_key_id = sa_ctx->enc_key_id; + err = mlx5_create_ipsec_obj(mdev, &ipsec_attrs, + &sa_ctx->ipsec_obj_id); + if (err) { + mlx5_core_dbg(mdev, "Failed to create IPsec object (err = %d)\n", err); + goto err_enc_key; + } + + *hw_handle = sa_ctx->ipsec_obj_id; + mxfrm->sa_ctx = sa_ctx; + mutex_unlock(&mxfrm->lock); + + return sa_ctx; + +err_enc_key: + mlx5_destroy_encryption_key(mdev, sa_ctx->enc_key_id); +err_sa_ctx: + mutex_unlock(&mxfrm->lock); + kfree(sa_ctx); + return ERR_PTR(err); +} + +static void mlx5_ipsec_offload_delete_sa_ctx(void *context) +{ + struct mlx5_ipsec_sa_ctx *sa_ctx = (struct mlx5_ipsec_sa_ctx *)context; + struct mlx5_ipsec_esp_xfrm *mxfrm = sa_ctx->mxfrm; + + mutex_lock(&mxfrm->lock); + mlx5_destroy_ipsec_obj(sa_ctx->dev, sa_ctx->ipsec_obj_id); + mlx5_destroy_encryption_key(sa_ctx->dev, sa_ctx->enc_key_id); + kfree(sa_ctx); + mxfrm->sa_ctx = NULL; + mutex_unlock(&mxfrm->lock); +} + +static int mlx5_ipsec_offload_init(struct mlx5_core_dev *mdev) +{ + return 0; +} + +static int mlx5_modify_ipsec_obj(struct mlx5_core_dev *mdev, + struct mlx5_ipsec_obj_attrs *attrs, + u32 ipsec_id) +{ + u32 in[MLX5_ST_SZ_DW(modify_ipsec_obj_in)] = {}; + u32 out[MLX5_ST_SZ_DW(query_ipsec_obj_out)]; + u64 modify_field_select = 0; + u64 general_obj_types; + void *obj; + int err; + + if (!(attrs->accel_flags & MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED)) + return 0; + + general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types); + if (!(general_obj_types & MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC)) + return -EINVAL; + + /* general object fields set */ + MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_IPSEC); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, ipsec_id); + err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); + if (err) { + mlx5_core_err(mdev, "Query IPsec object failed (Object id %d), err = %d\n", + ipsec_id, err); + return err; + } + + obj = MLX5_ADDR_OF(query_ipsec_obj_out, out, ipsec_object); + modify_field_select = MLX5_GET64(ipsec_obj, obj, modify_field_select); + + /* esn */ + if (!(modify_field_select & MLX5_MODIFY_IPSEC_BITMASK_ESN_OVERLAP) || + !(modify_field_select & MLX5_MODIFY_IPSEC_BITMASK_ESN_MSB)) + return -EOPNOTSUPP; + + obj = MLX5_ADDR_OF(modify_ipsec_obj_in, in, ipsec_object); + MLX5_SET(ipsec_obj, obj, esn_msb, attrs->esn_msb); + if (attrs->accel_flags & MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP) + MLX5_SET(ipsec_obj, obj, esn_overlap, 1); + + /* general object fields set */ + MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT); + + return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); +} + +static int mlx5_ipsec_offload_esp_modify_xfrm(struct mlx5_accel_esp_xfrm *xfrm, + const struct mlx5_accel_esp_xfrm_attrs *attrs) +{ + struct mlx5_ipsec_obj_attrs ipsec_attrs = {}; + struct mlx5_core_dev *mdev = xfrm->mdev; + struct mlx5_ipsec_esp_xfrm *mxfrm; + + int err = 0; + + if (!memcmp(&xfrm->attrs, attrs, sizeof(xfrm->attrs))) + return 0; + + if (mlx5_ipsec_offload_esp_validate_xfrm_attrs(mdev, attrs)) + return -EOPNOTSUPP; + + mxfrm = container_of(xfrm, struct mlx5_ipsec_esp_xfrm, accel_xfrm); + + mutex_lock(&mxfrm->lock); + + if (!mxfrm->sa_ctx) + /* Not bound xfrm, change only sw attrs */ + goto change_sw_xfrm_attrs; + + /* need to add find and replace in ipsec_rhash_sa the sa_ctx */ + /* modify device with new hw_sa */ + ipsec_attrs.accel_flags = attrs->flags; + ipsec_attrs.esn_msb = attrs->esn; + err = mlx5_modify_ipsec_obj(mdev, + &ipsec_attrs, + mxfrm->sa_ctx->ipsec_obj_id); + +change_sw_xfrm_attrs: + if (!err) + memcpy(&xfrm->attrs, attrs, sizeof(xfrm->attrs)); + + mutex_unlock(&mxfrm->lock); + return err; +} + +static const struct mlx5_accel_ipsec_ops ipsec_offload_ops = { + .device_caps = mlx5_ipsec_offload_device_caps, + .create_hw_context = mlx5_ipsec_offload_create_sa_ctx, + .free_hw_context = mlx5_ipsec_offload_delete_sa_ctx, + .init = mlx5_ipsec_offload_init, + .esp_create_xfrm = mlx5_ipsec_offload_esp_create_xfrm, + .esp_destroy_xfrm = mlx5_ipsec_offload_esp_destroy_xfrm, + .esp_modify_xfrm = mlx5_ipsec_offload_esp_modify_xfrm, +}; + +const struct mlx5_accel_ipsec_ops *mlx5_ipsec_offload_ops(struct mlx5_core_dev *mdev) +{ + if (!mlx5_ipsec_offload_device_caps(mdev)) + return NULL; + + return &ipsec_offload_ops; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec_offload.h b/drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec_offload.h new file mode 100644 index 000000000000..970c66d19c1d --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec_offload.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ + +#ifndef __MLX5_IPSEC_OFFLOAD_H__ +#define __MLX5_IPSEC_OFFLOAD_H__ + +#include <linux/mlx5/driver.h> +#include "accel/ipsec.h" + +#ifdef CONFIG_MLX5_IPSEC + +const struct mlx5_accel_ipsec_ops *mlx5_ipsec_offload_ops(struct mlx5_core_dev *mdev); +static inline bool mlx5_is_ipsec_device(struct mlx5_core_dev *mdev) +{ + if (!MLX5_CAP_GEN(mdev, ipsec_offload)) + return false; + + if (!MLX5_CAP_GEN(mdev, log_max_dek)) + return false; + + if (!(MLX5_CAP_GEN_64(mdev, general_obj_types) & + MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC)) + return false; + + return MLX5_CAP_IPSEC(mdev, ipsec_crypto_offload) && + MLX5_CAP_ETH(mdev, insert_trailer); +} + +#else +static inline const struct mlx5_accel_ipsec_ops * +mlx5_ipsec_offload_ops(struct mlx5_core_dev *mdev) { return NULL; } +static inline bool mlx5_is_ipsec_device(struct mlx5_core_dev *mdev) +{ + return false; +} + +#endif /* CONFIG_MLX5_IPSEC */ +#endif /* __MLX5_IPSEC_OFFLOAD_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/accel/tls.c b/drivers/net/ethernet/mellanox/mlx5/core/accel/tls.c index cbf3d76c05a8..6c2b86a26863 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/accel/tls.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/accel/tls.c @@ -113,7 +113,9 @@ int mlx5_ktls_create_key(struct mlx5_core_dev *mdev, return -EINVAL; } - return mlx5_create_encryption_key(mdev, key, sz_bytes, p_key_id); + return mlx5_create_encryption_key(mdev, key, sz_bytes, + MLX5_ACCEL_OBJ_TLS_KEY, + p_key_id); } void mlx5_ktls_destroy_key(struct mlx5_core_dev *mdev, u32 key_id) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/accel/tls.h b/drivers/net/ethernet/mellanox/mlx5/core/accel/tls.h index aefea467f7b3..fd874f0c380a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/accel/tls.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/accel/tls.h @@ -43,9 +43,20 @@ int mlx5_ktls_create_key(struct mlx5_core_dev *mdev, u32 *p_key_id); void mlx5_ktls_destroy_key(struct mlx5_core_dev *mdev, u32 key_id); +static inline bool mlx5_accel_is_ktls_tx(struct mlx5_core_dev *mdev) +{ + return MLX5_CAP_GEN(mdev, tls_tx); +} + +static inline bool mlx5_accel_is_ktls_rx(struct mlx5_core_dev *mdev) +{ + return MLX5_CAP_GEN(mdev, tls_rx); +} + static inline bool mlx5_accel_is_ktls_device(struct mlx5_core_dev *mdev) { - if (!MLX5_CAP_GEN(mdev, tls_tx)) + if (!mlx5_accel_is_ktls_tx(mdev) && + !mlx5_accel_is_ktls_rx(mdev)) return false; if (!MLX5_CAP_GEN(mdev, log_max_dek)) @@ -67,6 +78,12 @@ static inline bool mlx5e_ktls_type_check(struct mlx5_core_dev *mdev, return false; } #else +static inline bool mlx5_accel_is_ktls_tx(struct mlx5_core_dev *mdev) +{ return false; } + +static inline bool mlx5_accel_is_ktls_rx(struct mlx5_core_dev *mdev) +{ return false; } + static inline int mlx5_ktls_create_key(struct mlx5_core_dev *mdev, struct tls_crypto_info *crypto_info, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/alloc.c b/drivers/net/ethernet/mellanox/mlx5/core/alloc.c index 42198e64a7f4..8db4b5f0f963 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/alloc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/alloc.c @@ -299,11 +299,18 @@ void mlx5_fill_page_array(struct mlx5_frag_buf *buf, __be64 *pas) } EXPORT_SYMBOL_GPL(mlx5_fill_page_array); -void mlx5_fill_page_frag_array(struct mlx5_frag_buf *buf, __be64 *pas) +void mlx5_fill_page_frag_array_perm(struct mlx5_frag_buf *buf, __be64 *pas, u8 perm) { int i; + WARN_ON(perm & 0xfc); for (i = 0; i < buf->npages; i++) - pas[i] = cpu_to_be64(buf->frags[i].map); + pas[i] = cpu_to_be64(buf->frags[i].map | perm); +} +EXPORT_SYMBOL_GPL(mlx5_fill_page_frag_array_perm); + +void mlx5_fill_page_frag_array(struct mlx5_frag_buf *buf, __be64 *pas) +{ + mlx5_fill_page_frag_array_perm(buf, pas, 0); } EXPORT_SYMBOL_GPL(mlx5_fill_page_frag_array); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c index a99fe4b02b9b..c709e9a385f6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c @@ -113,6 +113,8 @@ static const struct devlink_ops mlx5_devlink_ops = { .eswitch_inline_mode_get = mlx5_devlink_eswitch_inline_mode_get, .eswitch_encap_mode_set = mlx5_devlink_eswitch_encap_mode_set, .eswitch_encap_mode_get = mlx5_devlink_eswitch_encap_mode_get, + .port_function_hw_addr_get = mlx5_devlink_port_function_hw_addr_get, + .port_function_hw_addr_set = mlx5_devlink_port_function_hw_addr_set, #endif .flash_update = mlx5_devlink_flash_update, .info_get = mlx5_devlink_info_get, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c b/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c index a7551274be58..ad3594c4afcb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c @@ -676,7 +676,7 @@ static void mlx5_fw_tracer_handle_traces(struct work_struct *work) block_count = tracer->buff.size / TRACER_BLOCK_SIZE_BYTE; start_offset = tracer->buff.consumer_index * TRACER_BLOCK_SIZE_BYTE; - /* Copy the block to local buffer to avoid HW override while being processed*/ + /* Copy the block to local buffer to avoid HW override while being processed */ memcpy(tmp_trace_block, tracer->buff.log_buf + start_offset, TRACER_BLOCK_SIZE_BYTE); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.c b/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.c index 17ab7efe693d..4924a5658853 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.c @@ -23,6 +23,9 @@ static const char *const mlx5_rsc_sgmt_name[] = { MLX5_SGMT_STR_ASSING(SX_SLICE_ALL), MLX5_SGMT_STR_ASSING(RDB), MLX5_SGMT_STR_ASSING(RX_SLICE_ALL), + MLX5_SGMT_STR_ASSING(PRM_QUERY_QP), + MLX5_SGMT_STR_ASSING(PRM_QUERY_CQ), + MLX5_SGMT_STR_ASSING(PRM_QUERY_MKEY), }; struct mlx5_rsc_dump { @@ -130,11 +133,13 @@ struct mlx5_rsc_dump_cmd *mlx5_rsc_dump_cmd_create(struct mlx5_core_dev *dev, cmd->mem_size = key->size; return cmd; } +EXPORT_SYMBOL(mlx5_rsc_dump_cmd_create); void mlx5_rsc_dump_cmd_destroy(struct mlx5_rsc_dump_cmd *cmd) { kfree(cmd); } +EXPORT_SYMBOL(mlx5_rsc_dump_cmd_destroy); int mlx5_rsc_dump_next(struct mlx5_core_dev *dev, struct mlx5_rsc_dump_cmd *cmd, struct page *page, int *size) @@ -155,6 +160,7 @@ int mlx5_rsc_dump_next(struct mlx5_core_dev *dev, struct mlx5_rsc_dump_cmd *cmd, return more_dump; } +EXPORT_SYMBOL(mlx5_rsc_dump_next); #define MLX5_RSC_DUMP_MENU_SEGMENT 0xffff static int mlx5_rsc_dump_menu(struct mlx5_core_dev *dev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.h b/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.h index 148270073e71..64c4956db6d2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.h @@ -4,41 +4,10 @@ #ifndef __MLX5_RSC_DUMP_H #define __MLX5_RSC_DUMP_H +#include <linux/mlx5/rsc_dump.h> #include <linux/mlx5/driver.h> #include "mlx5_core.h" -enum mlx5_sgmt_type { - MLX5_SGMT_TYPE_HW_CQPC, - MLX5_SGMT_TYPE_HW_SQPC, - MLX5_SGMT_TYPE_HW_RQPC, - MLX5_SGMT_TYPE_FULL_SRQC, - MLX5_SGMT_TYPE_FULL_CQC, - MLX5_SGMT_TYPE_FULL_EQC, - MLX5_SGMT_TYPE_FULL_QPC, - MLX5_SGMT_TYPE_SND_BUFF, - MLX5_SGMT_TYPE_RCV_BUFF, - MLX5_SGMT_TYPE_SRQ_BUFF, - MLX5_SGMT_TYPE_CQ_BUFF, - MLX5_SGMT_TYPE_EQ_BUFF, - MLX5_SGMT_TYPE_SX_SLICE, - MLX5_SGMT_TYPE_SX_SLICE_ALL, - MLX5_SGMT_TYPE_RDB, - MLX5_SGMT_TYPE_RX_SLICE_ALL, - MLX5_SGMT_TYPE_MENU, - MLX5_SGMT_TYPE_TERMINATE, - - MLX5_SGMT_TYPE_NUM, /* Keep last */ -}; - -struct mlx5_rsc_key { - enum mlx5_sgmt_type rsc; - int index1; - int index2; - int num_of_obj1; - int num_of_obj2; - int size; -}; - #define MLX5_RSC_DUMP_ALL 0xFFFF struct mlx5_rsc_dump_cmd; struct mlx5_rsc_dump; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 842db20493df..0cc2080fd847 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -45,6 +45,7 @@ #include <linux/mlx5/transobj.h> #include <linux/mlx5/fs.h> #include <linux/rhashtable.h> +#include <net/udp_tunnel.h> #include <net/switchdev.h> #include <net/xdp.h> #include <linux/dim.h> @@ -191,13 +192,8 @@ static inline int mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev) struct mlx5e_tx_wqe { struct mlx5_wqe_ctrl_seg ctrl; - union { - struct { - struct mlx5_wqe_eth_seg eth; - struct mlx5_wqe_data_seg data[0]; - }; - u8 tls_progress_params_ctx[0]; - }; + struct mlx5_wqe_eth_seg eth; + struct mlx5_wqe_data_seg data[0]; }; struct mlx5e_rx_wqe_ll { @@ -213,10 +209,7 @@ struct mlx5e_umr_wqe { struct mlx5_wqe_ctrl_seg ctrl; struct mlx5_wqe_umr_ctrl_seg uctrl; struct mlx5_mkey_seg mkc; - union { - struct mlx5_mtt inline_mtts[0]; - u8 tls_static_params_ctx[0]; - }; + struct mlx5_mtt inline_mtts[0]; }; extern const char mlx5e_self_tests[][ETH_GSTRING_LEN]; @@ -271,6 +264,7 @@ enum { MLX5E_RQ_STATE_AM, MLX5E_RQ_STATE_NO_CSUM_COMPLETE, MLX5E_RQ_STATE_CSUM_FULL, /* cqe_csum_full hw bit is set */ + MLX5E_RQ_STATE_FPGA_TLS, /* FPGA TLS enabled */ }; struct mlx5e_cq { @@ -537,6 +531,8 @@ typedef struct sk_buff * typedef bool (*mlx5e_fp_post_rx_wqes)(struct mlx5e_rq *rq); typedef void (*mlx5e_fp_dealloc_wqe)(struct mlx5e_rq*, u16); +int mlx5e_rq_set_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params, bool xsk); + enum mlx5e_rq_flag { MLX5E_RQ_FLAG_XDP_XMIT, MLX5E_RQ_FLAG_XDP_REDIRECT, @@ -651,9 +647,11 @@ struct mlx5e_channel { /* AF_XDP zero-copy */ struct mlx5e_rq xskrq; struct mlx5e_xdpsq xsksq; - struct mlx5e_icosq xskicosq; - /* xskicosq can be accessed from any CPU - the spinlock protects it. */ - spinlock_t xskicosq_lock; + + /* Async ICOSQ */ + struct mlx5e_icosq async_icosq; + /* async_icosq can be accessed from any CPU - the spinlock protects it. */ + spinlock_t async_icosq_lock; /* data path - accessed per napi poll */ struct irq_desc *irq_desc; @@ -795,6 +793,7 @@ struct mlx5e_priv { u16 drop_rq_q_counter; struct notifier_block events_nb; + struct udp_tunnel_nic_info nic_info; #ifdef CONFIG_MLX5_CORE_EN_DCB struct mlx5e_dcbx dcbx; #endif @@ -817,6 +816,13 @@ struct mlx5e_priv { struct mlx5e_scratchpad scratchpad; }; +struct mlx5e_rx_handlers { + mlx5e_fp_handle_rx_cqe handle_rx_cqe; + mlx5e_fp_handle_rx_cqe handle_rx_cqe_mpwqe; +}; + +extern const struct mlx5e_rx_handlers mlx5e_rx_handlers_nic; + struct mlx5e_profile { int (*init)(struct mlx5_core_dev *mdev, struct net_device *netdev, @@ -833,78 +839,17 @@ struct mlx5e_profile { void (*update_carrier)(struct mlx5e_priv *priv); unsigned int (*stats_grps_num)(struct mlx5e_priv *priv); mlx5e_stats_grp_t *stats_grps; - struct { - mlx5e_fp_handle_rx_cqe handle_rx_cqe; - mlx5e_fp_handle_rx_cqe handle_rx_cqe_mpwqe; - } rx_handlers; + const struct mlx5e_rx_handlers *rx_handlers; int max_tc; u8 rq_groups; }; void mlx5e_build_ptys2ethtool_map(void); -u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb, - struct net_device *sb_dev); -netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev); -void mlx5e_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb, - struct mlx5e_tx_wqe *wqe, u16 pi, bool xmit_more); - -void mlx5e_trigger_irq(struct mlx5e_icosq *sq); -void mlx5e_completion_event(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe); -void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, enum mlx5_event event); -int mlx5e_napi_poll(struct napi_struct *napi, int budget); -bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget); -int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget); -void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq); - -static inline u32 mlx5e_rqwq_get_size(struct mlx5e_rq *rq) -{ - switch (rq->wq_type) { - case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: - return mlx5_wq_ll_get_size(&rq->mpwqe.wq); - default: - return mlx5_wq_cyc_get_size(&rq->wqe.wq); - } -} - -static inline u32 mlx5e_rqwq_get_cur_sz(struct mlx5e_rq *rq) -{ - switch (rq->wq_type) { - case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: - return rq->mpwqe.wq.cur_sz; - default: - return rq->wqe.wq.cur_sz; - } -} - bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev); bool mlx5e_striding_rq_possible(struct mlx5_core_dev *mdev, struct mlx5e_params *params); -void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info); -void mlx5e_page_release_dynamic(struct mlx5e_rq *rq, - struct mlx5e_dma_info *dma_info, - bool recycle); -void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe); -void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe); -bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq); -int mlx5e_poll_ico_cq(struct mlx5e_cq *cq); -bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq); -void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix); -void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix); -struct sk_buff * -mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, - u16 cqe_bcnt, u32 head_offset, u32 page_idx); -struct sk_buff * -mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, - u16 cqe_bcnt, u32 head_offset, u32 page_idx); -struct sk_buff * -mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, - struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt); -struct sk_buff * -mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, - struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt); - void mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats); void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s); @@ -1007,8 +952,6 @@ void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev, int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state); void mlx5e_activate_rq(struct mlx5e_rq *rq); void mlx5e_deactivate_rq(struct mlx5e_rq *rq); -void mlx5e_free_rx_descs(struct mlx5e_rq *rq); -void mlx5e_free_rx_in_progress_descs(struct mlx5e_rq *rq); void mlx5e_activate_icosq(struct mlx5e_icosq *icosq); void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq); @@ -1033,6 +976,7 @@ int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev); void mlx5e_destroy_mdev_resources(struct mlx5_core_dev *mdev); int mlx5e_refresh_tirs(struct mlx5e_priv *priv, bool enable_uc_lb, bool enable_mc_lb); +void mlx5e_mkey_set_relaxed_ordering(struct mlx5_core_dev *mdev, void *mkc); /* common netdev helpers */ void mlx5e_create_q_counters(struct mlx5e_priv *priv); @@ -1070,6 +1014,7 @@ int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv); int mlx5e_set_dev_port_mtu_ctx(struct mlx5e_priv *priv, void *context); int mlx5e_change_mtu(struct net_device *netdev, int new_mtu, mlx5e_fp_preactivate preactivate); +void mlx5e_vxlan_set_netdev_info(struct mlx5e_priv *priv); /* ethtool helpers */ void mlx5e_ethtool_get_drvinfo(struct mlx5e_priv *priv, @@ -1138,8 +1083,6 @@ void mlx5e_build_rss_params(struct mlx5e_rss_params *rss_params, void mlx5e_rx_dim_work(struct work_struct *work); void mlx5e_tx_dim_work(struct work_struct *work); -void mlx5e_add_vxlan_port(struct net_device *netdev, struct udp_tunnel_info *ti); -void mlx5e_del_vxlan_port(struct net_device *netdev, struct udp_tunnel_info *ti); netdev_features_t mlx5e_features_check(struct sk_buff *skb, struct net_device *netdev, netdev_features_t features); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c index f8b2de4b04be..a69c62d72d16 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c @@ -6,17 +6,16 @@ int mlx5e_devlink_port_register(struct mlx5e_priv *priv) { struct devlink *devlink = priv_to_devlink(priv->mdev); + struct devlink_port_attrs attrs = {}; - if (mlx5_core_is_pf(priv->mdev)) - devlink_port_attrs_set(&priv->dl_port, - DEVLINK_PORT_FLAVOUR_PHYSICAL, - PCI_FUNC(priv->mdev->pdev->devfn), - false, 0, - NULL, 0); - else - devlink_port_attrs_set(&priv->dl_port, - DEVLINK_PORT_FLAVOUR_VIRTUAL, - 0, false, 0, NULL, 0); + if (mlx5_core_is_pf(priv->mdev)) { + attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL; + attrs.phys.port_number = PCI_FUNC(priv->mdev->pdev->devfn); + } else { + attrs.flavour = DEVLINK_PORT_FLAVOUR_VIRTUAL; + } + + devlink_port_attrs_set(&priv->dl_port, &attrs); return devlink_port_register(devlink, &priv->dl_port, 1); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h index 0416f7712109..6fdcd5e69476 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h @@ -4,6 +4,8 @@ #ifndef __MLX5E_FLOW_STEER_H__ #define __MLX5E_FLOW_STEER_H__ +#include "mod_hdr.h" + enum { MLX5E_TC_FT_LEVEL = 0, MLX5E_TC_TTC_FT_LEVEL, @@ -105,11 +107,16 @@ enum mlx5e_tunnel_types { bool mlx5e_tunnel_inner_ft_supported(struct mlx5_core_dev *mdev); +struct mlx5e_ttc_rule { + struct mlx5_flow_handle *rule; + struct mlx5_flow_destination default_dest; +}; + /* L3/L4 traffic type classifier */ struct mlx5e_ttc_table { - struct mlx5e_flow_table ft; - struct mlx5_flow_handle *rules[MLX5E_NUM_TT]; - struct mlx5_flow_handle *tunnel_rules[MLX5E_NUM_TUNNEL_TT]; + struct mlx5e_flow_table ft; + struct mlx5e_ttc_rule rules[MLX5E_NUM_TT]; + struct mlx5_flow_handle *tunnel_rules[MLX5E_NUM_TUNNEL_TT]; }; /* NIC prio FTS */ @@ -118,8 +125,15 @@ enum { MLX5E_L2_FT_LEVEL, MLX5E_TTC_FT_LEVEL, MLX5E_INNER_TTC_FT_LEVEL, +#ifdef CONFIG_MLX5_EN_TLS + MLX5E_ACCEL_FS_TCP_FT_LEVEL, +#endif #ifdef CONFIG_MLX5_EN_ARFS - MLX5E_ARFS_FT_LEVEL + MLX5E_ARFS_FT_LEVEL, +#endif +#ifdef CONFIG_MLX5_EN_IPSEC + MLX5E_ACCEL_FS_ESP_FT_LEVEL = MLX5E_INNER_TTC_FT_LEVEL + 1, + MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL, #endif }; @@ -211,6 +225,10 @@ static inline int mlx5e_arfs_enable(struct mlx5e_priv *priv) { return -EOPNOTSUP static inline int mlx5e_arfs_disable(struct mlx5e_priv *priv) { return -EOPNOTSUPP; } #endif +#ifdef CONFIG_MLX5_EN_TLS +struct mlx5e_accel_fs_tcp; +#endif + struct mlx5e_flow_steering { struct mlx5_flow_namespace *ns; #ifdef CONFIG_MLX5_EN_RXNFC @@ -224,6 +242,9 @@ struct mlx5e_flow_steering { #ifdef CONFIG_MLX5_EN_ARFS struct mlx5e_arfs_tables arfs; #endif +#ifdef CONFIG_MLX5_EN_TLS + struct mlx5e_accel_fs_tcp *accel_tcp; +#endif }; struct ttc_params { @@ -248,6 +269,11 @@ void mlx5e_destroy_inner_ttc_table(struct mlx5e_priv *priv, struct mlx5e_ttc_table *ttc); void mlx5e_destroy_flow_table(struct mlx5e_flow_table *ft); +int mlx5e_ttc_fwd_dest(struct mlx5e_priv *priv, enum mlx5e_traffic_types type, + struct mlx5_flow_destination *new_dest); +struct mlx5_flow_destination +mlx5e_ttc_get_default_dest(struct mlx5e_priv *priv, enum mlx5e_traffic_types type); +int mlx5e_ttc_fwd_default_dest(struct mlx5e_priv *priv, enum mlx5e_traffic_types type); void mlx5e_enable_cvlan_filter(struct mlx5e_priv *priv); void mlx5e_disable_cvlan_filter(struct mlx5e_priv *priv); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/health.c b/drivers/net/ethernet/mellanox/mlx5/core/en/health.c index 7283443868f3..3dc200bcfabd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/health.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/health.c @@ -5,7 +5,7 @@ #include "lib/eq.h" #include "lib/mlx5.h" -int mlx5e_reporter_named_obj_nest_start(struct devlink_fmsg *fmsg, char *name) +int mlx5e_health_fmsg_named_obj_nest_start(struct devlink_fmsg *fmsg, char *name) { int err; @@ -20,7 +20,7 @@ int mlx5e_reporter_named_obj_nest_start(struct devlink_fmsg *fmsg, char *name) return 0; } -int mlx5e_reporter_named_obj_nest_end(struct devlink_fmsg *fmsg) +int mlx5e_health_fmsg_named_obj_nest_end(struct devlink_fmsg *fmsg) { int err; @@ -35,7 +35,7 @@ int mlx5e_reporter_named_obj_nest_end(struct devlink_fmsg *fmsg) return 0; } -int mlx5e_reporter_cq_diagnose(struct mlx5e_cq *cq, struct devlink_fmsg *fmsg) +int mlx5e_health_cq_diag_fmsg(struct mlx5e_cq *cq, struct devlink_fmsg *fmsg) { struct mlx5e_priv *priv = cq->channel->priv; u32 out[MLX5_ST_SZ_DW(query_cq_out)] = {}; @@ -50,7 +50,7 @@ int mlx5e_reporter_cq_diagnose(struct mlx5e_cq *cq, struct devlink_fmsg *fmsg) cqc = MLX5_ADDR_OF(query_cq_out, out, cq_context); hw_status = MLX5_GET(cqc, cqc, status); - err = mlx5e_reporter_named_obj_nest_start(fmsg, "CQ"); + err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "CQ"); if (err) return err; @@ -62,14 +62,22 @@ int mlx5e_reporter_cq_diagnose(struct mlx5e_cq *cq, struct devlink_fmsg *fmsg) if (err) return err; - err = mlx5e_reporter_named_obj_nest_end(fmsg); + err = devlink_fmsg_u32_pair_put(fmsg, "ci", mlx5_cqwq_get_ci(&cq->wq)); + if (err) + return err; + + err = devlink_fmsg_u32_pair_put(fmsg, "size", mlx5_cqwq_get_size(&cq->wq)); + if (err) + return err; + + err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); if (err) return err; return 0; } -int mlx5e_reporter_cq_common_diagnose(struct mlx5e_cq *cq, struct devlink_fmsg *fmsg) +int mlx5e_health_cq_common_diag_fmsg(struct mlx5e_cq *cq, struct devlink_fmsg *fmsg) { u8 cq_log_stride; u32 cq_sz; @@ -78,7 +86,7 @@ int mlx5e_reporter_cq_common_diagnose(struct mlx5e_cq *cq, struct devlink_fmsg * cq_sz = mlx5_cqwq_get_size(&cq->wq); cq_log_stride = mlx5_cqwq_get_log_stride_size(&cq->wq); - err = mlx5e_reporter_named_obj_nest_start(fmsg, "CQ"); + err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "CQ"); if (err) return err; @@ -90,26 +98,48 @@ int mlx5e_reporter_cq_common_diagnose(struct mlx5e_cq *cq, struct devlink_fmsg * if (err) return err; - err = mlx5e_reporter_named_obj_nest_end(fmsg); + err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); if (err) return err; return 0; } -int mlx5e_health_create_reporters(struct mlx5e_priv *priv) +int mlx5e_health_eq_diag_fmsg(struct mlx5_eq_comp *eq, struct devlink_fmsg *fmsg) { int err; - err = mlx5e_reporter_tx_create(priv); + err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "EQ"); if (err) return err; - err = mlx5e_reporter_rx_create(priv); + err = devlink_fmsg_u8_pair_put(fmsg, "eqn", eq->core.eqn); if (err) return err; - return 0; + err = devlink_fmsg_u32_pair_put(fmsg, "irqn", eq->core.irqn); + if (err) + return err; + + err = devlink_fmsg_u32_pair_put(fmsg, "vecidx", eq->core.vecidx); + if (err) + return err; + + err = devlink_fmsg_u32_pair_put(fmsg, "ci", eq->core.cons_index); + if (err) + return err; + + err = devlink_fmsg_u32_pair_put(fmsg, "size", eq->core.nent); + if (err) + return err; + + return mlx5e_health_fmsg_named_obj_nest_end(fmsg); +} + +void mlx5e_health_create_reporters(struct mlx5e_priv *priv) +{ + mlx5e_reporter_tx_create(priv); + mlx5e_reporter_rx_create(priv); } void mlx5e_health_destroy_reporters(struct mlx5e_priv *priv) @@ -291,7 +321,7 @@ int mlx5e_health_queue_dump(struct mlx5e_priv *priv, struct devlink_fmsg *fmsg, if (err) return err; - err = mlx5e_reporter_named_obj_nest_start(fmsg, lbl); + err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, lbl); if (err) return err; @@ -303,7 +333,7 @@ int mlx5e_health_queue_dump(struct mlx5e_priv *priv, struct devlink_fmsg *fmsg, if (err) return err; - err = mlx5e_reporter_named_obj_nest_end(fmsg); + err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); if (err) return err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/health.h b/drivers/net/ethernet/mellanox/mlx5/core/en/health.h index 38f97f79ef16..b9aadddfd000 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/health.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/health.h @@ -16,23 +16,25 @@ static inline bool cqe_syndrome_needs_recover(u8 syndrome) syndrome == MLX5_CQE_SYNDROME_WR_FLUSH_ERR; } -int mlx5e_reporter_tx_create(struct mlx5e_priv *priv); +void mlx5e_reporter_tx_create(struct mlx5e_priv *priv); void mlx5e_reporter_tx_destroy(struct mlx5e_priv *priv); void mlx5e_reporter_tx_err_cqe(struct mlx5e_txqsq *sq); int mlx5e_reporter_tx_timeout(struct mlx5e_txqsq *sq); -int mlx5e_reporter_cq_diagnose(struct mlx5e_cq *cq, struct devlink_fmsg *fmsg); -int mlx5e_reporter_cq_common_diagnose(struct mlx5e_cq *cq, struct devlink_fmsg *fmsg); -int mlx5e_reporter_named_obj_nest_start(struct devlink_fmsg *fmsg, char *name); -int mlx5e_reporter_named_obj_nest_end(struct devlink_fmsg *fmsg); +int mlx5e_health_cq_diag_fmsg(struct mlx5e_cq *cq, struct devlink_fmsg *fmsg); +int mlx5e_health_cq_common_diag_fmsg(struct mlx5e_cq *cq, struct devlink_fmsg *fmsg); +int mlx5e_health_eq_diag_fmsg(struct mlx5_eq_comp *eq, struct devlink_fmsg *fmsg); +int mlx5e_health_fmsg_named_obj_nest_start(struct devlink_fmsg *fmsg, char *name); +int mlx5e_health_fmsg_named_obj_nest_end(struct devlink_fmsg *fmsg); -int mlx5e_reporter_rx_create(struct mlx5e_priv *priv); +void mlx5e_reporter_rx_create(struct mlx5e_priv *priv); void mlx5e_reporter_rx_destroy(struct mlx5e_priv *priv); void mlx5e_reporter_icosq_cqe_err(struct mlx5e_icosq *icosq); void mlx5e_reporter_rq_cqe_err(struct mlx5e_rq *rq); void mlx5e_reporter_rx_timeout(struct mlx5e_rq *rq); #define MLX5E_REPORTER_PER_Q_MAX_LEN 256 +#define MLX5E_REPORTER_FLUSH_TIMEOUT_MSEC 2000 struct mlx5e_err_ctx { int (*recover)(void *ctx); @@ -46,7 +48,7 @@ int mlx5e_health_recover_channels(struct mlx5e_priv *priv); int mlx5e_health_report(struct mlx5e_priv *priv, struct devlink_health_reporter *reporter, char *err_str, struct mlx5e_err_ctx *err_ctx); -int mlx5e_health_create_reporters(struct mlx5e_priv *priv); +void mlx5e_health_create_reporters(struct mlx5e_priv *priv); void mlx5e_health_destroy_reporters(struct mlx5e_priv *priv); void mlx5e_health_channels_update(struct mlx5e_priv *priv); int mlx5e_health_rsc_fmsg_dump(struct mlx5e_priv *priv, struct mlx5_rsc_key *key, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/mod_hdr.c b/drivers/net/ethernet/mellanox/mlx5/core/en/mod_hdr.c new file mode 100644 index 000000000000..7edde4d536fd --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/mod_hdr.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +// Copyright (c) 2020 Mellanox Technologies + +#include <linux/jhash.h> +#include "mod_hdr.h" + +#define MLX5_MH_ACT_SZ MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto) + +struct mod_hdr_key { + int num_actions; + void *actions; +}; + +struct mlx5e_mod_hdr_handle { + /* a node of a hash table which keeps all the mod_hdr entries */ + struct hlist_node mod_hdr_hlist; + + struct mod_hdr_key key; + + struct mlx5_modify_hdr *modify_hdr; + + refcount_t refcnt; + struct completion res_ready; + int compl_result; +}; + +static u32 hash_mod_hdr_info(struct mod_hdr_key *key) +{ + return jhash(key->actions, + key->num_actions * MLX5_MH_ACT_SZ, 0); +} + +static int cmp_mod_hdr_info(struct mod_hdr_key *a, struct mod_hdr_key *b) +{ + if (a->num_actions != b->num_actions) + return 1; + + return memcmp(a->actions, b->actions, + a->num_actions * MLX5_MH_ACT_SZ); +} + +void mlx5e_mod_hdr_tbl_init(struct mod_hdr_tbl *tbl) +{ + mutex_init(&tbl->lock); + hash_init(tbl->hlist); +} + +void mlx5e_mod_hdr_tbl_destroy(struct mod_hdr_tbl *tbl) +{ + mutex_destroy(&tbl->lock); +} + +static struct mlx5e_mod_hdr_handle *mod_hdr_get(struct mod_hdr_tbl *tbl, + struct mod_hdr_key *key, + u32 hash_key) +{ + struct mlx5e_mod_hdr_handle *mh, *found = NULL; + + hash_for_each_possible(tbl->hlist, mh, mod_hdr_hlist, hash_key) { + if (!cmp_mod_hdr_info(&mh->key, key)) { + refcount_inc(&mh->refcnt); + found = mh; + break; + } + } + + return found; +} + +struct mlx5e_mod_hdr_handle * +mlx5e_mod_hdr_attach(struct mlx5_core_dev *mdev, + struct mod_hdr_tbl *tbl, + enum mlx5_flow_namespace_type namespace, + struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts) +{ + int num_actions, actions_size, err; + struct mlx5e_mod_hdr_handle *mh; + struct mod_hdr_key key; + u32 hash_key; + + num_actions = mod_hdr_acts->num_actions; + actions_size = MLX5_MH_ACT_SZ * num_actions; + + key.actions = mod_hdr_acts->actions; + key.num_actions = num_actions; + + hash_key = hash_mod_hdr_info(&key); + + mutex_lock(&tbl->lock); + mh = mod_hdr_get(tbl, &key, hash_key); + if (mh) { + mutex_unlock(&tbl->lock); + wait_for_completion(&mh->res_ready); + + if (mh->compl_result < 0) { + err = -EREMOTEIO; + goto attach_header_err; + } + goto attach_header; + } + + mh = kzalloc(sizeof(*mh) + actions_size, GFP_KERNEL); + if (!mh) { + mutex_unlock(&tbl->lock); + return ERR_PTR(-ENOMEM); + } + + mh->key.actions = (void *)mh + sizeof(*mh); + memcpy(mh->key.actions, key.actions, actions_size); + mh->key.num_actions = num_actions; + refcount_set(&mh->refcnt, 1); + init_completion(&mh->res_ready); + + hash_add(tbl->hlist, &mh->mod_hdr_hlist, hash_key); + mutex_unlock(&tbl->lock); + + mh->modify_hdr = mlx5_modify_header_alloc(mdev, namespace, + mh->key.num_actions, + mh->key.actions); + if (IS_ERR(mh->modify_hdr)) { + err = PTR_ERR(mh->modify_hdr); + mh->compl_result = err; + goto alloc_header_err; + } + mh->compl_result = 1; + complete_all(&mh->res_ready); + +attach_header: + return mh; + +alloc_header_err: + complete_all(&mh->res_ready); +attach_header_err: + mlx5e_mod_hdr_detach(mdev, tbl, mh); + return ERR_PTR(err); +} + +void mlx5e_mod_hdr_detach(struct mlx5_core_dev *mdev, + struct mod_hdr_tbl *tbl, + struct mlx5e_mod_hdr_handle *mh) +{ + if (!refcount_dec_and_mutex_lock(&mh->refcnt, &tbl->lock)) + return; + hash_del(&mh->mod_hdr_hlist); + mutex_unlock(&tbl->lock); + + if (mh->compl_result > 0) + mlx5_modify_header_dealloc(mdev, mh->modify_hdr); + + kfree(mh); +} + +struct mlx5_modify_hdr *mlx5e_mod_hdr_get(struct mlx5e_mod_hdr_handle *mh) +{ + return mh->modify_hdr; +} + diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/mod_hdr.h b/drivers/net/ethernet/mellanox/mlx5/core/en/mod_hdr.h new file mode 100644 index 000000000000..33b23d8f9182 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/mod_hdr.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2020 Mellanox Technologies */ + +#ifndef __MLX5E_EN_MOD_HDR_H__ +#define __MLX5E_EN_MOD_HDR_H__ + +#include <linux/hashtable.h> +#include <linux/mlx5/fs.h> + +struct mlx5e_mod_hdr_handle; + +struct mlx5e_tc_mod_hdr_acts { + int num_actions; + int max_actions; + void *actions; +}; + +struct mlx5e_mod_hdr_handle * +mlx5e_mod_hdr_attach(struct mlx5_core_dev *mdev, + struct mod_hdr_tbl *tbl, + enum mlx5_flow_namespace_type namespace, + struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts); +void mlx5e_mod_hdr_detach(struct mlx5_core_dev *mdev, + struct mod_hdr_tbl *tbl, + struct mlx5e_mod_hdr_handle *mh); +struct mlx5_modify_hdr *mlx5e_mod_hdr_get(struct mlx5e_mod_hdr_handle *mh); + +void mlx5e_mod_hdr_tbl_init(struct mod_hdr_tbl *tbl); +void mlx5e_mod_hdr_tbl_destroy(struct mod_hdr_tbl *tbl); + +#endif /* __MLX5E_EN_MOD_HDR_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h index 989d8f429438..a87273e801b2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h @@ -11,33 +11,33 @@ struct mlx5e_xsk_param { u16 chunk_size; }; +struct mlx5e_cq_param { + u32 cqc[MLX5_ST_SZ_DW(cqc)]; + struct mlx5_wq_param wq; + u16 eq_ix; + u8 cq_period_mode; +}; + struct mlx5e_rq_param { + struct mlx5e_cq_param cqp; u32 rqc[MLX5_ST_SZ_DW(rqc)]; struct mlx5_wq_param wq; struct mlx5e_rq_frags_info frags_info; }; struct mlx5e_sq_param { + struct mlx5e_cq_param cqp; u32 sqc[MLX5_ST_SZ_DW(sqc)]; struct mlx5_wq_param wq; bool is_mpw; }; -struct mlx5e_cq_param { - u32 cqc[MLX5_ST_SZ_DW(cqc)]; - struct mlx5_wq_param wq; - u16 eq_ix; - u8 cq_period_mode; -}; - struct mlx5e_channel_param { struct mlx5e_rq_param rq; - struct mlx5e_sq_param sq; + struct mlx5e_sq_param txq_sq; struct mlx5e_sq_param xdp_sq; struct mlx5e_sq_param icosq; - struct mlx5e_cq_param rx_cq; - struct mlx5e_cq_param tx_cq; - struct mlx5e_cq_param icosq_cq; + struct mlx5e_sq_param async_icosq; }; static inline bool mlx5e_qid_get_ch_if_in_group(struct mlx5e_params *params, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/port.c b/drivers/net/ethernet/mellanox/mlx5/core/en/port.c index 3cf3e35053f7..5de1cb9f5330 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/port.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/port.c @@ -76,6 +76,9 @@ static const u32 mlx5e_ext_link_speed[MLX5E_EXT_LINK_MODES_NUMBER] = { [MLX5E_100GAUI_2_100GBASE_CR2_KR2] = 100000, [MLX5E_200GAUI_4_200GBASE_CR4_KR4] = 200000, [MLX5E_400GAUI_8] = 400000, + [MLX5E_100GAUI_1_100GBASE_CR_KR] = 100000, + [MLX5E_200GAUI_2_200GBASE_CR2_KR2] = 200000, + [MLX5E_400GAUI_4_400GBASE_CR4_KR4] = 400000, }; bool mlx5e_ptys_ext_supported(struct mlx5_core_dev *mdev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bond.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bond.c index bdb71332cbf2..3e44e4d820c5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bond.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bond.c @@ -183,13 +183,16 @@ void mlx5e_rep_bond_unslave(struct mlx5_eswitch *esw, static bool mlx5e_rep_is_lag_netdev(struct net_device *netdev) { - struct mlx5e_priv *priv = netdev_priv(netdev); - struct mlx5e_rep_priv *rpriv = priv->ppriv; + struct mlx5e_rep_priv *rpriv; + struct mlx5e_priv *priv; /* A given netdev is not a representor or not a slave of LAG configuration */ if (!mlx5e_eswitch_rep(netdev) || !bond_slave_get_rtnl(netdev)) return false; + priv = netdev_priv(netdev); + rpriv = priv->ppriv; + /* Egress acl forward to vport is supported only non-uplink representor */ return rpriv->rep->vport != MLX5_VPORT_UPLINK; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/neigh.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/neigh.c index c3d167fa944c..906292035088 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/neigh.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/neigh.c @@ -9,6 +9,7 @@ #include <linux/spinlock.h> #include <linux/notifier.h> #include <net/netevent.h> +#include <net/arp.h> #include "neigh.h" #include "tc.h" #include "en_rep.h" diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c index eefeb1cdc2ee..79cc42d88eec 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c @@ -404,7 +404,7 @@ static void mlx5e_rep_indr_block_unbind(void *cb_priv) static LIST_HEAD(mlx5e_block_cb_list); static int -mlx5e_rep_indr_setup_block(struct net_device *netdev, +mlx5e_rep_indr_setup_block(struct net_device *netdev, struct Qdisc *sch, struct mlx5e_rep_priv *rpriv, struct flow_block_offload *f, flow_setup_cb_t *setup_cb, @@ -442,7 +442,7 @@ mlx5e_rep_indr_setup_block(struct net_device *netdev, block_cb = flow_indr_block_cb_alloc(setup_cb, indr_priv, indr_priv, mlx5e_rep_indr_block_unbind, - f, netdev, data, rpriv, + f, netdev, sch, data, rpriv, cleanup); if (IS_ERR(block_cb)) { list_del(&indr_priv->list); @@ -472,18 +472,18 @@ mlx5e_rep_indr_setup_block(struct net_device *netdev, } static -int mlx5e_rep_indr_setup_cb(struct net_device *netdev, void *cb_priv, +int mlx5e_rep_indr_setup_cb(struct net_device *netdev, struct Qdisc *sch, void *cb_priv, enum tc_setup_type type, void *type_data, void *data, void (*cleanup)(struct flow_block_cb *block_cb)) { switch (type) { case TC_SETUP_BLOCK: - return mlx5e_rep_indr_setup_block(netdev, cb_priv, type_data, + return mlx5e_rep_indr_setup_block(netdev, sch, cb_priv, type_data, mlx5e_rep_indr_setup_tc_cb, data, cleanup); case TC_SETUP_FT: - return mlx5e_rep_indr_setup_block(netdev, cb_priv, type_data, + return mlx5e_rep_indr_setup_block(netdev, sch, cb_priv, type_data, mlx5e_rep_indr_setup_ft_cb, data, cleanup); default: @@ -551,19 +551,31 @@ static bool mlx5e_restore_tunnel(struct mlx5e_priv *priv, struct sk_buff *skb, } } - tun_dst = tun_rx_dst(enc_opts.key.len); + if (key.enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { + tun_dst = __ip_tun_set_dst(key.enc_ipv4.src, key.enc_ipv4.dst, + key.enc_ip.tos, key.enc_ip.ttl, + key.enc_tp.dst, TUNNEL_KEY, + key32_to_tunnel_id(key.enc_key_id.keyid), + enc_opts.key.len); + } else if (key.enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { + tun_dst = __ipv6_tun_set_dst(&key.enc_ipv6.src, &key.enc_ipv6.dst, + key.enc_ip.tos, key.enc_ip.ttl, + key.enc_tp.dst, 0, TUNNEL_KEY, + key32_to_tunnel_id(key.enc_key_id.keyid), + enc_opts.key.len); + } else { + netdev_dbg(priv->netdev, + "Couldn't restore tunnel, unsupported addr_type: %d\n", + key.enc_control.addr_type); + return false; + } + if (!tun_dst) { - WARN_ON_ONCE(true); + netdev_dbg(priv->netdev, "Couldn't restore tunnel, no tun_dst\n"); return false; } - ip_tunnel_key_init(&tun_dst->u.tun_info.key, - key.enc_ipv4.src, key.enc_ipv4.dst, - key.enc_ip.tos, key.enc_ip.ttl, - 0, /* label */ - key.enc_tp.src, key.enc_tp.dst, - key32_to_tunnel_id(key.enc_key_id.keyid), - TUNNEL_KEY); + tun_dst->u.tun_info.key.tp_src = key.enc_tp.src; if (enc_opts.key.len) ip_tunnel_info_opts_set(&tun_dst->u.tun_info, @@ -594,7 +606,7 @@ bool mlx5e_rep_tc_update_skb(struct mlx5_cqe64 *cqe, struct mlx5e_tc_update_priv *tc_priv) { #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) - u32 chain = 0, reg_c0, reg_c1, tunnel_id, tuple_id; + u32 chain = 0, reg_c0, reg_c1, tunnel_id, zone_restore_id; struct mlx5_rep_uplink_priv *uplink_priv; struct mlx5e_rep_priv *uplink_rpriv; struct tc_skb_ext *tc_skb_ext; @@ -631,11 +643,12 @@ bool mlx5e_rep_tc_update_skb(struct mlx5_cqe64 *cqe, tc_skb_ext->chain = chain; - tuple_id = reg_c1 & TUPLE_ID_MAX; + zone_restore_id = reg_c1 & ZONE_RESTORE_MAX; uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH); uplink_priv = &uplink_rpriv->uplink_priv; - if (!mlx5e_tc_ct_restore_flow(uplink_priv, skb, tuple_id)) + if (!mlx5e_tc_ct_restore_flow(uplink_priv, skb, + zone_restore_id)) return false; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c index c209579fc213..9913647a1faf 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c @@ -3,6 +3,7 @@ #include "health.h" #include "params.h" +#include "txrx.h" static int mlx5e_query_rq_state(struct mlx5_core_dev *dev, u32 rqn, u8 *state) { @@ -29,7 +30,8 @@ out: static int mlx5e_wait_for_icosq_flush(struct mlx5e_icosq *icosq) { - unsigned long exp_time = jiffies + msecs_to_jiffies(2000); + unsigned long exp_time = jiffies + + msecs_to_jiffies(MLX5E_REPORTER_FLUSH_TIMEOUT_MSEC); while (time_before(jiffies, exp_time)) { if (icosq->cc == icosq->pc) @@ -123,25 +125,9 @@ static int mlx5e_rq_to_ready(struct mlx5e_rq *rq, int curr_state) static int mlx5e_rx_reporter_err_rq_cqe_recover(void *ctx) { - struct mlx5_core_dev *mdev; - struct net_device *dev; - struct mlx5e_rq *rq; - u8 state; + struct mlx5e_rq *rq = ctx; int err; - rq = ctx; - mdev = rq->mdev; - dev = rq->netdev; - err = mlx5e_query_rq_state(mdev, rq->rqn, &state); - if (err) { - netdev_err(dev, "Failed to query RQ 0x%x state. err = %d\n", - rq->rqn, err); - goto out; - } - - if (state != MLX5_RQC_STATE_ERR) - goto out; - mlx5e_deactivate_rq(rq); mlx5e_free_rx_descs(rq); @@ -191,19 +177,71 @@ static int mlx5e_rx_reporter_recover(struct devlink_health_reporter *reporter, mlx5e_health_recover_channels(priv); } +static int mlx5e_reporter_icosq_diagnose(struct mlx5e_icosq *icosq, u8 hw_state, + struct devlink_fmsg *fmsg) +{ + int err; + + err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "ICOSQ"); + if (err) + return err; + + err = devlink_fmsg_u32_pair_put(fmsg, "sqn", icosq->sqn); + if (err) + return err; + + err = devlink_fmsg_u8_pair_put(fmsg, "HW state", hw_state); + if (err) + return err; + + err = devlink_fmsg_u32_pair_put(fmsg, "cc", icosq->cc); + if (err) + return err; + + err = devlink_fmsg_u32_pair_put(fmsg, "pc", icosq->pc); + if (err) + return err; + + err = devlink_fmsg_u32_pair_put(fmsg, "WQE size", + mlx5_wq_cyc_get_size(&icosq->wq)); + if (err) + return err; + + err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "CQ"); + if (err) + return err; + + err = devlink_fmsg_u32_pair_put(fmsg, "cqn", icosq->cq.mcq.cqn); + if (err) + return err; + + err = devlink_fmsg_u32_pair_put(fmsg, "cc", icosq->cq.wq.cc); + if (err) + return err; + + err = devlink_fmsg_u32_pair_put(fmsg, "size", mlx5_cqwq_get_size(&icosq->cq.wq)); + if (err) + return err; + + err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); + if (err) + return err; + + return mlx5e_health_fmsg_named_obj_nest_end(fmsg); +} + static int mlx5e_rx_reporter_build_diagnose_output(struct mlx5e_rq *rq, struct devlink_fmsg *fmsg) { struct mlx5e_priv *priv = rq->channel->priv; - struct mlx5e_params *params; struct mlx5e_icosq *icosq; u8 icosq_hw_state; + u16 wqe_counter; int wqes_sz; u8 hw_state; u16 wq_head; int err; - params = &priv->channels.params; icosq = &rq->channel->icosq; err = mlx5e_query_rq_state(priv->mdev, rq->rqn, &hw_state); if (err) @@ -214,8 +252,8 @@ static int mlx5e_rx_reporter_build_diagnose_output(struct mlx5e_rq *rq, return err; wqes_sz = mlx5e_rqwq_get_cur_sz(rq); - wq_head = params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ ? - rq->mpwqe.wq.head : mlx5_wq_cyc_get_head(&rq->wqe.wq); + wq_head = mlx5e_rqwq_get_head(rq); + wqe_counter = mlx5e_rqwq_get_wqe_counter(rq); err = devlink_fmsg_obj_nest_start(fmsg); if (err) @@ -237,6 +275,10 @@ static int mlx5e_rx_reporter_build_diagnose_output(struct mlx5e_rq *rq, if (err) return err; + err = devlink_fmsg_u32_pair_put(fmsg, "WQE counter", wqe_counter); + if (err) + return err; + err = devlink_fmsg_u32_pair_put(fmsg, "posted WQEs", wqes_sz); if (err) return err; @@ -245,11 +287,15 @@ static int mlx5e_rx_reporter_build_diagnose_output(struct mlx5e_rq *rq, if (err) return err; - err = devlink_fmsg_u8_pair_put(fmsg, "ICOSQ HW state", icosq_hw_state); + err = mlx5e_health_cq_diag_fmsg(&rq->cq, fmsg); if (err) return err; - err = mlx5e_reporter_cq_diagnose(&rq->cq, fmsg); + err = mlx5e_health_eq_diag_fmsg(rq->cq.mcq.eq, fmsg); + if (err) + return err; + + err = mlx5e_reporter_icosq_diagnose(icosq, icosq_hw_state, fmsg); if (err) return err; @@ -279,11 +325,11 @@ static int mlx5e_rx_reporter_diagnose(struct devlink_health_reporter *reporter, rq_sz = mlx5e_rqwq_get_size(generic_rq); rq_stride = BIT(mlx5e_mpwqe_get_log_stride_size(priv->mdev, params, NULL)); - err = mlx5e_reporter_named_obj_nest_start(fmsg, "Common config"); + err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "Common config"); if (err) goto unlock; - err = mlx5e_reporter_named_obj_nest_start(fmsg, "RQ"); + err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "RQ"); if (err) goto unlock; @@ -299,15 +345,15 @@ static int mlx5e_rx_reporter_diagnose(struct devlink_health_reporter *reporter, if (err) goto unlock; - err = mlx5e_reporter_named_obj_nest_end(fmsg); + err = mlx5e_health_cq_common_diag_fmsg(&generic_rq->cq, fmsg); if (err) goto unlock; - err = mlx5e_reporter_cq_common_diagnose(&generic_rq->cq, fmsg); + err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); if (err) goto unlock; - err = mlx5e_reporter_named_obj_nest_end(fmsg); + err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); if (err) goto unlock; @@ -340,7 +386,7 @@ static int mlx5e_rx_reporter_dump_icosq(struct mlx5e_priv *priv, struct devlink_ if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) return 0; - err = mlx5e_reporter_named_obj_nest_start(fmsg, "SX Slice"); + err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "SX Slice"); if (err) return err; @@ -350,15 +396,15 @@ static int mlx5e_rx_reporter_dump_icosq(struct mlx5e_priv *priv, struct devlink_ if (err) return err; - err = mlx5e_reporter_named_obj_nest_end(fmsg); + err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); if (err) return err; - err = mlx5e_reporter_named_obj_nest_start(fmsg, "ICOSQ"); + err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "ICOSQ"); if (err) return err; - err = mlx5e_reporter_named_obj_nest_start(fmsg, "QPC"); + err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "QPC"); if (err) return err; @@ -370,11 +416,11 @@ static int mlx5e_rx_reporter_dump_icosq(struct mlx5e_priv *priv, struct devlink_ if (err) return err; - err = mlx5e_reporter_named_obj_nest_end(fmsg); + err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); if (err) return err; - err = mlx5e_reporter_named_obj_nest_start(fmsg, "send_buff"); + err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "send_buff"); if (err) return err; @@ -385,11 +431,11 @@ static int mlx5e_rx_reporter_dump_icosq(struct mlx5e_priv *priv, struct devlink_ if (err) return err; - err = mlx5e_reporter_named_obj_nest_end(fmsg); + err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); if (err) return err; - return mlx5e_reporter_named_obj_nest_end(fmsg); + return mlx5e_health_fmsg_named_obj_nest_end(fmsg); } static int mlx5e_rx_reporter_dump_rq(struct mlx5e_priv *priv, struct devlink_fmsg *fmsg, @@ -402,7 +448,7 @@ static int mlx5e_rx_reporter_dump_rq(struct mlx5e_priv *priv, struct devlink_fms if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) return 0; - err = mlx5e_reporter_named_obj_nest_start(fmsg, "RX Slice"); + err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "RX Slice"); if (err) return err; @@ -412,15 +458,15 @@ static int mlx5e_rx_reporter_dump_rq(struct mlx5e_priv *priv, struct devlink_fms if (err) return err; - err = mlx5e_reporter_named_obj_nest_end(fmsg); + err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); if (err) return err; - err = mlx5e_reporter_named_obj_nest_start(fmsg, "RQ"); + err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "RQ"); if (err) return err; - err = mlx5e_reporter_named_obj_nest_start(fmsg, "QPC"); + err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "QPC"); if (err) return err; @@ -432,11 +478,11 @@ static int mlx5e_rx_reporter_dump_rq(struct mlx5e_priv *priv, struct devlink_fms if (err) return err; - err = mlx5e_reporter_named_obj_nest_end(fmsg); + err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); if (err) return err; - err = mlx5e_reporter_named_obj_nest_start(fmsg, "receive_buff"); + err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "receive_buff"); if (err) return err; @@ -446,11 +492,11 @@ static int mlx5e_rx_reporter_dump_rq(struct mlx5e_priv *priv, struct devlink_fms if (err) return err; - err = mlx5e_reporter_named_obj_nest_end(fmsg); + err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); if (err) return err; - return mlx5e_reporter_named_obj_nest_end(fmsg); + return mlx5e_health_fmsg_named_obj_nest_end(fmsg); } static int mlx5e_rx_reporter_dump_all_rqs(struct mlx5e_priv *priv, @@ -462,7 +508,7 @@ static int mlx5e_rx_reporter_dump_all_rqs(struct mlx5e_priv *priv, if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) return 0; - err = mlx5e_reporter_named_obj_nest_start(fmsg, "RX Slice"); + err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "RX Slice"); if (err) return err; @@ -472,7 +518,7 @@ static int mlx5e_rx_reporter_dump_all_rqs(struct mlx5e_priv *priv, if (err) return err; - err = mlx5e_reporter_named_obj_nest_end(fmsg); + err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); if (err) return err; @@ -563,22 +609,18 @@ static const struct devlink_health_reporter_ops mlx5_rx_reporter_ops = { #define MLX5E_REPORTER_RX_GRACEFUL_PERIOD 500 -int mlx5e_reporter_rx_create(struct mlx5e_priv *priv) +void mlx5e_reporter_rx_create(struct mlx5e_priv *priv) { - struct devlink *devlink = priv_to_devlink(priv->mdev); struct devlink_health_reporter *reporter; - reporter = devlink_health_reporter_create(devlink, - &mlx5_rx_reporter_ops, - MLX5E_REPORTER_RX_GRACEFUL_PERIOD, - priv); + reporter = devlink_port_health_reporter_create(&priv->dl_port, &mlx5_rx_reporter_ops, + MLX5E_REPORTER_RX_GRACEFUL_PERIOD, priv); if (IS_ERR(reporter)) { netdev_warn(priv->netdev, "Failed to create rx reporter, err = %ld\n", PTR_ERR(reporter)); - return PTR_ERR(reporter); + return; } priv->rx_reporter = reporter; - return 0; } void mlx5e_reporter_rx_destroy(struct mlx5e_priv *priv) @@ -586,5 +628,5 @@ void mlx5e_reporter_rx_destroy(struct mlx5e_priv *priv) if (!priv->rx_reporter) return; - devlink_health_reporter_destroy(priv->rx_reporter); + devlink_port_health_reporter_destroy(priv->rx_reporter); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c index 9805fc085512..8be6eaa3eeb1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c @@ -5,7 +5,8 @@ static int mlx5e_wait_for_sq_flush(struct mlx5e_txqsq *sq) { - unsigned long exp_time = jiffies + msecs_to_jiffies(2000); + unsigned long exp_time = jiffies + + msecs_to_jiffies(MLX5E_REPORTER_FLUSH_TIMEOUT_MSEC); while (time_before(jiffies, exp_time)) { if (sq->cc == sq->pc) @@ -82,17 +83,40 @@ out: return err; } +struct mlx5e_tx_timeout_ctx { + struct mlx5e_txqsq *sq; + signed int status; +}; + static int mlx5e_tx_reporter_timeout_recover(void *ctx) { + struct mlx5e_tx_timeout_ctx *to_ctx; + struct mlx5e_priv *priv; struct mlx5_eq_comp *eq; struct mlx5e_txqsq *sq; int err; - sq = ctx; + to_ctx = ctx; + sq = to_ctx->sq; eq = sq->cq.mcq.eq; + priv = sq->channel->priv; err = mlx5e_health_channel_eq_recover(eq, sq->channel); - if (err) - clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); + if (!err) { + to_ctx->status = 0; /* this sq recovered */ + return err; + } + + err = mlx5e_safe_reopen_channels(priv); + if (!err) { + to_ctx->status = 1; /* all channels recovered */ + return err; + } + + to_ctx->status = err; + clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); + netdev_err(priv->netdev, + "mlx5e_safe_reopen_channels failed recovering from a tx_timeout, err(%d).\n", + err); return err; } @@ -165,7 +189,11 @@ mlx5e_tx_reporter_build_diagnose_output(struct devlink_fmsg *fmsg, if (err) return err; - err = mlx5e_reporter_cq_diagnose(&sq->cq, fmsg); + err = mlx5e_health_cq_diag_fmsg(&sq->cq, fmsg); + if (err) + return err; + + err = mlx5e_health_eq_diag_fmsg(sq->cq.mcq.eq, fmsg); if (err) return err; @@ -194,11 +222,11 @@ static int mlx5e_tx_reporter_diagnose(struct devlink_health_reporter *reporter, sq_sz = mlx5_wq_cyc_get_size(&generic_sq->wq); sq_stride = MLX5_SEND_WQE_BB; - err = mlx5e_reporter_named_obj_nest_start(fmsg, "Common Config"); + err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "Common Config"); if (err) goto unlock; - err = mlx5e_reporter_named_obj_nest_start(fmsg, "SQ"); + err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "SQ"); if (err) goto unlock; @@ -210,15 +238,15 @@ static int mlx5e_tx_reporter_diagnose(struct devlink_health_reporter *reporter, if (err) goto unlock; - err = mlx5e_reporter_cq_common_diagnose(&generic_sq->cq, fmsg); + err = mlx5e_health_cq_common_diag_fmsg(&generic_sq->cq, fmsg); if (err) goto unlock; - err = mlx5e_reporter_named_obj_nest_end(fmsg); + err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); if (err) goto unlock; - err = mlx5e_reporter_named_obj_nest_end(fmsg); + err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); if (err) goto unlock; @@ -256,7 +284,7 @@ static int mlx5e_tx_reporter_dump_sq(struct mlx5e_priv *priv, struct devlink_fms if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) return 0; - err = mlx5e_reporter_named_obj_nest_start(fmsg, "SX Slice"); + err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "SX Slice"); if (err) return err; @@ -266,15 +294,15 @@ static int mlx5e_tx_reporter_dump_sq(struct mlx5e_priv *priv, struct devlink_fms if (err) return err; - err = mlx5e_reporter_named_obj_nest_end(fmsg); + err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); if (err) return err; - err = mlx5e_reporter_named_obj_nest_start(fmsg, "SQ"); + err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "SQ"); if (err) return err; - err = mlx5e_reporter_named_obj_nest_start(fmsg, "QPC"); + err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "QPC"); if (err) return err; @@ -286,11 +314,11 @@ static int mlx5e_tx_reporter_dump_sq(struct mlx5e_priv *priv, struct devlink_fms if (err) return err; - err = mlx5e_reporter_named_obj_nest_end(fmsg); + err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); if (err) return err; - err = mlx5e_reporter_named_obj_nest_start(fmsg, "send_buff"); + err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "send_buff"); if (err) return err; @@ -300,11 +328,11 @@ static int mlx5e_tx_reporter_dump_sq(struct mlx5e_priv *priv, struct devlink_fms if (err) return err; - err = mlx5e_reporter_named_obj_nest_end(fmsg); + err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); if (err) return err; - return mlx5e_reporter_named_obj_nest_end(fmsg); + return mlx5e_health_fmsg_named_obj_nest_end(fmsg); } static int mlx5e_tx_reporter_dump_all_sqs(struct mlx5e_priv *priv, @@ -316,7 +344,7 @@ static int mlx5e_tx_reporter_dump_all_sqs(struct mlx5e_priv *priv, if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) return 0; - err = mlx5e_reporter_named_obj_nest_start(fmsg, "SX Slice"); + err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "SX Slice"); if (err) return err; @@ -326,7 +354,7 @@ static int mlx5e_tx_reporter_dump_all_sqs(struct mlx5e_priv *priv, if (err) return err; - err = mlx5e_reporter_named_obj_nest_end(fmsg); + err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); if (err) return err; @@ -384,9 +412,11 @@ int mlx5e_reporter_tx_timeout(struct mlx5e_txqsq *sq) { struct mlx5e_priv *priv = sq->channel->priv; char err_str[MLX5E_REPORTER_PER_Q_MAX_LEN]; + struct mlx5e_tx_timeout_ctx to_ctx = {}; struct mlx5e_err_ctx err_ctx = {}; - err_ctx.ctx = sq; + to_ctx.sq = sq; + err_ctx.ctx = &to_ctx; err_ctx.recover = mlx5e_tx_reporter_timeout_recover; err_ctx.dump = mlx5e_tx_reporter_dump_sq; snprintf(err_str, sizeof(err_str), @@ -394,7 +424,8 @@ int mlx5e_reporter_tx_timeout(struct mlx5e_txqsq *sq) sq->channel->ix, sq->sqn, sq->cq.mcq.cqn, sq->cc, sq->pc, jiffies_to_usecs(jiffies - sq->txq->trans_start)); - return mlx5e_health_report(priv, priv->tx_reporter, err_str, &err_ctx); + mlx5e_health_report(priv, priv->tx_reporter, err_str, &err_ctx); + return to_ctx.status; } static const struct devlink_health_reporter_ops mlx5_tx_reporter_ops = { @@ -406,25 +437,19 @@ static const struct devlink_health_reporter_ops mlx5_tx_reporter_ops = { #define MLX5_REPORTER_TX_GRACEFUL_PERIOD 500 -int mlx5e_reporter_tx_create(struct mlx5e_priv *priv) +void mlx5e_reporter_tx_create(struct mlx5e_priv *priv) { struct devlink_health_reporter *reporter; - struct mlx5_core_dev *mdev = priv->mdev; - struct devlink *devlink; - - devlink = priv_to_devlink(mdev); - reporter = - devlink_health_reporter_create(devlink, &mlx5_tx_reporter_ops, - MLX5_REPORTER_TX_GRACEFUL_PERIOD, - priv); + + reporter = devlink_port_health_reporter_create(&priv->dl_port, &mlx5_tx_reporter_ops, + MLX5_REPORTER_TX_GRACEFUL_PERIOD, priv); if (IS_ERR(reporter)) { netdev_warn(priv->netdev, "Failed to create tx reporter, err = %ld\n", PTR_ERR(reporter)); - return PTR_ERR(reporter); + return; } priv->tx_reporter = reporter; - return 0; } void mlx5e_reporter_tx_destroy(struct mlx5e_priv *priv) @@ -432,5 +457,5 @@ void mlx5e_reporter_tx_destroy(struct mlx5e_priv *priv) if (!priv->tx_reporter) return; - devlink_health_reporter_destroy(priv->tx_reporter); + devlink_port_health_reporter_destroy(priv->tx_reporter); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c index aad1c29b23db..c6bc9224c3b1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c @@ -16,6 +16,8 @@ #include "esw/chains.h" #include "en/tc_ct.h" +#include "en/mod_hdr.h" +#include "en/mapping.h" #include "en.h" #include "en_tc.h" #include "en_rep.h" @@ -30,6 +32,9 @@ #define MLX5_FTE_ID_MAX GENMASK(MLX5_FTE_ID_BITS - 1, 0) #define MLX5_FTE_ID_MASK MLX5_FTE_ID_MAX +#define MLX5_CT_LABELS_BITS (mlx5e_tc_attr_to_reg_mappings[LABELS_TO_REG].mlen * 8) +#define MLX5_CT_LABELS_MASK GENMASK(MLX5_CT_LABELS_BITS - 1, 0) + #define ct_dbg(fmt, args...)\ netdev_dbg(ct_priv->netdev, "ct_debug: " fmt "\n", ##args) @@ -39,10 +44,14 @@ struct mlx5_tc_ct_priv { struct idr fte_ids; struct xarray tuple_ids; struct rhashtable zone_ht; + struct rhashtable ct_tuples_ht; + struct rhashtable ct_tuples_nat_ht; struct mlx5_flow_table *ct; struct mlx5_flow_table *ct_nat; struct mlx5_flow_table *post_ct; struct mutex control_lock; /* guards parallel adds/dels */ + struct mapping_ctx *zone_mapping; + struct mapping_ctx *labels_mapping; }; struct mlx5_ct_flow { @@ -57,8 +66,8 @@ struct mlx5_ct_flow { struct mlx5_ct_zone_rule { struct mlx5_flow_handle *rule; + struct mlx5e_mod_hdr_handle *mh; struct mlx5_esw_flow_attr attr; - int tupleid; bool nat; }; @@ -74,6 +83,7 @@ struct mlx5_tc_ct_pre { struct mlx5_ct_ft { struct rhash_head node; u16 zone; + u32 zone_restore_id; refcount_t refcount; struct nf_flowtable *nf_ft; struct mlx5_tc_ct_priv *ct_priv; @@ -82,12 +92,37 @@ struct mlx5_ct_ft { struct mlx5_tc_ct_pre pre_ct_nat; }; -struct mlx5_ct_entry { +struct mlx5_ct_tuple { + u16 addr_type; + __be16 n_proto; + u8 ip_proto; + struct { + union { + __be32 src_v4; + struct in6_addr src_v6; + }; + union { + __be32 dst_v4; + struct in6_addr dst_v6; + }; + } ip; + struct { + __be16 src; + __be16 dst; + } port; + u16 zone; +}; + +struct mlx5_ct_entry { struct rhash_head node; + struct rhash_head tuple_node; + struct rhash_head tuple_nat_node; struct mlx5_fc *counter; unsigned long cookie; unsigned long restore_cookie; + struct mlx5_ct_tuple tuple; + struct mlx5_ct_tuple tuple_nat; struct mlx5_ct_zone_rule zone_rules[2]; }; @@ -106,6 +141,22 @@ static const struct rhashtable_params zone_params = { .automatic_shrinking = true, }; +static const struct rhashtable_params tuples_ht_params = { + .head_offset = offsetof(struct mlx5_ct_entry, tuple_node), + .key_offset = offsetof(struct mlx5_ct_entry, tuple), + .key_len = sizeof(((struct mlx5_ct_entry *)0)->tuple), + .automatic_shrinking = true, + .min_size = 16 * 1024, +}; + +static const struct rhashtable_params tuples_nat_ht_params = { + .head_offset = offsetof(struct mlx5_ct_entry, tuple_nat_node), + .key_offset = offsetof(struct mlx5_ct_entry, tuple_nat), + .key_len = sizeof(((struct mlx5_ct_entry *)0)->tuple_nat), + .automatic_shrinking = true, + .min_size = 16 * 1024, +}; + static struct mlx5_tc_ct_priv * mlx5_tc_ct_get_ct_priv(struct mlx5e_priv *priv) { @@ -119,6 +170,115 @@ mlx5_tc_ct_get_ct_priv(struct mlx5e_priv *priv) } static int +mlx5_tc_ct_rule_to_tuple(struct mlx5_ct_tuple *tuple, struct flow_rule *rule) +{ + struct flow_match_control control; + struct flow_match_basic basic; + + flow_rule_match_basic(rule, &basic); + flow_rule_match_control(rule, &control); + + tuple->n_proto = basic.key->n_proto; + tuple->ip_proto = basic.key->ip_proto; + tuple->addr_type = control.key->addr_type; + + if (tuple->addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { + struct flow_match_ipv4_addrs match; + + flow_rule_match_ipv4_addrs(rule, &match); + tuple->ip.src_v4 = match.key->src; + tuple->ip.dst_v4 = match.key->dst; + } else if (tuple->addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { + struct flow_match_ipv6_addrs match; + + flow_rule_match_ipv6_addrs(rule, &match); + tuple->ip.src_v6 = match.key->src; + tuple->ip.dst_v6 = match.key->dst; + } else { + return -EOPNOTSUPP; + } + + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { + struct flow_match_ports match; + + flow_rule_match_ports(rule, &match); + switch (tuple->ip_proto) { + case IPPROTO_TCP: + case IPPROTO_UDP: + tuple->port.src = match.key->src; + tuple->port.dst = match.key->dst; + break; + default: + return -EOPNOTSUPP; + } + } else { + return -EOPNOTSUPP; + } + + return 0; +} + +static int +mlx5_tc_ct_rule_to_tuple_nat(struct mlx5_ct_tuple *tuple, + struct flow_rule *rule) +{ + struct flow_action *flow_action = &rule->action; + struct flow_action_entry *act; + u32 offset, val, ip6_offset; + int i; + + flow_action_for_each(i, act, flow_action) { + if (act->id != FLOW_ACTION_MANGLE) + continue; + + offset = act->mangle.offset; + val = act->mangle.val; + switch (act->mangle.htype) { + case FLOW_ACT_MANGLE_HDR_TYPE_IP4: + if (offset == offsetof(struct iphdr, saddr)) + tuple->ip.src_v4 = cpu_to_be32(val); + else if (offset == offsetof(struct iphdr, daddr)) + tuple->ip.dst_v4 = cpu_to_be32(val); + else + return -EOPNOTSUPP; + break; + + case FLOW_ACT_MANGLE_HDR_TYPE_IP6: + ip6_offset = (offset - offsetof(struct ipv6hdr, saddr)); + ip6_offset /= 4; + if (ip6_offset < 8) + tuple->ip.src_v6.s6_addr32[ip6_offset] = cpu_to_be32(val); + else + return -EOPNOTSUPP; + break; + + case FLOW_ACT_MANGLE_HDR_TYPE_TCP: + if (offset == offsetof(struct tcphdr, source)) + tuple->port.src = cpu_to_be16(val); + else if (offset == offsetof(struct tcphdr, dest)) + tuple->port.dst = cpu_to_be16(val); + else + return -EOPNOTSUPP; + break; + + case FLOW_ACT_MANGLE_HDR_TYPE_UDP: + if (offset == offsetof(struct udphdr, source)) + tuple->port.src = cpu_to_be16(val); + else if (offset == offsetof(struct udphdr, dest)) + tuple->port.dst = cpu_to_be16(val); + else + return -EOPNOTSUPP; + break; + + default: + return -EOPNOTSUPP; + } + } + + return 0; +} + +static int mlx5_tc_ct_set_tuple_match(struct mlx5e_priv *priv, struct mlx5_flow_spec *spec, struct flow_rule *rule) { @@ -243,11 +403,12 @@ mlx5_tc_ct_entry_del_rule(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_esw_flow_attr *attr = &zone_rule->attr; struct mlx5_eswitch *esw = ct_priv->esw; - ct_dbg("Deleting ct entry rule in zone %d", entry->zone); + ct_dbg("Deleting ct entry rule in zone %d", entry->tuple.zone); mlx5_eswitch_del_offloaded_rule(esw, zone_rule->rule, attr); - mlx5_modify_header_dealloc(esw->dev, attr->modify_hdr); - xa_erase(&ct_priv->tuple_ids, zone_rule->tupleid); + mlx5e_mod_hdr_detach(ct_priv->esw->dev, + &esw->offloads.mod_hdr, zone_rule->mh); + mapping_remove(ct_priv->labels_mapping, attr->ct_attr.ct_labels_id); } static void @@ -280,8 +441,8 @@ mlx5_tc_ct_entry_set_registers(struct mlx5_tc_ct_priv *ct_priv, struct mlx5e_tc_mod_hdr_acts *mod_acts, u8 ct_state, u32 mark, - u32 label, - u32 tupleid) + u32 labels_id, + u8 zone_restore_id) { struct mlx5_eswitch *esw = ct_priv->esw; int err; @@ -297,12 +458,12 @@ mlx5_tc_ct_entry_set_registers(struct mlx5_tc_ct_priv *ct_priv, return err; err = mlx5e_tc_match_to_reg_set(esw->dev, mod_acts, - LABELS_TO_REG, label); + LABELS_TO_REG, labels_id); if (err) return err; err = mlx5e_tc_match_to_reg_set(esw->dev, mod_acts, - TUPLEID_TO_REG, tupleid); + ZONE_RESTORE_TO_REG, zone_restore_id); if (err) return err; @@ -429,12 +590,10 @@ static int mlx5_tc_ct_entry_create_mod_hdr(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_esw_flow_attr *attr, struct flow_rule *flow_rule, - u32 tupleid, - bool nat) + struct mlx5e_mod_hdr_handle **mh, + u8 zone_restore_id, bool nat) { struct mlx5e_tc_mod_hdr_acts mod_acts = {}; - struct mlx5_eswitch *esw = ct_priv->esw; - struct mlx5_modify_hdr *mod_hdr; struct flow_action_entry *meta; u16 ct_state = 0; int err; @@ -443,13 +602,10 @@ mlx5_tc_ct_entry_create_mod_hdr(struct mlx5_tc_ct_priv *ct_priv, if (!meta) return -EOPNOTSUPP; - if (meta->ct_metadata.labels[1] || - meta->ct_metadata.labels[2] || - meta->ct_metadata.labels[3]) { - ct_dbg("Failed to offload ct entry due to unsupported label"); + err = mapping_add(ct_priv->labels_mapping, meta->ct_metadata.labels, + &attr->ct_attr.ct_labels_id); + if (err) return -EOPNOTSUPP; - } - if (nat) { err = mlx5_tc_ct_entry_create_nat(ct_priv, flow_rule, &mod_acts); @@ -463,25 +619,27 @@ mlx5_tc_ct_entry_create_mod_hdr(struct mlx5_tc_ct_priv *ct_priv, err = mlx5_tc_ct_entry_set_registers(ct_priv, &mod_acts, ct_state, meta->ct_metadata.mark, - meta->ct_metadata.labels[0], - tupleid); + attr->ct_attr.ct_labels_id, + zone_restore_id); if (err) goto err_mapping; - mod_hdr = mlx5_modify_header_alloc(esw->dev, MLX5_FLOW_NAMESPACE_FDB, - mod_acts.num_actions, - mod_acts.actions); - if (IS_ERR(mod_hdr)) { - err = PTR_ERR(mod_hdr); + *mh = mlx5e_mod_hdr_attach(ct_priv->esw->dev, + &ct_priv->esw->offloads.mod_hdr, + MLX5_FLOW_NAMESPACE_FDB, + &mod_acts); + if (IS_ERR(*mh)) { + err = PTR_ERR(*mh); goto err_mapping; } - attr->modify_hdr = mod_hdr; + attr->modify_hdr = mlx5e_mod_hdr_get(*mh); dealloc_mod_hdr_actions(&mod_acts); return 0; err_mapping: dealloc_mod_hdr_actions(&mod_acts); + mapping_remove(ct_priv->labels_mapping, attr->ct_attr.ct_labels_id); return err; } @@ -489,13 +647,12 @@ static int mlx5_tc_ct_entry_add_rule(struct mlx5_tc_ct_priv *ct_priv, struct flow_rule *flow_rule, struct mlx5_ct_entry *entry, - bool nat) + bool nat, u8 zone_restore_id) { struct mlx5_ct_zone_rule *zone_rule = &entry->zone_rules[nat]; struct mlx5_esw_flow_attr *attr = &zone_rule->attr; struct mlx5_eswitch *esw = ct_priv->esw; struct mlx5_flow_spec *spec = NULL; - u32 tupleid; int err; zone_rule->nat = nat; @@ -504,18 +661,9 @@ mlx5_tc_ct_entry_add_rule(struct mlx5_tc_ct_priv *ct_priv, if (!spec) return -ENOMEM; - /* Get tuple unique id */ - err = xa_alloc(&ct_priv->tuple_ids, &tupleid, zone_rule, - XA_LIMIT(1, TUPLE_ID_MAX), GFP_KERNEL); - if (err) { - netdev_warn(ct_priv->netdev, - "Failed to allocate tuple id, err: %d\n", err); - goto err_xa_alloc; - } - zone_rule->tupleid = tupleid; - err = mlx5_tc_ct_entry_create_mod_hdr(ct_priv, attr, flow_rule, - tupleid, nat); + &zone_rule->mh, + zone_restore_id, nat); if (err) { ct_dbg("Failed to create ct entry mod hdr"); goto err_mod_hdr; @@ -533,7 +681,7 @@ mlx5_tc_ct_entry_add_rule(struct mlx5_tc_ct_priv *ct_priv, mlx5_tc_ct_set_tuple_match(netdev_priv(ct_priv->netdev), spec, flow_rule); mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG, - entry->zone & MLX5_CT_ZONE_MASK, + entry->tuple.zone & MLX5_CT_ZONE_MASK, MLX5_CT_ZONE_MASK); zone_rule->rule = mlx5_eswitch_add_offloaded_rule(esw, spec, attr); @@ -544,15 +692,14 @@ mlx5_tc_ct_entry_add_rule(struct mlx5_tc_ct_priv *ct_priv, } kfree(spec); - ct_dbg("Offloaded ct entry rule in zone %d", entry->zone); + ct_dbg("Offloaded ct entry rule in zone %d", entry->tuple.zone); return 0; err_rule: - mlx5_modify_header_dealloc(esw->dev, attr->modify_hdr); + mlx5e_mod_hdr_detach(ct_priv->esw->dev, + &esw->offloads.mod_hdr, zone_rule->mh); err_mod_hdr: - xa_erase(&ct_priv->tuple_ids, zone_rule->tupleid); -err_xa_alloc: kfree(spec); return err; } @@ -560,7 +707,8 @@ err_xa_alloc: static int mlx5_tc_ct_entry_add_rules(struct mlx5_tc_ct_priv *ct_priv, struct flow_rule *flow_rule, - struct mlx5_ct_entry *entry) + struct mlx5_ct_entry *entry, + u8 zone_restore_id) { struct mlx5_eswitch *esw = ct_priv->esw; int err; @@ -572,11 +720,13 @@ mlx5_tc_ct_entry_add_rules(struct mlx5_tc_ct_priv *ct_priv, return err; } - err = mlx5_tc_ct_entry_add_rule(ct_priv, flow_rule, entry, false); + err = mlx5_tc_ct_entry_add_rule(ct_priv, flow_rule, entry, false, + zone_restore_id); if (err) goto err_orig; - err = mlx5_tc_ct_entry_add_rule(ct_priv, flow_rule, entry, true); + err = mlx5_tc_ct_entry_add_rule(ct_priv, flow_rule, entry, true, + zone_restore_id); if (err) goto err_nat; @@ -613,11 +763,35 @@ mlx5_tc_ct_block_flow_offload_add(struct mlx5_ct_ft *ft, if (!entry) return -ENOMEM; - entry->zone = ft->zone; + entry->tuple.zone = ft->zone; entry->cookie = flow->cookie; entry->restore_cookie = meta_action->ct_metadata.cookie; - err = mlx5_tc_ct_entry_add_rules(ct_priv, flow_rule, entry); + err = mlx5_tc_ct_rule_to_tuple(&entry->tuple, flow_rule); + if (err) + goto err_set; + + memcpy(&entry->tuple_nat, &entry->tuple, sizeof(entry->tuple)); + err = mlx5_tc_ct_rule_to_tuple_nat(&entry->tuple_nat, flow_rule); + if (err) + goto err_set; + + err = rhashtable_insert_fast(&ct_priv->ct_tuples_ht, + &entry->tuple_node, + tuples_ht_params); + if (err) + goto err_tuple; + + if (memcmp(&entry->tuple, &entry->tuple_nat, sizeof(entry->tuple))) { + err = rhashtable_insert_fast(&ct_priv->ct_tuples_nat_ht, + &entry->tuple_nat_node, + tuples_nat_ht_params); + if (err) + goto err_tuple_nat; + } + + err = mlx5_tc_ct_entry_add_rules(ct_priv, flow_rule, entry, + ft->zone_restore_id); if (err) goto err_rules; @@ -631,12 +805,34 @@ mlx5_tc_ct_block_flow_offload_add(struct mlx5_ct_ft *ft, err_insert: mlx5_tc_ct_entry_del_rules(ct_priv, entry); err_rules: + rhashtable_remove_fast(&ct_priv->ct_tuples_nat_ht, + &entry->tuple_nat_node, tuples_nat_ht_params); +err_tuple_nat: + if (entry->tuple_node.next) + rhashtable_remove_fast(&ct_priv->ct_tuples_ht, + &entry->tuple_node, + tuples_ht_params); +err_tuple: +err_set: kfree(entry); netdev_warn(ct_priv->netdev, "Failed to offload ct entry, err: %d\n", err); return err; } +static void +mlx5_tc_ct_del_ft_entry(struct mlx5_tc_ct_priv *ct_priv, + struct mlx5_ct_entry *entry) +{ + mlx5_tc_ct_entry_del_rules(ct_priv, entry); + if (entry->tuple_node.next) + rhashtable_remove_fast(&ct_priv->ct_tuples_nat_ht, + &entry->tuple_nat_node, + tuples_nat_ht_params); + rhashtable_remove_fast(&ct_priv->ct_tuples_ht, &entry->tuple_node, + tuples_ht_params); +} + static int mlx5_tc_ct_block_flow_offload_del(struct mlx5_ct_ft *ft, struct flow_cls_offload *flow) @@ -649,7 +845,7 @@ mlx5_tc_ct_block_flow_offload_del(struct mlx5_ct_ft *ft, if (!entry) return -ENOENT; - mlx5_tc_ct_entry_del_rules(ft->ct_priv, entry); + mlx5_tc_ct_del_ft_entry(ft->ct_priv, entry); WARN_ON(rhashtable_remove_fast(&ft->ct_entries_ht, &entry->node, cts_ht_params)); @@ -672,7 +868,7 @@ mlx5_tc_ct_block_flow_offload_stats(struct mlx5_ct_ft *ft, return -ENOENT; mlx5_fc_query_cached(entry->counter, &bytes, &packets, &lastuse); - flow_stats_update(&f->stats, bytes, packets, lastuse, + flow_stats_update(&f->stats, bytes, packets, 0, lastuse, FLOW_ACTION_HW_STATS_DELAYED); return 0; @@ -702,10 +898,71 @@ mlx5_tc_ct_block_flow_offload(enum tc_setup_type type, void *type_data, return -EOPNOTSUPP; } +static bool +mlx5_tc_ct_skb_to_tuple(struct sk_buff *skb, struct mlx5_ct_tuple *tuple, + u16 zone) +{ + struct flow_keys flow_keys; + + skb_reset_network_header(skb); + skb_flow_dissect_flow_keys(skb, &flow_keys, 0); + + tuple->zone = zone; + + if (flow_keys.basic.ip_proto != IPPROTO_TCP && + flow_keys.basic.ip_proto != IPPROTO_UDP) + return false; + + tuple->port.src = flow_keys.ports.src; + tuple->port.dst = flow_keys.ports.dst; + tuple->n_proto = flow_keys.basic.n_proto; + tuple->ip_proto = flow_keys.basic.ip_proto; + + switch (flow_keys.basic.n_proto) { + case htons(ETH_P_IP): + tuple->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; + tuple->ip.src_v4 = flow_keys.addrs.v4addrs.src; + tuple->ip.dst_v4 = flow_keys.addrs.v4addrs.dst; + break; + + case htons(ETH_P_IPV6): + tuple->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; + tuple->ip.src_v6 = flow_keys.addrs.v6addrs.src; + tuple->ip.dst_v6 = flow_keys.addrs.v6addrs.dst; + break; + default: + goto out; + } + + return true; + +out: + return false; +} + +int +mlx5_tc_ct_add_no_trk_match(struct mlx5e_priv *priv, + struct mlx5_flow_spec *spec) +{ + u32 ctstate = 0, ctstate_mask = 0; + + mlx5e_tc_match_to_reg_get_match(spec, CTSTATE_TO_REG, + &ctstate, &ctstate_mask); + if (ctstate_mask) + return -EOPNOTSUPP; + + ctstate_mask |= MLX5_CT_STATE_TRK_BIT; + mlx5e_tc_match_to_reg_match(spec, CTSTATE_TO_REG, + ctstate, ctstate_mask); + + return 0; +} + int mlx5_tc_ct_parse_match(struct mlx5e_priv *priv, struct mlx5_flow_spec *spec, struct flow_cls_offload *f, + struct mlx5_ct_attr *ct_attr, struct netlink_ext_ack *extack) { struct mlx5_tc_ct_priv *ct_priv = mlx5_tc_ct_get_ct_priv(priv); @@ -716,6 +973,7 @@ mlx5_tc_ct_parse_match(struct mlx5e_priv *priv, u16 ct_state_on, ct_state_off; u16 ct_state, ct_state_mask; struct flow_match_ct match; + u32 ct_labels[4]; if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CT)) return 0; @@ -742,12 +1000,6 @@ mlx5_tc_ct_parse_match(struct mlx5e_priv *priv, return -EOPNOTSUPP; } - if (mask->ct_labels[1] || mask->ct_labels[2] || mask->ct_labels[3]) { - NL_SET_ERR_MSG_MOD(extack, - "only lower 32bits of ct_labels are supported for offload"); - return -EOPNOTSUPP; - } - ct_state_on = ct_state & ct_state_mask; ct_state_off = (ct_state & ct_state_mask) ^ ct_state_mask; trk = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_TRACKED; @@ -776,10 +1028,17 @@ mlx5_tc_ct_parse_match(struct mlx5e_priv *priv, if (mask->ct_mark) mlx5e_tc_match_to_reg_match(spec, MARK_TO_REG, key->ct_mark, mask->ct_mark); - if (mask->ct_labels[0]) - mlx5e_tc_match_to_reg_match(spec, LABELS_TO_REG, - key->ct_labels[0], - mask->ct_labels[0]); + if (mask->ct_labels[0] || mask->ct_labels[1] || mask->ct_labels[2] || + mask->ct_labels[3]) { + ct_labels[0] = key->ct_labels[0] & mask->ct_labels[0]; + ct_labels[1] = key->ct_labels[1] & mask->ct_labels[1]; + ct_labels[2] = key->ct_labels[2] & mask->ct_labels[2]; + ct_labels[3] = key->ct_labels[3] & mask->ct_labels[3]; + if (mapping_add(ct_priv->labels_mapping, ct_labels, &ct_attr->ct_labels_id)) + return -EOPNOTSUPP; + mlx5e_tc_match_to_reg_match(spec, LABELS_TO_REG, ct_attr->ct_labels_id, + MLX5_CT_LABELS_MASK); + } return 0; } @@ -1054,6 +1313,10 @@ mlx5_tc_ct_add_ft_cb(struct mlx5_tc_ct_priv *ct_priv, u16 zone, if (!ft) return ERR_PTR(-ENOMEM); + err = mapping_add(ct_priv->zone_mapping, &zone, &ft->zone_restore_id); + if (err) + goto err_mapping; + ft->zone = zone; ft->nf_ft = nf_ft; ft->ct_priv = ct_priv; @@ -1086,6 +1349,8 @@ err_insert: err_init: mlx5_tc_ct_free_pre_ct_tables(ft); err_alloc_pre_ct: + mapping_remove(ct_priv->zone_mapping, ft->zone_restore_id); +err_mapping: kfree(ft); return ERR_PTR(err); } @@ -1096,7 +1361,7 @@ mlx5_tc_ct_flush_ft_entry(void *ptr, void *arg) struct mlx5_tc_ct_priv *ct_priv = arg; struct mlx5_ct_entry *entry = ptr; - mlx5_tc_ct_entry_del_rules(ct_priv, entry); + mlx5_tc_ct_del_ft_entry(ct_priv, entry); kfree(entry); } @@ -1113,6 +1378,7 @@ mlx5_tc_ct_del_ft_cb(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_ft *ft) mlx5_tc_ct_flush_ft_entry, ct_priv); mlx5_tc_ct_free_pre_ct_tables(ft); + mapping_remove(ct_priv->zone_mapping, ft->zone_restore_id); kfree(ft); } @@ -1138,8 +1404,9 @@ mlx5_tc_ct_del_ft_cb(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_ft *ft) * + tuple + zone match + * +--------------------+ * | set mark - * | set label + * | set labels_id * | set established + * | set zone_restore * | do nat (if needed) * v * +--------------+ @@ -1147,12 +1414,11 @@ mlx5_tc_ct_del_ft_cb(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_ft *ft) * + fte_id match +------------------------> * +--------------+ */ -static int +static struct mlx5_flow_handle * __mlx5_tc_ct_flow_offload(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow, struct mlx5_flow_spec *orig_spec, - struct mlx5_esw_flow_attr *attr, - struct mlx5_flow_handle **flow_rule) + struct mlx5_esw_flow_attr *attr) { struct mlx5_tc_ct_priv *ct_priv = mlx5_tc_ct_get_ct_priv(priv); bool nat = attr->ct_attr.ct_action & TCA_CT_ACT_NAT; @@ -1172,7 +1438,7 @@ __mlx5_tc_ct_flow_offload(struct mlx5e_priv *priv, if (!post_ct_spec || !ct_flow) { kfree(post_ct_spec); kfree(ct_flow); - return -ENOMEM; + return ERR_PTR(-ENOMEM); } /* Register for CT established events */ @@ -1293,11 +1559,10 @@ __mlx5_tc_ct_flow_offload(struct mlx5e_priv *priv, } attr->ct_attr.ct_flow = ct_flow; - *flow_rule = ct_flow->post_ct_rule; dealloc_mod_hdr_actions(&pre_mod_acts); kfree(post_ct_spec); - return 0; + return rule; err_insert_orig: mlx5_eswitch_del_offloaded_rule(ct_priv->esw, ct_flow->post_ct_rule, @@ -1315,16 +1580,14 @@ err_ft: kfree(post_ct_spec); kfree(ct_flow); netdev_warn(priv->netdev, "Failed to offload ct flow, err %d\n", err); - return err; + return ERR_PTR(err); } -static int +static struct mlx5_flow_handle * __mlx5_tc_ct_flow_offload_clear(struct mlx5e_priv *priv, - struct mlx5e_tc_flow *flow, struct mlx5_flow_spec *orig_spec, struct mlx5_esw_flow_attr *attr, - struct mlx5e_tc_mod_hdr_acts *mod_acts, - struct mlx5_flow_handle **flow_rule) + struct mlx5e_tc_mod_hdr_acts *mod_acts) { struct mlx5_tc_ct_priv *ct_priv = mlx5_tc_ct_get_ct_priv(priv); struct mlx5_eswitch *esw = ct_priv->esw; @@ -1336,7 +1599,7 @@ __mlx5_tc_ct_flow_offload_clear(struct mlx5e_priv *priv, ct_flow = kzalloc(sizeof(*ct_flow), GFP_KERNEL); if (!ct_flow) - return -ENOMEM; + return ERR_PTR(-ENOMEM); /* Base esw attributes on original rule attribute */ pre_ct_attr = &ct_flow->pre_ct_attr; @@ -1371,16 +1634,14 @@ __mlx5_tc_ct_flow_offload_clear(struct mlx5e_priv *priv, attr->ct_attr.ct_flow = ct_flow; ct_flow->pre_ct_rule = rule; - *flow_rule = rule; - - return 0; + return rule; err_insert: mlx5_modify_header_dealloc(priv->mdev, mod_hdr); err_set_registers: netdev_warn(priv->netdev, "Failed to offload ct clear flow, err %d\n", err); - return err; + return ERR_PTR(err); } struct mlx5_flow_handle * @@ -1392,22 +1653,18 @@ mlx5_tc_ct_flow_offload(struct mlx5e_priv *priv, { bool clear_action = attr->ct_attr.ct_action & TCA_CT_ACT_CLEAR; struct mlx5_tc_ct_priv *ct_priv = mlx5_tc_ct_get_ct_priv(priv); - struct mlx5_flow_handle *rule = ERR_PTR(-EINVAL); - int err; + struct mlx5_flow_handle *rule; if (!ct_priv) return ERR_PTR(-EOPNOTSUPP); mutex_lock(&ct_priv->control_lock); + if (clear_action) - err = __mlx5_tc_ct_flow_offload_clear(priv, flow, spec, attr, - mod_hdr_acts, &rule); + rule = __mlx5_tc_ct_flow_offload_clear(priv, spec, attr, mod_hdr_acts); else - err = __mlx5_tc_ct_flow_offload(priv, flow, spec, attr, - &rule); + rule = __mlx5_tc_ct_flow_offload(priv, flow, spec, attr); mutex_unlock(&ct_priv->control_lock); - if (err) - return ERR_PTR(err); return rule; } @@ -1535,6 +1792,18 @@ mlx5_tc_ct_init(struct mlx5_rep_uplink_priv *uplink_priv) goto err_alloc; } + ct_priv->zone_mapping = mapping_create(sizeof(u16), 0, true); + if (IS_ERR(ct_priv->zone_mapping)) { + err = PTR_ERR(ct_priv->zone_mapping); + goto err_mapping_zone; + } + + ct_priv->labels_mapping = mapping_create(sizeof(u32) * 4, 0, true); + if (IS_ERR(ct_priv->labels_mapping)) { + err = PTR_ERR(ct_priv->labels_mapping); + goto err_mapping_labels; + } + ct_priv->esw = esw; ct_priv->netdev = rpriv->netdev; ct_priv->ct = mlx5_esw_chains_create_global_table(esw); @@ -1561,9 +1830,10 @@ mlx5_tc_ct_init(struct mlx5_rep_uplink_priv *uplink_priv) } idr_init(&ct_priv->fte_ids); - xa_init_flags(&ct_priv->tuple_ids, XA_FLAGS_ALLOC1); mutex_init(&ct_priv->control_lock); rhashtable_init(&ct_priv->zone_ht, &zone_params); + rhashtable_init(&ct_priv->ct_tuples_ht, &tuples_ht_params); + rhashtable_init(&ct_priv->ct_tuples_nat_ht, &tuples_nat_ht_params); /* Done, set ct_priv to know it initializted */ uplink_priv->ct_priv = ct_priv; @@ -1575,6 +1845,10 @@ err_post_ct_tbl: err_ct_nat_tbl: mlx5_esw_chains_destroy_global_table(esw, ct_priv->ct); err_ct_tbl: + mapping_destroy(ct_priv->labels_mapping); +err_mapping_labels: + mapping_destroy(ct_priv->zone_mapping); +err_mapping_zone: kfree(ct_priv); err_alloc: err_support: @@ -1593,10 +1867,13 @@ mlx5_tc_ct_clean(struct mlx5_rep_uplink_priv *uplink_priv) mlx5_esw_chains_destroy_global_table(ct_priv->esw, ct_priv->post_ct); mlx5_esw_chains_destroy_global_table(ct_priv->esw, ct_priv->ct_nat); mlx5_esw_chains_destroy_global_table(ct_priv->esw, ct_priv->ct); + mapping_destroy(ct_priv->zone_mapping); + mapping_destroy(ct_priv->labels_mapping); + rhashtable_destroy(&ct_priv->ct_tuples_ht); + rhashtable_destroy(&ct_priv->ct_tuples_nat_ht); rhashtable_destroy(&ct_priv->zone_ht); mutex_destroy(&ct_priv->control_lock); - xa_destroy(&ct_priv->tuple_ids); idr_destroy(&ct_priv->fte_ids); kfree(ct_priv); @@ -1605,22 +1882,30 @@ mlx5_tc_ct_clean(struct mlx5_rep_uplink_priv *uplink_priv) bool mlx5e_tc_ct_restore_flow(struct mlx5_rep_uplink_priv *uplink_priv, - struct sk_buff *skb, u32 tupleid) + struct sk_buff *skb, u8 zone_restore_id) { struct mlx5_tc_ct_priv *ct_priv = uplink_priv->ct_priv; - struct mlx5_ct_zone_rule *zone_rule; + struct mlx5_ct_tuple tuple = {}; struct mlx5_ct_entry *entry; + u16 zone; - if (!ct_priv || !tupleid) + if (!ct_priv || !zone_restore_id) return true; - zone_rule = xa_load(&ct_priv->tuple_ids, tupleid); - if (!zone_rule) + if (mapping_find(ct_priv->zone_mapping, zone_restore_id, &zone)) return false; - entry = container_of(zone_rule, struct mlx5_ct_entry, - zone_rules[zone_rule->nat]); - tcf_ct_flow_table_restore_skb(skb, entry->restore_cookie); + if (!mlx5_tc_ct_skb_to_tuple(skb, &tuple, zone)) + return false; + entry = rhashtable_lookup_fast(&ct_priv->ct_tuples_ht, &tuple, + tuples_ht_params); + if (!entry) + entry = rhashtable_lookup_fast(&ct_priv->ct_tuples_nat_ht, + &tuple, tuples_nat_ht_params); + if (!entry) + return false; + + tcf_ct_flow_table_restore_skb(skb, entry->restore_cookie); return true; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h index 626f6c04882e..3baef917a677 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h @@ -25,6 +25,7 @@ struct mlx5_ct_attr { u16 ct_action; struct mlx5_ct_flow *ct_flow; struct nf_flowtable *nf_ft; + u32 ct_labels_id; }; #define zone_to_reg_ct {\ @@ -67,16 +68,17 @@ struct mlx5_ct_attr { misc_parameters_2.metadata_reg_c_5),\ } -#define tupleid_to_reg_ct {\ +#define zone_restore_to_reg_ct {\ .mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_1,\ .moffset = 0,\ - .mlen = 3,\ + .mlen = 1,\ .soffset = MLX5_BYTE_OFF(fte_match_param,\ - misc_parameters_2.metadata_reg_c_1),\ + misc_parameters_2.metadata_reg_c_1) + 3,\ } -#define TUPLE_ID_BITS (mlx5e_tc_attr_to_reg_mappings[TUPLEID_TO_REG].mlen * 8) -#define TUPLE_ID_MAX GENMASK(TUPLE_ID_BITS - 1, 0) +#define REG_MAPPING_MLEN(reg) (mlx5e_tc_attr_to_reg_mappings[reg].mlen) +#define ZONE_RESTORE_BITS (REG_MAPPING_MLEN(ZONE_RESTORE_TO_REG) * 8) +#define ZONE_RESTORE_MAX GENMASK(ZONE_RESTORE_BITS - 1, 0) #if IS_ENABLED(CONFIG_MLX5_TC_CT) @@ -89,8 +91,12 @@ int mlx5_tc_ct_parse_match(struct mlx5e_priv *priv, struct mlx5_flow_spec *spec, struct flow_cls_offload *f, + struct mlx5_ct_attr *ct_attr, struct netlink_ext_ack *extack); int +mlx5_tc_ct_add_no_trk_match(struct mlx5e_priv *priv, + struct mlx5_flow_spec *spec); +int mlx5_tc_ct_parse_action(struct mlx5e_priv *priv, struct mlx5_esw_flow_attr *attr, const struct flow_action_entry *act, @@ -109,7 +115,7 @@ mlx5_tc_ct_delete_flow(struct mlx5e_priv *priv, bool mlx5e_tc_ct_restore_flow(struct mlx5_rep_uplink_priv *uplink_priv, - struct sk_buff *skb, u32 tupleid); + struct sk_buff *skb, u8 zone_restore_id); #else /* CONFIG_MLX5_TC_CT */ @@ -128,6 +134,7 @@ static inline int mlx5_tc_ct_parse_match(struct mlx5e_priv *priv, struct mlx5_flow_spec *spec, struct flow_cls_offload *f, + struct mlx5_ct_attr *ct_attr, struct netlink_ext_ack *extack) { struct flow_rule *rule = flow_cls_offload_flow_rule(f); @@ -141,6 +148,13 @@ mlx5_tc_ct_parse_match(struct mlx5e_priv *priv, } static inline int +mlx5_tc_ct_add_no_trk_match(struct mlx5e_priv *priv, + struct mlx5_flow_spec *spec) +{ + return 0; +} + +static inline int mlx5_tc_ct_parse_action(struct mlx5e_priv *priv, struct mlx5_esw_flow_attr *attr, const struct flow_action_entry *act, @@ -170,10 +184,10 @@ mlx5_tc_ct_delete_flow(struct mlx5e_priv *priv, static inline bool mlx5e_tc_ct_restore_flow(struct mlx5_rep_uplink_priv *uplink_priv, - struct sk_buff *skb, u32 tupleid) + struct sk_buff *skb, u8 zone_restore_id) { - if (!tupleid) - return true; + if (!zone_restore_id) + return true; return false; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_geneve.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_geneve.c index 951ea26d96bc..e472ed0eacfb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_geneve.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_geneve.c @@ -301,6 +301,8 @@ static int mlx5e_tc_tun_parse_geneve_params(struct mlx5e_priv *priv, MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type, ETH_P_TEB); } + spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; + return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_gre.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_gre.c index 58b13192df23..2805416c32a3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_gre.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_gre.c @@ -80,6 +80,8 @@ static int mlx5e_tc_tun_parse_gretap(struct mlx5e_priv *priv, gre_key.key, be32_to_cpu(enc_keyid.key->keyid)); } + spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; + return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_vxlan.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_vxlan.c index 37b176801bcc..038a0f1cecec 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_vxlan.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_vxlan.c @@ -136,6 +136,8 @@ static int mlx5e_tc_tun_parse_vxlan(struct mlx5e_priv *priv, MLX5_SET(fte_match_set_misc, misc_v, vxlan_vni, be32_to_cpu(enc_keyid.key->keyid)); + spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; + return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h index bfd3e1161bc6..9334c9c3e208 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h @@ -5,14 +5,47 @@ #define __MLX5_EN_TXRX_H___ #include "en.h" +#include <linux/indirect_call_wrapper.h> #define INL_HDR_START_SZ (sizeof(((struct mlx5_wqe_eth_seg *)NULL)->inline_hdr.start)) enum mlx5e_icosq_wqe_type { MLX5E_ICOSQ_WQE_NOP, MLX5E_ICOSQ_WQE_UMR_RX, +#ifdef CONFIG_MLX5_EN_TLS + MLX5E_ICOSQ_WQE_UMR_TLS, + MLX5E_ICOSQ_WQE_SET_PSV_TLS, + MLX5E_ICOSQ_WQE_GET_PSV_TLS, +#endif }; +/* General */ +void mlx5e_trigger_irq(struct mlx5e_icosq *sq); +void mlx5e_completion_event(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe); +void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, enum mlx5_event event); +int mlx5e_napi_poll(struct napi_struct *napi, int budget); +int mlx5e_poll_ico_cq(struct mlx5e_cq *cq); + +/* RX */ +void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info); +void mlx5e_page_release_dynamic(struct mlx5e_rq *rq, + struct mlx5e_dma_info *dma_info, + bool recycle); +INDIRECT_CALLABLE_DECLARE(bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)); +INDIRECT_CALLABLE_DECLARE(bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq)); +int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget); +void mlx5e_free_rx_descs(struct mlx5e_rq *rq); +void mlx5e_free_rx_in_progress_descs(struct mlx5e_rq *rq); + +/* TX */ +u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb, + struct net_device *sb_dev); +netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev); +void mlx5e_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb, + struct mlx5e_tx_wqe *wqe, u16 pi, bool xmit_more); +bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget); +void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq); + static inline bool mlx5e_wqc_has_room_for(struct mlx5_wq_cyc *wq, u16 cc, u16 pc, u16 n) { @@ -114,9 +147,19 @@ struct mlx5e_icosq_wqe_info { struct { struct mlx5e_rq *rq; } umr; +#ifdef CONFIG_MLX5_EN_TLS + struct { + struct mlx5e_ktls_offload_context_rx *priv_rx; + } tls_set_params; + struct { + struct mlx5e_ktls_rx_resync_buf *buf; + } tls_get_params; +#endif }; }; +void mlx5e_free_icosq_descs(struct mlx5e_icosq *sq); + static inline u16 mlx5e_icosq_get_next_pi(struct mlx5e_icosq *sq, u16 size) { struct mlx5_wq_cyc *wq = &sq->wq; @@ -182,7 +225,7 @@ mlx5e_notify_hw(struct mlx5_wq_cyc *wq, u16 pc, void __iomem *uar_map, static inline bool mlx5e_transport_inline_tx_wqe(struct mlx5_wqe_ctrl_seg *cseg) { - return cseg && !!cseg->tisn; + return cseg && !!cseg->tis_tir_num; } static inline u8 @@ -253,7 +296,7 @@ static inline void mlx5e_rqwq_reset(struct mlx5e_rq *rq) } } -static inline void mlx5e_dump_error_cqe(struct mlx5e_cq *cq, u32 sqn, +static inline void mlx5e_dump_error_cqe(struct mlx5e_cq *cq, u32 qn, struct mlx5_err_cqe *err_cqe) { struct mlx5_cqwq *wq = &cq->wq; @@ -262,13 +305,53 @@ static inline void mlx5e_dump_error_cqe(struct mlx5e_cq *cq, u32 sqn, ci = mlx5_cqwq_ctr2ix(wq, wq->cc - 1); netdev_err(cq->channel->netdev, - "Error cqe on cqn 0x%x, ci 0x%x, sqn 0x%x, opcode 0x%x, syndrome 0x%x, vendor syndrome 0x%x\n", - cq->mcq.cqn, ci, sqn, + "Error cqe on cqn 0x%x, ci 0x%x, qn 0x%x, opcode 0x%x, syndrome 0x%x, vendor syndrome 0x%x\n", + cq->mcq.cqn, ci, qn, get_cqe_opcode((struct mlx5_cqe64 *)err_cqe), err_cqe->syndrome, err_cqe->vendor_err_synd); mlx5_dump_err_cqe(cq->mdev, err_cqe); } +static inline u32 mlx5e_rqwq_get_size(struct mlx5e_rq *rq) +{ + switch (rq->wq_type) { + case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: + return mlx5_wq_ll_get_size(&rq->mpwqe.wq); + default: + return mlx5_wq_cyc_get_size(&rq->wqe.wq); + } +} + +static inline u32 mlx5e_rqwq_get_cur_sz(struct mlx5e_rq *rq) +{ + switch (rq->wq_type) { + case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: + return rq->mpwqe.wq.cur_sz; + default: + return rq->wqe.wq.cur_sz; + } +} + +static inline u16 mlx5e_rqwq_get_head(struct mlx5e_rq *rq) +{ + switch (rq->wq_type) { + case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: + return mlx5_wq_ll_get_head(&rq->mpwqe.wq); + default: + return mlx5_wq_cyc_get_head(&rq->wqe.wq); + } +} + +static inline u16 mlx5e_rqwq_get_wqe_counter(struct mlx5e_rq *rq) +{ + switch (rq->wq_type) { + case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: + return mlx5_wq_ll_get_counter(&rq->mpwqe.wq); + default: + return mlx5_wq_cyc_get_counter(&rq->wqe.wq); + } +} + /* SW parser related functions */ struct mlx5e_swp_spec { @@ -305,7 +388,7 @@ mlx5e_set_eseg_swp(struct sk_buff *skb, struct mlx5_wqe_eth_seg *eseg, switch (swp_spec->tun_l4_proto) { case IPPROTO_UDP: eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L4_UDP; - /* fall through */ + fallthrough; case IPPROTO_TCP: eseg->swp_inner_l4_offset = skb_inner_transport_offset(skb) / 2; break; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c index c9d308e91965..0e6946fc121f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c @@ -114,7 +114,8 @@ mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq, xdpi.page.di = *di; } - return sq->xmit_xdp_frame(sq, &xdptxd, &xdpi, 0); + return INDIRECT_CALL_2(sq->xmit_xdp_frame, mlx5e_xmit_xdp_frame_mpwqe, + mlx5e_xmit_xdp_frame, sq, &xdptxd, &xdpi, 0); } /* returns true if packet was consumed by xdp */ @@ -151,11 +152,11 @@ bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct mlx5e_dma_info *di, return true; default: bpf_warn_invalid_xdp_action(act); - /* fall through */ + fallthrough; case XDP_ABORTED: xdp_abort: trace_xdp_exception(rq->netdev, prog, act); - /* fall through */ + fallthrough; case XDP_DROP: rq->stats->xdp_drop++; return true; @@ -237,7 +238,7 @@ enum { MLX5E_XDP_CHECK_START_MPWQE = 2, }; -static int mlx5e_xmit_xdp_frame_check_mpwqe(struct mlx5e_xdpsq *sq) +INDIRECT_CALLABLE_SCOPE int mlx5e_xmit_xdp_frame_check_mpwqe(struct mlx5e_xdpsq *sq) { if (unlikely(!sq->mpwqe.wqe)) { const u16 stop_room = mlx5e_stop_room_for_wqe(MLX5_SEND_WQE_MAX_WQEBBS); @@ -256,10 +257,9 @@ static int mlx5e_xmit_xdp_frame_check_mpwqe(struct mlx5e_xdpsq *sq) return MLX5E_XDP_CHECK_OK; } -static bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, - struct mlx5e_xdp_xmit_data *xdptxd, - struct mlx5e_xdp_info *xdpi, - int check_result) +INDIRECT_CALLABLE_SCOPE bool +mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, struct mlx5e_xdp_xmit_data *xdptxd, + struct mlx5e_xdp_info *xdpi, int check_result) { struct mlx5e_xdp_mpwqe *session = &sq->mpwqe; struct mlx5e_xdpsq_stats *stats = sq->stats; @@ -293,7 +293,7 @@ static bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, return true; } -static int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq) +INDIRECT_CALLABLE_SCOPE int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq) { if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, 1))) { /* SQ is full, ring doorbell */ @@ -305,10 +305,9 @@ static int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq) return MLX5E_XDP_CHECK_OK; } -static bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, - struct mlx5e_xdp_xmit_data *xdptxd, - struct mlx5e_xdp_info *xdpi, - int check_result) +INDIRECT_CALLABLE_SCOPE bool +mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xdp_xmit_data *xdptxd, + struct mlx5e_xdp_info *xdpi, int check_result) { struct mlx5_wq_cyc *wq = &sq->wq; u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); @@ -506,6 +505,7 @@ int mlx5e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, struct xdp_frame *xdpf = frames[i]; struct mlx5e_xdp_xmit_data xdptxd; struct mlx5e_xdp_info xdpi; + bool ret; xdptxd.data = xdpf->data; xdptxd.len = xdpf->len; @@ -522,7 +522,9 @@ int mlx5e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, xdpi.frame.xdpf = xdpf; xdpi.frame.dma_addr = xdptxd.dma_addr; - if (unlikely(!sq->xmit_xdp_frame(sq, &xdptxd, &xdpi, 0))) { + ret = INDIRECT_CALL_2(sq->xmit_xdp_frame, mlx5e_xmit_xdp_frame_mpwqe, + mlx5e_xmit_xdp_frame, sq, &xdptxd, &xdpi, 0); + if (unlikely(!ret)) { dma_unmap_single(sq->pdev, xdptxd.dma_addr, xdptxd.len, DMA_TO_DEVICE); xdp_return_frame_rx_napi(xdpf); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h index ca48c293151b..e806c13d491f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h @@ -32,6 +32,8 @@ #ifndef __MLX5_EN_XDP_H__ #define __MLX5_EN_XDP_H__ +#include <linux/indirect_call_wrapper.h> + #include "en.h" #include "en/txrx.h" @@ -70,6 +72,17 @@ void mlx5e_xdp_rx_poll_complete(struct mlx5e_rq *rq); int mlx5e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, u32 flags); +INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, + struct mlx5e_xdp_xmit_data *xdptxd, + struct mlx5e_xdp_info *xdpi, + int check_result)); +INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, + struct mlx5e_xdp_xmit_data *xdptxd, + struct mlx5e_xdp_info *xdpi, + int check_result)); +INDIRECT_CALLABLE_DECLARE(int mlx5e_xmit_xdp_frame_check_mpwqe(struct mlx5e_xdpsq *sq)); +INDIRECT_CALLABLE_DECLARE(int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq)); + static inline void mlx5e_xdp_tx_enable(struct mlx5e_priv *priv) { set_bit(MLX5E_STATE_XDP_TX_ENABLED, &priv->state); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c index 2c80205dc939..dd9df519d383 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c @@ -3,6 +3,7 @@ #include "setup.h" #include "en/params.h" +#include "en/txrx.h" /* It matches XDP_UMEM_MIN_CHUNK_SIZE, but as this constant is private and may * change unexpectedly, and mlx5e has a minimum valid stride size for striding @@ -34,31 +35,13 @@ bool mlx5e_validate_xsk_param(struct mlx5e_params *params, } } -static void mlx5e_build_xskicosq_param(struct mlx5e_priv *priv, - u8 log_wq_size, - struct mlx5e_sq_param *param) -{ - void *sqc = param->sqc; - void *wq = MLX5_ADDR_OF(sqc, sqc, wq); - - mlx5e_build_sq_param_common(priv, param); - - MLX5_SET(wq, wq, log_wq_sz, log_wq_size); -} - static void mlx5e_build_xsk_cparam(struct mlx5e_priv *priv, struct mlx5e_params *params, struct mlx5e_xsk_param *xsk, struct mlx5e_channel_param *cparam) { - const u8 xskicosq_size = MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; - mlx5e_build_rq_param(priv, params, xsk, &cparam->rq); mlx5e_build_xdpsq_param(priv, params, &cparam->xdp_sq); - mlx5e_build_xskicosq_param(priv, xskicosq_size, &cparam->icosq); - mlx5e_build_rx_cq_param(priv, params, xsk, &cparam->rx_cq); - mlx5e_build_tx_cq_param(priv, params, &cparam->tx_cq); - mlx5e_build_ico_cq_param(priv, xskicosq_size, &cparam->icosq_cq); } int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params, @@ -66,7 +49,6 @@ int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params, struct mlx5e_channel *c) { struct mlx5e_channel_param *cparam; - struct dim_cq_moder icocq_moder = {}; int err; if (!mlx5e_validate_xsk_param(params, xsk, priv->mdev)) @@ -78,7 +60,7 @@ int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params, mlx5e_build_xsk_cparam(priv, params, xsk, cparam); - err = mlx5e_open_cq(c, params->rx_cq_moderation, &cparam->rx_cq, &c->xskrq.cq); + err = mlx5e_open_cq(c, params->rx_cq_moderation, &cparam->rq.cqp, &c->xskrq.cq); if (unlikely(err)) goto err_free_cparam; @@ -86,7 +68,7 @@ int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params, if (unlikely(err)) goto err_close_rx_cq; - err = mlx5e_open_cq(c, params->tx_cq_moderation, &cparam->tx_cq, &c->xsksq.cq); + err = mlx5e_open_cq(c, params->tx_cq_moderation, &cparam->xdp_sq.cqp, &c->xsksq.cq); if (unlikely(err)) goto err_close_rq; @@ -100,31 +82,12 @@ int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params, if (unlikely(err)) goto err_close_tx_cq; - err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->xskicosq.cq); - if (unlikely(err)) - goto err_close_sq; - - /* Create a dedicated SQ for posting NOPs whenever we need an IRQ to be - * triggered and NAPI to be called on the correct CPU. - */ - err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->xskicosq); - if (unlikely(err)) - goto err_close_icocq; - kvfree(cparam); - spin_lock_init(&c->xskicosq_lock); - set_bit(MLX5E_CHANNEL_STATE_XSK, c->state); return 0; -err_close_icocq: - mlx5e_close_cq(&c->xskicosq.cq); - -err_close_sq: - mlx5e_close_xdpsq(&c->xsksq); - err_close_tx_cq: mlx5e_close_cq(&c->xsksq.cq); @@ -148,32 +111,27 @@ void mlx5e_close_xsk(struct mlx5e_channel *c) mlx5e_close_rq(&c->xskrq); mlx5e_close_cq(&c->xskrq.cq); - mlx5e_close_icosq(&c->xskicosq); - mlx5e_close_cq(&c->xskicosq.cq); mlx5e_close_xdpsq(&c->xsksq); mlx5e_close_cq(&c->xsksq.cq); memset(&c->xskrq, 0, sizeof(c->xskrq)); memset(&c->xsksq, 0, sizeof(c->xsksq)); - memset(&c->xskicosq, 0, sizeof(c->xskicosq)); } void mlx5e_activate_xsk(struct mlx5e_channel *c) { - mlx5e_activate_icosq(&c->xskicosq); set_bit(MLX5E_RQ_STATE_ENABLED, &c->xskrq.state); /* TX queue is created active. */ - spin_lock(&c->xskicosq_lock); - mlx5e_trigger_irq(&c->xskicosq); - spin_unlock(&c->xskicosq_lock); + spin_lock(&c->async_icosq_lock); + mlx5e_trigger_irq(&c->async_icosq); + spin_unlock(&c->async_icosq_lock); } void mlx5e_deactivate_xsk(struct mlx5e_channel *c) { mlx5e_deactivate_rq(&c->xskrq); /* TX queue is disabled on close. */ - mlx5e_deactivate_icosq(&c->xskicosq); } static int mlx5e_redirect_xsk_rqt(struct mlx5e_priv *priv, u16 ix, u32 rqn) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c index 83dce9cdb8c2..4d892f6cecb3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c @@ -26,19 +26,19 @@ int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) return -ENXIO; if (!napi_if_scheduled_mark_missed(&c->napi)) { - /* To avoid WQE overrun, don't post a NOP if XSKICOSQ is not + /* To avoid WQE overrun, don't post a NOP if async_icosq is not * active and not polled by NAPI. Return 0, because the upcoming * activate will trigger the IRQ for us. */ - if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &c->xskicosq.state))) + if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &c->async_icosq.state))) return 0; - if (test_and_set_bit(MLX5E_SQ_STATE_PENDING_XSK_TX, &c->xskicosq.state)) + if (test_and_set_bit(MLX5E_SQ_STATE_PENDING_XSK_TX, &c->async_icosq.state)) return 0; - spin_lock(&c->xskicosq_lock); - mlx5e_trigger_irq(&c->xskicosq); - spin_unlock(&c->xskicosq_lock); + spin_lock(&c->async_icosq_lock); + mlx5e_trigger_irq(&c->async_icosq); + spin_unlock(&c->async_icosq_lock); } return 0; @@ -75,8 +75,12 @@ bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget) xdpi.mode = MLX5E_XDP_XMIT_MODE_XSK; for (; budget; budget--) { - int check_result = sq->xmit_xdp_frame_check(sq); + int check_result = INDIRECT_CALL_2(sq->xmit_xdp_frame_check, + mlx5e_xmit_xdp_frame_check_mpwqe, + mlx5e_xmit_xdp_frame_check, + sq); struct xdp_desc desc; + bool ret; if (unlikely(check_result < 0)) { work_done = false; @@ -98,7 +102,9 @@ bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget) xsk_buff_raw_dma_sync_for_device(umem, xdptxd.dma_addr, xdptxd.len); - if (unlikely(!sq->xmit_xdp_frame(sq, &xdptxd, &xdpi, check_result))) { + ret = INDIRECT_CALL_2(sq->xmit_xdp_frame, mlx5e_xmit_xdp_frame_mpwqe, + mlx5e_xmit_xdp_frame, sq, &xdptxd, &xdpi, check_result); + if (unlikely(!ret)) { if (sq->mpwqe.wqe) mlx5e_xdp_mpwqe_complete(sq); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c index 7b17fcd0a56d..331ca2b0f8a4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.c @@ -215,16 +215,3 @@ int mlx5e_xsk_setup_umem(struct net_device *dev, struct xdp_umem *umem, u16 qid) return umem ? mlx5e_xsk_enable_umem(priv, umem, ix) : mlx5e_xsk_disable_umem(priv, ix); } - -u16 mlx5e_xsk_first_unused_channel(struct mlx5e_params *params, struct mlx5e_xsk *xsk) -{ - u16 res = xsk->refcnt ? params->num_channels : 0; - - while (res) { - if (mlx5e_xsk_get_umem(params, xsk, res - 1)) - break; - --res; - } - - return res; -} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.h index 25b4cbe58b54..bada94973586 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/umem.h @@ -26,6 +26,4 @@ int mlx5e_xsk_setup_umem(struct net_device *dev, struct xdp_umem *umem, u16 qid) int mlx5e_xsk_resize_reuseq(struct xdp_umem *umem, u32 nentries); -u16 mlx5e_xsk_first_unused_channel(struct mlx5e_params *params, struct mlx5e_xsk *xsk); - #endif /* __MLX5_EN_XSK_UMEM_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h index fac145dcf2ce..110476bdeffb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h @@ -37,6 +37,7 @@ #include <linux/skbuff.h> #include <linux/netdevice.h> #include "en_accel/ipsec_rxtx.h" +#include "en_accel/tls.h" #include "en_accel/tls_rxtx.h" #include "en.h" #include "en/txrx.h" @@ -147,4 +148,13 @@ static inline bool mlx5e_accel_tx_finish(struct mlx5e_priv *priv, return true; } +static inline int mlx5e_accel_init_rx(struct mlx5e_priv *priv) +{ + return mlx5e_ktls_init_rx(priv); +} + +static inline void mlx5e_accel_cleanup_rx(struct mlx5e_priv *priv) +{ + mlx5e_ktls_cleanup_rx(priv); +} #endif /* __MLX5E_EN_ACCEL_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c new file mode 100644 index 000000000000..4cdd9eac647d --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c @@ -0,0 +1,400 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ + +#include <linux/netdevice.h> +#include "en_accel/fs_tcp.h" +#include "fs_core.h" + +enum accel_fs_tcp_type { + ACCEL_FS_IPV4_TCP, + ACCEL_FS_IPV6_TCP, + ACCEL_FS_TCP_NUM_TYPES, +}; + +struct mlx5e_accel_fs_tcp { + struct mlx5e_flow_table tables[ACCEL_FS_TCP_NUM_TYPES]; + struct mlx5_flow_handle *default_rules[ACCEL_FS_TCP_NUM_TYPES]; +}; + +static enum mlx5e_traffic_types fs_accel2tt(enum accel_fs_tcp_type i) +{ + switch (i) { + case ACCEL_FS_IPV4_TCP: + return MLX5E_TT_IPV4_TCP; + default: /* ACCEL_FS_IPV6_TCP */ + return MLX5E_TT_IPV6_TCP; + } +} + +static void accel_fs_tcp_set_ipv4_flow(struct mlx5_flow_spec *spec, struct sock *sk) +{ + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_TCP); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4); + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), + &inet_sk(sk)->inet_daddr, 4); + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), + &inet_sk(sk)->inet_rcv_saddr, 4); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); +} + +static void accel_fs_tcp_set_ipv6_flow(struct mlx5_flow_spec *spec, struct sock *sk) +{ + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_TCP); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6); + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), + &sk->sk_v6_daddr, 16); + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), + &inet6_sk(sk)->saddr, 16); + memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, + outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), + 0xff, 16); + memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, + outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), + 0xff, 16); +} + +void mlx5e_accel_fs_del_sk(struct mlx5_flow_handle *rule) +{ + mlx5_del_flow_rules(rule); +} + +struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_priv *priv, + struct sock *sk, u32 tirn, + uint32_t flow_tag) +{ + struct mlx5_flow_destination dest = {}; + struct mlx5e_flow_table *ft = NULL; + struct mlx5e_accel_fs_tcp *fs_tcp; + MLX5_DECLARE_FLOW_ACT(flow_act); + struct mlx5_flow_handle *flow; + struct mlx5_flow_spec *spec; + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) + return ERR_PTR(-ENOMEM); + + fs_tcp = priv->fs.accel_tcp; + + spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; + + switch (sk->sk_family) { + case AF_INET: + accel_fs_tcp_set_ipv4_flow(spec, sk); + ft = &fs_tcp->tables[ACCEL_FS_IPV4_TCP]; + mlx5e_dbg(HW, priv, "%s flow is %pI4:%d -> %pI4:%d\n", __func__, + &inet_sk(sk)->inet_rcv_saddr, + inet_sk(sk)->inet_sport, + &inet_sk(sk)->inet_daddr, + inet_sk(sk)->inet_dport); + break; +#if IS_ENABLED(CONFIG_IPV6) + case AF_INET6: + if (!sk->sk_ipv6only && + ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) { + accel_fs_tcp_set_ipv4_flow(spec, sk); + ft = &fs_tcp->tables[ACCEL_FS_IPV4_TCP]; + } else { + accel_fs_tcp_set_ipv6_flow(spec, sk); + ft = &fs_tcp->tables[ACCEL_FS_IPV6_TCP]; + } + break; +#endif + default: + break; + } + + if (!ft) { + flow = ERR_PTR(-EINVAL); + goto out; + } + + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + outer_headers.tcp_dport); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + outer_headers.tcp_sport); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_dport, + ntohs(inet_sk(sk)->inet_sport)); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_sport, + ntohs(inet_sk(sk)->inet_dport)); + + dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; + dest.tir_num = tirn; + if (flow_tag != MLX5_FS_DEFAULT_FLOW_TAG) { + spec->flow_context.flow_tag = flow_tag; + spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG; + } + + flow = mlx5_add_flow_rules(ft->t, spec, &flow_act, &dest, 1); + + if (IS_ERR(flow)) + netdev_err(priv->netdev, "mlx5_add_flow_rules() failed, flow is %ld\n", + PTR_ERR(flow)); + +out: + kvfree(spec); + return flow; +} + +static int accel_fs_tcp_add_default_rule(struct mlx5e_priv *priv, + enum accel_fs_tcp_type type) +{ + struct mlx5e_flow_table *accel_fs_t; + struct mlx5_flow_destination dest; + struct mlx5e_accel_fs_tcp *fs_tcp; + MLX5_DECLARE_FLOW_ACT(flow_act); + struct mlx5_flow_handle *rule; + int err = 0; + + fs_tcp = priv->fs.accel_tcp; + accel_fs_t = &fs_tcp->tables[type]; + + dest = mlx5e_ttc_get_default_dest(priv, fs_accel2tt(type)); + rule = mlx5_add_flow_rules(accel_fs_t->t, NULL, &flow_act, &dest, 1); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + netdev_err(priv->netdev, + "%s: add default rule failed, accel_fs type=%d, err %d\n", + __func__, type, err); + return err; + } + + fs_tcp->default_rules[type] = rule; + return 0; +} + +#define MLX5E_ACCEL_FS_TCP_NUM_GROUPS (2) +#define MLX5E_ACCEL_FS_TCP_GROUP1_SIZE (BIT(16) - 1) +#define MLX5E_ACCEL_FS_TCP_GROUP2_SIZE (BIT(0)) +#define MLX5E_ACCEL_FS_TCP_TABLE_SIZE (MLX5E_ACCEL_FS_TCP_GROUP1_SIZE +\ + MLX5E_ACCEL_FS_TCP_GROUP2_SIZE) +static int accel_fs_tcp_create_groups(struct mlx5e_flow_table *ft, + enum accel_fs_tcp_type type) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + void *outer_headers_c; + int ix = 0; + u32 *in; + int err; + u8 *mc; + + ft->g = kcalloc(MLX5E_ACCEL_FS_TCP_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL); + in = kvzalloc(inlen, GFP_KERNEL); + if (!in || !ft->g) { + kvfree(ft->g); + kvfree(in); + return -ENOMEM; + } + + mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); + outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers); + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol); + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_version); + + switch (type) { + case ACCEL_FS_IPV4_TCP: + case ACCEL_FS_IPV6_TCP: + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_dport); + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_sport); + break; + default: + err = -EINVAL; + goto out; + } + + switch (type) { + case ACCEL_FS_IPV4_TCP: + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, + src_ipv4_src_ipv6.ipv4_layout.ipv4); + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, + dst_ipv4_dst_ipv6.ipv4_layout.ipv4); + break; + case ACCEL_FS_IPV6_TCP: + memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, + src_ipv4_src_ipv6.ipv6_layout.ipv6), + 0xff, 16); + memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, + dst_ipv4_dst_ipv6.ipv6_layout.ipv6), + 0xff, 16); + break; + default: + err = -EINVAL; + goto out; + } + + MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); + MLX5_SET_CFG(in, start_flow_index, ix); + ix += MLX5E_ACCEL_FS_TCP_GROUP1_SIZE; + MLX5_SET_CFG(in, end_flow_index, ix - 1); + ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); + if (IS_ERR(ft->g[ft->num_groups])) + goto err; + ft->num_groups++; + + /* Default Flow Group */ + memset(in, 0, inlen); + MLX5_SET_CFG(in, start_flow_index, ix); + ix += MLX5E_ACCEL_FS_TCP_GROUP2_SIZE; + MLX5_SET_CFG(in, end_flow_index, ix - 1); + ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); + if (IS_ERR(ft->g[ft->num_groups])) + goto err; + ft->num_groups++; + + kvfree(in); + return 0; + +err: + err = PTR_ERR(ft->g[ft->num_groups]); + ft->g[ft->num_groups] = NULL; +out: + kvfree(in); + + return err; +} + +static int accel_fs_tcp_create_table(struct mlx5e_priv *priv, enum accel_fs_tcp_type type) +{ + struct mlx5e_flow_table *ft = &priv->fs.accel_tcp->tables[type]; + struct mlx5_flow_table_attr ft_attr = {}; + int err; + + ft->num_groups = 0; + + ft_attr.max_fte = MLX5E_ACCEL_FS_TCP_TABLE_SIZE; + ft_attr.level = MLX5E_ACCEL_FS_TCP_FT_LEVEL; + ft_attr.prio = MLX5E_NIC_PRIO; + + ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr); + if (IS_ERR(ft->t)) { + err = PTR_ERR(ft->t); + ft->t = NULL; + return err; + } + + netdev_dbg(priv->netdev, "Created fs accel table id %u level %u\n", + ft->t->id, ft->t->level); + + err = accel_fs_tcp_create_groups(ft, type); + if (err) + goto err; + + err = accel_fs_tcp_add_default_rule(priv, type); + if (err) + goto err; + + return 0; +err: + mlx5e_destroy_flow_table(ft); + return err; +} + +static int accel_fs_tcp_disable(struct mlx5e_priv *priv) +{ + int err, i; + + for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) { + /* Modify ttc rules destination to point back to the indir TIRs */ + err = mlx5e_ttc_fwd_default_dest(priv, fs_accel2tt(i)); + if (err) { + netdev_err(priv->netdev, + "%s: modify ttc[%d] default destination failed, err(%d)\n", + __func__, fs_accel2tt(i), err); + return err; + } + } + + return 0; +} + +static int accel_fs_tcp_enable(struct mlx5e_priv *priv) +{ + struct mlx5_flow_destination dest = {}; + int err, i; + + dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) { + dest.ft = priv->fs.accel_tcp->tables[i].t; + + /* Modify ttc rules destination to point on the accel_fs FTs */ + err = mlx5e_ttc_fwd_dest(priv, fs_accel2tt(i), &dest); + if (err) { + netdev_err(priv->netdev, + "%s: modify ttc[%d] destination to accel failed, err(%d)\n", + __func__, fs_accel2tt(i), err); + return err; + } + } + return 0; +} + +static void accel_fs_tcp_destroy_table(struct mlx5e_priv *priv, int i) +{ + struct mlx5e_accel_fs_tcp *fs_tcp; + + fs_tcp = priv->fs.accel_tcp; + if (IS_ERR_OR_NULL(fs_tcp->tables[i].t)) + return; + + mlx5_del_flow_rules(fs_tcp->default_rules[i]); + mlx5e_destroy_flow_table(&fs_tcp->tables[i]); + fs_tcp->tables[i].t = NULL; +} + +void mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv) +{ + int i; + + if (!priv->fs.accel_tcp) + return; + + accel_fs_tcp_disable(priv); + + for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) + accel_fs_tcp_destroy_table(priv, i); + + kfree(priv->fs.accel_tcp); + priv->fs.accel_tcp = NULL; +} + +int mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv) +{ + int i, err; + + if (!MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version)) + return -EOPNOTSUPP; + + priv->fs.accel_tcp = kzalloc(sizeof(*priv->fs.accel_tcp), GFP_KERNEL); + if (!priv->fs.accel_tcp) + return -ENOMEM; + + for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) { + err = accel_fs_tcp_create_table(priv, i); + if (err) + goto err_destroy_tables; + } + + err = accel_fs_tcp_enable(priv); + if (err) + goto err_destroy_tables; + + return 0; + +err_destroy_tables: + while (--i >= 0) + accel_fs_tcp_destroy_table(priv, i); + + kfree(priv->fs.accel_tcp); + priv->fs.accel_tcp = NULL; + return err; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.h new file mode 100644 index 000000000000..589235824543 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ + +#ifndef __MLX5E_ACCEL_FS_TCP_H__ +#define __MLX5E_ACCEL_FS_TCP_H__ + +#include "en.h" + +#ifdef CONFIG_MLX5_EN_TLS +int mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv); +void mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv); +struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_priv *priv, + struct sock *sk, u32 tirn, + uint32_t flow_tag); +void mlx5e_accel_fs_del_sk(struct mlx5_flow_handle *rule); +#else +static inline int mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv) { return 0; } +static inline void mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv) {} +static inline struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_priv *priv, + struct sock *sk, u32 tirn, + uint32_t flow_tag) +{ return ERR_PTR(-EOPNOTSUPP); } +static inline void mlx5e_accel_fs_del_sk(struct mlx5_flow_handle *rule) {} +#endif + +#endif /* __MLX5E_ACCEL_FS_TCP_H__ */ + diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index 92eb3bad4acd..d39989cddd90 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -40,7 +40,7 @@ #include "en.h" #include "en_accel/ipsec.h" #include "en_accel/ipsec_rxtx.h" - +#include "en_accel/ipsec_fs.h" static struct mlx5e_ipsec_sa_entry *to_ipsec_sa_entry(struct xfrm_state *x) { @@ -111,7 +111,7 @@ static void mlx5e_ipsec_sadb_rx_del(struct mlx5e_ipsec_sa_entry *sa_entry) static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry) { struct xfrm_replay_state_esn *replay_esn; - u32 seq_bottom; + u32 seq_bottom = 0; u8 overlap; u32 *esn; @@ -121,7 +121,9 @@ static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry) } replay_esn = sa_entry->x->replay_esn; - seq_bottom = replay_esn->seq - replay_esn->replay_window + 1; + if (replay_esn->seq >= replay_esn->replay_window) + seq_bottom = replay_esn->seq - replay_esn->replay_window + 1; + overlap = sa_entry->esn_state.overlap; sa_entry->esn_state.esn = xfrm_replay_seqhi(sa_entry->x, @@ -207,7 +209,7 @@ mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) { - struct net_device *netdev = x->xso.dev; + struct net_device *netdev = x->xso.real_dev; struct mlx5e_priv *priv; priv = netdev_priv(netdev); @@ -282,10 +284,31 @@ static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) return 0; } +static int mlx5e_xfrm_fs_add_rule(struct mlx5e_priv *priv, + struct mlx5e_ipsec_sa_entry *sa_entry) +{ + if (!mlx5_is_ipsec_device(priv->mdev)) + return 0; + + return mlx5e_accel_ipsec_fs_add_rule(priv, &sa_entry->xfrm->attrs, + sa_entry->ipsec_obj_id, + &sa_entry->ipsec_rule); +} + +static void mlx5e_xfrm_fs_del_rule(struct mlx5e_priv *priv, + struct mlx5e_ipsec_sa_entry *sa_entry) +{ + if (!mlx5_is_ipsec_device(priv->mdev)) + return; + + mlx5e_accel_ipsec_fs_del_rule(priv, &sa_entry->xfrm->attrs, + &sa_entry->ipsec_rule); +} + static int mlx5e_xfrm_add_state(struct xfrm_state *x) { struct mlx5e_ipsec_sa_entry *sa_entry = NULL; - struct net_device *netdev = x->xso.dev; + struct net_device *netdev = x->xso.real_dev; struct mlx5_accel_esp_xfrm_attrs attrs; struct mlx5e_priv *priv; unsigned int sa_handle; @@ -329,10 +352,15 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) goto err_xfrm; } + sa_entry->ipsec_obj_id = sa_handle; + err = mlx5e_xfrm_fs_add_rule(priv, sa_entry); + if (err) + goto err_hw_ctx; + if (x->xso.flags & XFRM_OFFLOAD_INBOUND) { err = mlx5e_ipsec_sadb_rx_add(sa_entry, sa_handle); if (err) - goto err_hw_ctx; + goto err_add_rule; } else { sa_entry->set_iv_op = (x->props.flags & XFRM_STATE_ESN) ? mlx5e_ipsec_set_iv_esn : mlx5e_ipsec_set_iv; @@ -341,8 +369,10 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) x->xso.offload_handle = (unsigned long)sa_entry; goto out; +err_add_rule: + mlx5e_xfrm_fs_del_rule(priv, sa_entry); err_hw_ctx: - mlx5_accel_esp_free_hw_context(sa_entry->hw_context); + mlx5_accel_esp_free_hw_context(priv->mdev, sa_entry->hw_context); err_xfrm: mlx5_accel_esp_destroy_xfrm(sa_entry->xfrm); err_sa_entry: @@ -366,13 +396,15 @@ static void mlx5e_xfrm_del_state(struct xfrm_state *x) static void mlx5e_xfrm_free_state(struct xfrm_state *x) { struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); + struct mlx5e_priv *priv = netdev_priv(x->xso.dev); if (!sa_entry) return; if (sa_entry->hw_context) { flush_workqueue(sa_entry->ipsec->wq); - mlx5_accel_esp_free_hw_context(sa_entry->hw_context); + mlx5e_xfrm_fs_del_rule(priv, sa_entry); + mlx5_accel_esp_free_hw_context(sa_entry->xfrm->mdev, sa_entry->hw_context); mlx5_accel_esp_destroy_xfrm(sa_entry->xfrm); } @@ -405,6 +437,8 @@ int mlx5e_ipsec_init(struct mlx5e_priv *priv) kfree(ipsec); return -ENOMEM; } + + mlx5e_accel_ipsec_fs_init(priv); netdev_dbg(priv->netdev, "IPSec attached to netdevice\n"); return 0; } @@ -416,6 +450,7 @@ void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv) if (!ipsec) return; + mlx5e_accel_ipsec_fs_cleanup(priv); destroy_workqueue(ipsec->wq); ida_destroy(&ipsec->halloc); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index c85151a1e008..0fc8b4d4f4a3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -75,6 +75,8 @@ struct mlx5e_ipsec_stats { u64 ipsec_cmd_drop; }; +struct mlx5e_accel_fs_esp; + struct mlx5e_ipsec { struct mlx5e_priv *en_priv; DECLARE_HASHTABLE(sadb_rx, MLX5E_IPSEC_SADB_RX_BITS); @@ -84,6 +86,7 @@ struct mlx5e_ipsec { struct mlx5e_ipsec_sw_stats sw_stats; struct mlx5e_ipsec_stats stats; struct workqueue_struct *wq; + struct mlx5e_accel_fs_esp *rx_fs; }; struct mlx5e_ipsec_esn_state { @@ -92,6 +95,11 @@ struct mlx5e_ipsec_esn_state { u8 overlap: 1; }; +struct mlx5e_ipsec_rule { + struct mlx5_flow_handle *rule; + struct mlx5_modify_hdr *set_modify_hdr; +}; + struct mlx5e_ipsec_sa_entry { struct hlist_node hlist; /* Item in SADB_RX hashtable */ struct mlx5e_ipsec_esn_state esn_state; @@ -102,6 +110,8 @@ struct mlx5e_ipsec_sa_entry { void *hw_context; void (*set_iv_op)(struct sk_buff *skb, struct xfrm_state *x, struct xfrm_offload *xo); + u32 ipsec_obj_id; + struct mlx5e_ipsec_rule ipsec_rule; }; void mlx5e_ipsec_build_inverse_table(void); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c new file mode 100644 index 000000000000..429428bbc903 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -0,0 +1,544 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ + +#include <linux/netdevice.h> +#include "accel/ipsec_offload.h" +#include "ipsec_fs.h" +#include "fs_core.h" + +#define NUM_IPSEC_FTE BIT(15) + +enum accel_fs_esp_type { + ACCEL_FS_ESP4, + ACCEL_FS_ESP6, + ACCEL_FS_ESP_NUM_TYPES, +}; + +struct mlx5e_ipsec_rx_err { + struct mlx5_flow_table *ft; + struct mlx5_flow_handle *rule; + struct mlx5_modify_hdr *copy_modify_hdr; +}; + +struct mlx5e_accel_fs_esp_prot { + struct mlx5_flow_table *ft; + struct mlx5_flow_group *miss_group; + struct mlx5_flow_handle *miss_rule; + struct mlx5_flow_destination default_dest; + struct mlx5e_ipsec_rx_err rx_err; + u32 refcnt; + struct mutex prot_mutex; /* protect ESP4/ESP6 protocol */ +}; + +struct mlx5e_accel_fs_esp { + struct mlx5e_accel_fs_esp_prot fs_prot[ACCEL_FS_ESP_NUM_TYPES]; +}; + +/* IPsec RX flow steering */ +static enum mlx5e_traffic_types fs_esp2tt(enum accel_fs_esp_type i) +{ + if (i == ACCEL_FS_ESP4) + return MLX5E_TT_IPV4_IPSEC_ESP; + return MLX5E_TT_IPV6_IPSEC_ESP; +} + +static int rx_err_add_rule(struct mlx5e_priv *priv, + struct mlx5e_accel_fs_esp_prot *fs_prot, + struct mlx5e_ipsec_rx_err *rx_err) +{ + u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; + struct mlx5_core_dev *mdev = priv->mdev; + struct mlx5_flow_act flow_act = {}; + struct mlx5_modify_hdr *modify_hdr; + struct mlx5_flow_handle *fte; + struct mlx5_flow_spec *spec; + int err = 0; + + spec = kzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) + return -ENOMEM; + + /* Action to copy 7 bit ipsec_syndrome to regB[0:6] */ + MLX5_SET(copy_action_in, action, action_type, MLX5_ACTION_TYPE_COPY); + MLX5_SET(copy_action_in, action, src_field, MLX5_ACTION_IN_FIELD_IPSEC_SYNDROME); + MLX5_SET(copy_action_in, action, src_offset, 0); + MLX5_SET(copy_action_in, action, length, 7); + MLX5_SET(copy_action_in, action, dst_field, MLX5_ACTION_IN_FIELD_METADATA_REG_B); + MLX5_SET(copy_action_in, action, dst_offset, 0); + + modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_KERNEL, + 1, action); + + if (IS_ERR(modify_hdr)) { + err = PTR_ERR(modify_hdr); + netdev_err(priv->netdev, + "fail to alloc ipsec copy modify_header_id err=%d\n", err); + goto out_spec; + } + + /* create fte */ + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR | + MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + flow_act.modify_hdr = modify_hdr; + fte = mlx5_add_flow_rules(rx_err->ft, spec, &flow_act, + &fs_prot->default_dest, 1); + if (IS_ERR(fte)) { + err = PTR_ERR(fte); + netdev_err(priv->netdev, "fail to add ipsec rx err copy rule err=%d\n", err); + goto out; + } + + rx_err->rule = fte; + rx_err->copy_modify_hdr = modify_hdr; + +out: + if (err) + mlx5_modify_header_dealloc(mdev, modify_hdr); +out_spec: + kfree(spec); + return err; +} + +static void rx_err_del_rule(struct mlx5e_priv *priv, + struct mlx5e_ipsec_rx_err *rx_err) +{ + if (rx_err->rule) { + mlx5_del_flow_rules(rx_err->rule); + rx_err->rule = NULL; + } + + if (rx_err->copy_modify_hdr) { + mlx5_modify_header_dealloc(priv->mdev, rx_err->copy_modify_hdr); + rx_err->copy_modify_hdr = NULL; + } +} + +static void rx_err_destroy_ft(struct mlx5e_priv *priv, struct mlx5e_ipsec_rx_err *rx_err) +{ + rx_err_del_rule(priv, rx_err); + + if (rx_err->ft) { + mlx5_destroy_flow_table(rx_err->ft); + rx_err->ft = NULL; + } +} + +static int rx_err_create_ft(struct mlx5e_priv *priv, + struct mlx5e_accel_fs_esp_prot *fs_prot, + struct mlx5e_ipsec_rx_err *rx_err) +{ + struct mlx5_flow_table_attr ft_attr = {}; + struct mlx5_flow_table *ft; + int err; + + ft_attr.max_fte = 1; + ft_attr.autogroup.max_num_groups = 1; + ft_attr.level = MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL; + ft_attr.prio = MLX5E_NIC_PRIO; + ft = mlx5_create_auto_grouped_flow_table(priv->fs.ns, &ft_attr); + if (IS_ERR(ft)) { + err = PTR_ERR(ft); + netdev_err(priv->netdev, "fail to create ipsec rx inline ft err=%d\n", err); + return err; + } + + rx_err->ft = ft; + err = rx_err_add_rule(priv, fs_prot, rx_err); + if (err) + goto out_err; + + return 0; + +out_err: + mlx5_destroy_flow_table(ft); + rx_err->ft = NULL; + return err; +} + +static void rx_fs_destroy(struct mlx5e_accel_fs_esp_prot *fs_prot) +{ + if (fs_prot->miss_rule) { + mlx5_del_flow_rules(fs_prot->miss_rule); + fs_prot->miss_rule = NULL; + } + + if (fs_prot->miss_group) { + mlx5_destroy_flow_group(fs_prot->miss_group); + fs_prot->miss_group = NULL; + } + + if (fs_prot->ft) { + mlx5_destroy_flow_table(fs_prot->ft); + fs_prot->ft = NULL; + } +} + +static int rx_fs_create(struct mlx5e_priv *priv, + struct mlx5e_accel_fs_esp_prot *fs_prot) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + struct mlx5_flow_table_attr ft_attr = {}; + struct mlx5_flow_group *miss_group; + struct mlx5_flow_handle *miss_rule; + MLX5_DECLARE_FLOW_ACT(flow_act); + struct mlx5_flow_spec *spec; + struct mlx5_flow_table *ft; + u32 *flow_group_in; + int err = 0; + + flow_group_in = kvzalloc(inlen, GFP_KERNEL); + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!flow_group_in || !spec) { + err = -ENOMEM; + goto out; + } + + /* Create FT */ + ft_attr.max_fte = NUM_IPSEC_FTE; + ft_attr.level = MLX5E_ACCEL_FS_ESP_FT_LEVEL; + ft_attr.prio = MLX5E_NIC_PRIO; + ft_attr.autogroup.num_reserved_entries = 1; + ft_attr.autogroup.max_num_groups = 1; + ft = mlx5_create_auto_grouped_flow_table(priv->fs.ns, &ft_attr); + if (IS_ERR(ft)) { + err = PTR_ERR(ft); + netdev_err(priv->netdev, "fail to create ipsec rx ft err=%d\n", err); + goto out; + } + fs_prot->ft = ft; + + /* Create miss_group */ + MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ft->max_fte - 1); + MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ft->max_fte - 1); + miss_group = mlx5_create_flow_group(ft, flow_group_in); + if (IS_ERR(miss_group)) { + err = PTR_ERR(miss_group); + netdev_err(priv->netdev, "fail to create ipsec rx miss_group err=%d\n", err); + goto out; + } + fs_prot->miss_group = miss_group; + + /* Create miss rule */ + miss_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &fs_prot->default_dest, 1); + if (IS_ERR(miss_rule)) { + err = PTR_ERR(miss_rule); + netdev_err(priv->netdev, "fail to create ipsec rx miss_rule err=%d\n", err); + goto out; + } + fs_prot->miss_rule = miss_rule; + +out: + kfree(flow_group_in); + kfree(spec); + return err; +} + +static int rx_destroy(struct mlx5e_priv *priv, enum accel_fs_esp_type type) +{ + struct mlx5e_accel_fs_esp_prot *fs_prot; + struct mlx5e_accel_fs_esp *accel_esp; + + accel_esp = priv->ipsec->rx_fs; + + /* The netdev unreg already happened, so all offloaded rule are already removed */ + fs_prot = &accel_esp->fs_prot[type]; + + rx_fs_destroy(fs_prot); + + rx_err_destroy_ft(priv, &fs_prot->rx_err); + + return 0; +} + +static int rx_create(struct mlx5e_priv *priv, enum accel_fs_esp_type type) +{ + struct mlx5e_accel_fs_esp_prot *fs_prot; + struct mlx5e_accel_fs_esp *accel_esp; + int err; + + accel_esp = priv->ipsec->rx_fs; + fs_prot = &accel_esp->fs_prot[type]; + + fs_prot->default_dest = mlx5e_ttc_get_default_dest(priv, fs_esp2tt(type)); + + err = rx_err_create_ft(priv, fs_prot, &fs_prot->rx_err); + if (err) + return err; + + err = rx_fs_create(priv, fs_prot); + if (err) + rx_destroy(priv, type); + + return err; +} + +static int rx_ft_get(struct mlx5e_priv *priv, enum accel_fs_esp_type type) +{ + struct mlx5e_accel_fs_esp_prot *fs_prot; + struct mlx5_flow_destination dest = {}; + struct mlx5e_accel_fs_esp *accel_esp; + int err = 0; + + accel_esp = priv->ipsec->rx_fs; + fs_prot = &accel_esp->fs_prot[type]; + mutex_lock(&fs_prot->prot_mutex); + if (fs_prot->refcnt++) + goto out; + + /* create FT */ + err = rx_create(priv, type); + if (err) { + fs_prot->refcnt--; + goto out; + } + + /* connect */ + dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest.ft = fs_prot->ft; + mlx5e_ttc_fwd_dest(priv, fs_esp2tt(type), &dest); + +out: + mutex_unlock(&fs_prot->prot_mutex); + return err; +} + +static void rx_ft_put(struct mlx5e_priv *priv, enum accel_fs_esp_type type) +{ + struct mlx5e_accel_fs_esp_prot *fs_prot; + struct mlx5e_accel_fs_esp *accel_esp; + + accel_esp = priv->ipsec->rx_fs; + fs_prot = &accel_esp->fs_prot[type]; + mutex_lock(&fs_prot->prot_mutex); + if (--fs_prot->refcnt) + goto out; + + /* disconnect */ + mlx5e_ttc_fwd_default_dest(priv, fs_esp2tt(type)); + + /* remove FT */ + rx_destroy(priv, type); + +out: + mutex_unlock(&fs_prot->prot_mutex); +} + +static void setup_fte_common(struct mlx5_accel_esp_xfrm_attrs *attrs, + u32 ipsec_obj_id, + struct mlx5_flow_spec *spec, + struct mlx5_flow_act *flow_act) +{ + u8 ip_version = attrs->is_ipv6 ? 6 : 4; + + spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS | MLX5_MATCH_MISC_PARAMETERS; + + /* ip_version */ + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, ip_version); + + /* Non fragmented */ + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.frag); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.frag, 0); + + /* ESP header */ + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_ESP); + + /* SPI number */ + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters.outer_esp_spi); + MLX5_SET(fte_match_param, spec->match_value, misc_parameters.outer_esp_spi, + be32_to_cpu(attrs->spi)); + + if (ip_version == 4) { + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), + &attrs->saddr.a4, 4); + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), + &attrs->daddr.a4, 4); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); + } else { + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), + &attrs->saddr.a6, 16); + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), + &attrs->daddr.a6, 16); + memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, + outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), + 0xff, 16); + memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, + outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), + 0xff, 16); + } + + flow_act->ipsec_obj_id = ipsec_obj_id; + flow_act->flags |= FLOW_ACT_NO_APPEND; +} + +static int rx_add_rule(struct mlx5e_priv *priv, + struct mlx5_accel_esp_xfrm_attrs *attrs, + u32 ipsec_obj_id, + struct mlx5e_ipsec_rule *ipsec_rule) +{ + u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; + struct mlx5_modify_hdr *modify_hdr = NULL; + struct mlx5e_accel_fs_esp_prot *fs_prot; + struct mlx5_flow_destination dest = {}; + struct mlx5e_accel_fs_esp *accel_esp; + struct mlx5_flow_act flow_act = {}; + struct mlx5_flow_handle *rule; + enum accel_fs_esp_type type; + struct mlx5_flow_spec *spec; + int err = 0; + + accel_esp = priv->ipsec->rx_fs; + type = attrs->is_ipv6 ? ACCEL_FS_ESP6 : ACCEL_FS_ESP4; + fs_prot = &accel_esp->fs_prot[type]; + + err = rx_ft_get(priv, type); + if (err) + return err; + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) { + err = -ENOMEM; + goto out_err; + } + + setup_fte_common(attrs, ipsec_obj_id, spec, &flow_act); + + /* Set 1 bit ipsec marker */ + /* Set 24 bit ipsec_obj_id */ + MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET); + MLX5_SET(set_action_in, action, field, MLX5_ACTION_IN_FIELD_METADATA_REG_B); + MLX5_SET(set_action_in, action, data, (ipsec_obj_id << 1) | 0x1); + MLX5_SET(set_action_in, action, offset, 7); + MLX5_SET(set_action_in, action, length, 25); + + modify_hdr = mlx5_modify_header_alloc(priv->mdev, MLX5_FLOW_NAMESPACE_KERNEL, + 1, action); + if (IS_ERR(modify_hdr)) { + err = PTR_ERR(modify_hdr); + netdev_err(priv->netdev, + "fail to alloc ipsec set modify_header_id err=%d\n", err); + modify_hdr = NULL; + goto out_err; + } + + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | + MLX5_FLOW_CONTEXT_ACTION_IPSEC_DECRYPT | + MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; + dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + flow_act.modify_hdr = modify_hdr; + dest.ft = fs_prot->rx_err.ft; + rule = mlx5_add_flow_rules(fs_prot->ft, spec, &flow_act, &dest, 1); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + netdev_err(priv->netdev, "fail to add ipsec rule attrs->action=0x%x, err=%d\n", + attrs->action, err); + goto out_err; + } + + ipsec_rule->rule = rule; + ipsec_rule->set_modify_hdr = modify_hdr; + goto out; + +out_err: + if (modify_hdr) + mlx5_modify_header_dealloc(priv->mdev, modify_hdr); + rx_ft_put(priv, type); + +out: + kvfree(spec); + return err; +} + +static void rx_del_rule(struct mlx5e_priv *priv, + struct mlx5_accel_esp_xfrm_attrs *attrs, + struct mlx5e_ipsec_rule *ipsec_rule) +{ + mlx5_del_flow_rules(ipsec_rule->rule); + ipsec_rule->rule = NULL; + + mlx5_modify_header_dealloc(priv->mdev, ipsec_rule->set_modify_hdr); + ipsec_rule->set_modify_hdr = NULL; + + rx_ft_put(priv, attrs->is_ipv6 ? ACCEL_FS_ESP6 : ACCEL_FS_ESP4); +} + +int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_priv *priv, + struct mlx5_accel_esp_xfrm_attrs *attrs, + u32 ipsec_obj_id, + struct mlx5e_ipsec_rule *ipsec_rule) +{ + if (!priv->ipsec->rx_fs || attrs->action != MLX5_ACCEL_ESP_ACTION_DECRYPT) + return -EOPNOTSUPP; + + return rx_add_rule(priv, attrs, ipsec_obj_id, ipsec_rule); +} + +void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_priv *priv, + struct mlx5_accel_esp_xfrm_attrs *attrs, + struct mlx5e_ipsec_rule *ipsec_rule) +{ + if (!priv->ipsec->rx_fs) + return; + + rx_del_rule(priv, attrs, ipsec_rule); +} + +static void fs_cleanup_rx(struct mlx5e_priv *priv) +{ + struct mlx5e_accel_fs_esp_prot *fs_prot; + struct mlx5e_accel_fs_esp *accel_esp; + enum accel_fs_esp_type i; + + accel_esp = priv->ipsec->rx_fs; + for (i = 0; i < ACCEL_FS_ESP_NUM_TYPES; i++) { + fs_prot = &accel_esp->fs_prot[i]; + mutex_destroy(&fs_prot->prot_mutex); + WARN_ON(fs_prot->refcnt); + } + kfree(priv->ipsec->rx_fs); + priv->ipsec->rx_fs = NULL; +} + +static int fs_init_rx(struct mlx5e_priv *priv) +{ + struct mlx5e_accel_fs_esp_prot *fs_prot; + struct mlx5e_accel_fs_esp *accel_esp; + enum accel_fs_esp_type i; + + priv->ipsec->rx_fs = + kzalloc(sizeof(struct mlx5e_accel_fs_esp), GFP_KERNEL); + if (!priv->ipsec->rx_fs) + return -ENOMEM; + + accel_esp = priv->ipsec->rx_fs; + for (i = 0; i < ACCEL_FS_ESP_NUM_TYPES; i++) { + fs_prot = &accel_esp->fs_prot[i]; + mutex_init(&fs_prot->prot_mutex); + } + + return 0; +} + +void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_priv *priv) +{ + if (!priv->ipsec->rx_fs) + return; + + fs_cleanup_rx(priv); +} + +int mlx5e_accel_ipsec_fs_init(struct mlx5e_priv *priv) +{ + if (!mlx5_is_ipsec_device(priv->mdev) || !priv->ipsec) + return -EOPNOTSUPP; + + return fs_init_rx(priv); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.h new file mode 100644 index 000000000000..3389b3bb3ef8 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ + +#ifndef __MLX5_IPSEC_STEERING_H__ +#define __MLX5_IPSEC_STEERING_H__ + +#include "en.h" +#include "ipsec.h" +#include "accel/ipsec_offload.h" +#include "en/fs.h" + +#ifdef CONFIG_MLX5_EN_IPSEC +void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_priv *priv); +int mlx5e_accel_ipsec_fs_init(struct mlx5e_priv *priv); +int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_priv *priv, + struct mlx5_accel_esp_xfrm_attrs *attrs, + u32 ipsec_obj_id, + struct mlx5e_ipsec_rule *ipsec_rule); +void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_priv *priv, + struct mlx5_accel_esp_xfrm_attrs *attrs, + struct mlx5e_ipsec_rule *ipsec_rule); +#else +static inline void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_priv *priv) {} +static inline int mlx5e_accel_ipsec_fs_init(struct mlx5e_priv *priv) { return 0; } +#endif +#endif /* __MLX5_IPSEC_STEERING_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c index 824b87ac8f9e..93a8d68815ad 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c @@ -360,6 +360,62 @@ struct sk_buff *mlx5e_ipsec_handle_rx_skb(struct net_device *netdev, return skb; } +enum { + MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_DECRYPTED, + MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_AUTH_FAILED, + MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_BAD_TRAILER, +}; + +void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev, + struct sk_buff *skb, + struct mlx5_cqe64 *cqe) +{ + u32 ipsec_meta_data = be32_to_cpu(cqe->ft_metadata); + u8 ipsec_syndrome = ipsec_meta_data & 0xFF; + struct mlx5e_priv *priv; + struct xfrm_offload *xo; + struct xfrm_state *xs; + struct sec_path *sp; + u32 sa_handle; + + sa_handle = MLX5_IPSEC_METADATA_HANDLE(ipsec_meta_data); + priv = netdev_priv(netdev); + sp = secpath_set(skb); + if (unlikely(!sp)) { + atomic64_inc(&priv->ipsec->sw_stats.ipsec_rx_drop_sp_alloc); + return; + } + + xs = mlx5e_ipsec_sadb_rx_lookup(priv->ipsec, sa_handle); + if (unlikely(!xs)) { + atomic64_inc(&priv->ipsec->sw_stats.ipsec_rx_drop_sadb_miss); + return; + } + + sp = skb_sec_path(skb); + sp->xvec[sp->len++] = xs; + sp->olen++; + + xo = xfrm_offload(skb); + xo->flags = CRYPTO_DONE; + + switch (ipsec_syndrome & MLX5_IPSEC_METADATA_SYNDROM_MASK) { + case MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_DECRYPTED: + xo->status = CRYPTO_SUCCESS; + if (WARN_ON_ONCE(priv->ipsec->no_trailer)) + xo->flags |= XFRM_ESP_NO_TRAILER; + break; + case MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_AUTH_FAILED: + xo->status = CRYPTO_TUNNEL_ESP_AUTH_FAILED; + break; + case MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_BAD_TRAILER: + xo->status = CRYPTO_INVALID_PACKET_SYNTAX; + break; + default: + atomic64_inc(&priv->ipsec->sw_stats.ipsec_rx_drop_syndrome); + } +} + bool mlx5e_ipsec_feature_check(struct sk_buff *skb, struct net_device *netdev, netdev_features_t features) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h index ba02643586a5..f96e786db158 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h @@ -34,16 +34,19 @@ #ifndef __MLX5E_IPSEC_RXTX_H__ #define __MLX5E_IPSEC_RXTX_H__ -#ifdef CONFIG_MLX5_EN_IPSEC - #include <linux/skbuff.h> #include <net/xfrm.h> #include "en.h" #include "en/txrx.h" +#define MLX5_IPSEC_METADATA_MARKER_MASK (0x80) +#define MLX5_IPSEC_METADATA_SYNDROM_MASK (0x7F) +#define MLX5_IPSEC_METADATA_HANDLE(metadata) (((metadata) >> 8) & 0xFF) + +#ifdef CONFIG_MLX5_EN_IPSEC + struct sk_buff *mlx5e_ipsec_handle_rx_skb(struct net_device *netdev, struct sk_buff *skb, u32 *cqe_bcnt); -void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe); void mlx5e_ipsec_inverse_table_init(void); bool mlx5e_ipsec_feature_check(struct sk_buff *skb, struct net_device *netdev, @@ -55,7 +58,21 @@ void mlx5e_ipsec_set_iv(struct sk_buff *skb, struct xfrm_state *x, bool mlx5e_ipsec_handle_tx_skb(struct mlx5e_priv *priv, struct mlx5_wqe_eth_seg *eseg, struct sk_buff *skb); +void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev, + struct sk_buff *skb, + struct mlx5_cqe64 *cqe); +static inline bool mlx5_ipsec_is_rx_flow(struct mlx5_cqe64 *cqe) +{ + return !!(MLX5_IPSEC_METADATA_MARKER_MASK & be32_to_cpu(cqe->ft_metadata)); +} +#else +static inline +void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev, + struct sk_buff *skb, + struct mlx5_cqe64 *cqe) +{} +static inline bool mlx5_ipsec_is_rx_flow(struct mlx5_cqe64 *cqe) { return false; } #endif /* CONFIG_MLX5_EN_IPSEC */ #endif /* __MLX5E_IPSEC_RXTX_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c index 452fcf59c36b..1b392696280d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c @@ -3,31 +3,8 @@ #include "en.h" #include "en_accel/ktls.h" - -u16 mlx5e_ktls_get_stop_room(struct mlx5e_txqsq *sq) -{ - u16 num_dumps, stop_room = 0; - - num_dumps = mlx5e_ktls_dumps_num_wqes(sq, MAX_SKB_FRAGS, TLS_MAX_PAYLOAD_SIZE); - - stop_room += mlx5e_stop_room_for_wqe(MLX5E_KTLS_STATIC_WQEBBS); - stop_room += mlx5e_stop_room_for_wqe(MLX5E_KTLS_PROGRESS_WQEBBS); - stop_room += num_dumps * mlx5e_stop_room_for_wqe(MLX5E_KTLS_DUMP_WQEBBS); - - return stop_room; -} - -static int mlx5e_ktls_create_tis(struct mlx5_core_dev *mdev, u32 *tisn) -{ - u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; - void *tisc; - - tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); - - MLX5_SET(tisc, tisc, tls_en, 1); - - return mlx5e_create_tis(mdev, in, tisn); -} +#include "en_accel/ktls_utils.h" +#include "en_accel/fs_tcp.h" static int mlx5e_ktls_add(struct net_device *netdev, struct sock *sk, enum tls_offload_ctx_dir direction, @@ -35,42 +12,17 @@ static int mlx5e_ktls_add(struct net_device *netdev, struct sock *sk, u32 start_offload_tcp_sn) { struct mlx5e_priv *priv = netdev_priv(netdev); - struct mlx5e_ktls_offload_context_tx *tx_priv; - struct tls_context *tls_ctx = tls_get_ctx(sk); struct mlx5_core_dev *mdev = priv->mdev; int err; - if (WARN_ON(direction != TLS_OFFLOAD_CTX_DIR_TX)) - return -EINVAL; - if (WARN_ON(!mlx5e_ktls_type_check(mdev, crypto_info))) return -EOPNOTSUPP; - tx_priv = kvzalloc(sizeof(*tx_priv), GFP_KERNEL); - if (!tx_priv) - return -ENOMEM; - - tx_priv->expected_seq = start_offload_tcp_sn; - tx_priv->crypto_info = *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info; - mlx5e_set_ktls_tx_priv_ctx(tls_ctx, tx_priv); - - /* tc and underlay_qpn values are not in use for tls tis */ - err = mlx5e_ktls_create_tis(mdev, &tx_priv->tisn); - if (err) - goto create_tis_fail; - - err = mlx5_ktls_create_key(mdev, crypto_info, &tx_priv->key_id); - if (err) - goto encryption_key_create_fail; + if (direction == TLS_OFFLOAD_CTX_DIR_TX) + err = mlx5e_ktls_add_tx(netdev, sk, crypto_info, start_offload_tcp_sn); + else + err = mlx5e_ktls_add_rx(netdev, sk, crypto_info, start_offload_tcp_sn); - mlx5e_ktls_tx_offload_set_pending(tx_priv); - - return 0; - -encryption_key_create_fail: - mlx5e_destroy_tis(priv->mdev, tx_priv->tisn); -create_tis_fail: - kvfree(tx_priv); return err; } @@ -78,29 +30,72 @@ static void mlx5e_ktls_del(struct net_device *netdev, struct tls_context *tls_ctx, enum tls_offload_ctx_dir direction) { - struct mlx5e_priv *priv = netdev_priv(netdev); - struct mlx5e_ktls_offload_context_tx *tx_priv = - mlx5e_get_ktls_tx_priv_ctx(tls_ctx); + if (direction == TLS_OFFLOAD_CTX_DIR_TX) + mlx5e_ktls_del_tx(netdev, tls_ctx); + else + mlx5e_ktls_del_rx(netdev, tls_ctx); +} - mlx5e_destroy_tis(priv->mdev, tx_priv->tisn); - mlx5_ktls_destroy_key(priv->mdev, tx_priv->key_id); - kvfree(tx_priv); +static int mlx5e_ktls_resync(struct net_device *netdev, + struct sock *sk, u32 seq, u8 *rcd_sn, + enum tls_offload_ctx_dir direction) +{ + if (unlikely(direction != TLS_OFFLOAD_CTX_DIR_RX)) + return -EOPNOTSUPP; + + mlx5e_ktls_rx_resync(netdev, sk, seq, rcd_sn); + return 0; } static const struct tlsdev_ops mlx5e_ktls_ops = { .tls_dev_add = mlx5e_ktls_add, .tls_dev_del = mlx5e_ktls_del, + .tls_dev_resync = mlx5e_ktls_resync, }; void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv) { struct net_device *netdev = priv->netdev; + struct mlx5_core_dev *mdev = priv->mdev; - if (!mlx5_accel_is_ktls_device(priv->mdev)) - return; + if (mlx5_accel_is_ktls_tx(mdev)) { + netdev->hw_features |= NETIF_F_HW_TLS_TX; + netdev->features |= NETIF_F_HW_TLS_TX; + } - netdev->hw_features |= NETIF_F_HW_TLS_TX; - netdev->features |= NETIF_F_HW_TLS_TX; + if (mlx5_accel_is_ktls_rx(mdev)) + netdev->hw_features |= NETIF_F_HW_TLS_RX; netdev->tlsdev_ops = &mlx5e_ktls_ops; } + +int mlx5e_ktls_set_feature_rx(struct net_device *netdev, bool enable) +{ + struct mlx5e_priv *priv = netdev_priv(netdev); + int err = 0; + + mutex_lock(&priv->state_lock); + if (enable) + err = mlx5e_accel_fs_tcp_create(priv); + else + mlx5e_accel_fs_tcp_destroy(priv); + mutex_unlock(&priv->state_lock); + + return err; +} + +int mlx5e_ktls_init_rx(struct mlx5e_priv *priv) +{ + int err = 0; + + if (priv->netdev->features & NETIF_F_HW_TLS_RX) + err = mlx5e_accel_fs_tcp_create(priv); + + return err; +} + +void mlx5e_ktls_cleanup_rx(struct mlx5e_priv *priv) +{ + if (priv->netdev->features & NETIF_F_HW_TLS_RX) + mlx5e_accel_fs_tcp_destroy(priv); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h index c6180892cfcb..baa58b62e8df 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h @@ -7,122 +7,32 @@ #include "en.h" #ifdef CONFIG_MLX5_EN_TLS -#include <net/tls.h> -#include "accel/tls.h" -#include "en_accel/tls_rxtx.h" -#define MLX5E_KTLS_STATIC_UMR_WQE_SZ \ - (offsetof(struct mlx5e_umr_wqe, tls_static_params_ctx) + \ - MLX5_ST_SZ_BYTES(tls_static_params)) -#define MLX5E_KTLS_STATIC_WQEBBS \ - (DIV_ROUND_UP(MLX5E_KTLS_STATIC_UMR_WQE_SZ, MLX5_SEND_WQE_BB)) - -#define MLX5E_KTLS_PROGRESS_WQE_SZ \ - (offsetof(struct mlx5e_tx_wqe, tls_progress_params_ctx) + \ - MLX5_ST_SZ_BYTES(tls_progress_params)) -#define MLX5E_KTLS_PROGRESS_WQEBBS \ - (DIV_ROUND_UP(MLX5E_KTLS_PROGRESS_WQE_SZ, MLX5_SEND_WQE_BB)) - -struct mlx5e_dump_wqe { - struct mlx5_wqe_ctrl_seg ctrl; - struct mlx5_wqe_data_seg data; -}; - -#define MLX5E_TLS_FETCH_UMR_WQE(sq, pi) \ - ((struct mlx5e_umr_wqe *)mlx5e_fetch_wqe(&(sq)->wq, pi, MLX5E_KTLS_STATIC_UMR_WQE_SZ)) -#define MLX5E_TLS_FETCH_PROGRESS_WQE(sq, pi) \ - ((struct mlx5e_tx_wqe *)mlx5e_fetch_wqe(&(sq)->wq, pi, MLX5E_KTLS_PROGRESS_WQE_SZ)) -#define MLX5E_TLS_FETCH_DUMP_WQE(sq, pi) \ - ((struct mlx5e_dump_wqe *)mlx5e_fetch_wqe(&(sq)->wq, pi, \ - sizeof(struct mlx5e_dump_wqe))) - -#define MLX5E_KTLS_DUMP_WQEBBS \ - (DIV_ROUND_UP(sizeof(struct mlx5e_dump_wqe), MLX5_SEND_WQE_BB)) - -enum { - MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD = 0, - MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_OFFLOAD = 1, - MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_AUTHENTICATION = 2, -}; - -enum { - MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START = 0, - MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_TRACKING = 1, - MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_SEARCHING = 2, -}; - -struct mlx5e_ktls_offload_context_tx { - struct tls_offload_context_tx *tx_ctx; - struct tls12_crypto_info_aes_gcm_128 crypto_info; - u32 expected_seq; - u32 tisn; - u32 key_id; - bool ctx_post_pending; -}; - -struct mlx5e_ktls_offload_context_tx_shadow { - struct tls_offload_context_tx tx_ctx; - struct mlx5e_ktls_offload_context_tx *priv_tx; -}; +void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv); +int mlx5e_ktls_init_rx(struct mlx5e_priv *priv); +void mlx5e_ktls_cleanup_rx(struct mlx5e_priv *priv); +int mlx5e_ktls_set_feature_rx(struct net_device *netdev, bool enable); +#else -static inline void -mlx5e_set_ktls_tx_priv_ctx(struct tls_context *tls_ctx, - struct mlx5e_ktls_offload_context_tx *priv_tx) +static inline void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv) { - struct tls_offload_context_tx *tx_ctx = tls_offload_ctx_tx(tls_ctx); - struct mlx5e_ktls_offload_context_tx_shadow *shadow; - - BUILD_BUG_ON(sizeof(*shadow) > TLS_OFFLOAD_CONTEXT_SIZE_TX); - - shadow = (struct mlx5e_ktls_offload_context_tx_shadow *)tx_ctx; - - shadow->priv_tx = priv_tx; - priv_tx->tx_ctx = tx_ctx; } -static inline struct mlx5e_ktls_offload_context_tx * -mlx5e_get_ktls_tx_priv_ctx(struct tls_context *tls_ctx) +static inline int mlx5e_ktls_init_rx(struct mlx5e_priv *priv) { - struct tls_offload_context_tx *tx_ctx = tls_offload_ctx_tx(tls_ctx); - struct mlx5e_ktls_offload_context_tx_shadow *shadow; - - BUILD_BUG_ON(sizeof(*shadow) > TLS_OFFLOAD_CONTEXT_SIZE_TX); - - shadow = (struct mlx5e_ktls_offload_context_tx_shadow *)tx_ctx; - - return shadow->priv_tx; + return 0; } -void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv); -void mlx5e_ktls_tx_offload_set_pending(struct mlx5e_ktls_offload_context_tx *priv_tx); - -bool mlx5e_ktls_handle_tx_skb(struct tls_context *tls_ctx, struct mlx5e_txqsq *sq, - struct sk_buff *skb, int datalen, - struct mlx5e_accel_tx_tls_state *state); -void mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq, - struct mlx5e_tx_wqe_info *wi, - u32 *dma_fifo_cc); -u16 mlx5e_ktls_get_stop_room(struct mlx5e_txqsq *sq); - -static inline u8 -mlx5e_ktls_dumps_num_wqes(struct mlx5e_txqsq *sq, unsigned int nfrags, - unsigned int sync_len) +static inline void mlx5e_ktls_cleanup_rx(struct mlx5e_priv *priv) { - /* Given the MTU and sync_len, calculates an upper bound for the - * number of DUMP WQEs needed for the TX resync of a record. - */ - return nfrags + DIV_ROUND_UP(sync_len, sq->hw_mtu); } -#else -static inline void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv) +static inline int mlx5e_ktls_set_feature_rx(struct net_device *netdev, bool enable) { + netdev_warn(netdev, "kTLS is not supported\n"); + return -EOPNOTSUPP; } -static inline void -mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq, - struct mlx5e_tx_wqe_info *wi, - u32 *dma_fifo_cc) {} #endif #endif /* __MLX5E_TLS_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c new file mode 100644 index 000000000000..acf6d80a6bb7 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c @@ -0,0 +1,680 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +// Copyright (c) 2019 Mellanox Technologies. + +#include <net/inet6_hashtables.h> +#include "en_accel/en_accel.h" +#include "en_accel/tls.h" +#include "en_accel/ktls_txrx.h" +#include "en_accel/ktls_utils.h" +#include "en_accel/fs_tcp.h" + +struct accel_rule { + struct work_struct work; + struct mlx5e_priv *priv; + struct mlx5_flow_handle *rule; +}; + +#define PROGRESS_PARAMS_WRITE_UNIT 64 +#define PROGRESS_PARAMS_PADDED_SIZE \ + (ALIGN(sizeof(struct mlx5_wqe_tls_progress_params_seg), \ + PROGRESS_PARAMS_WRITE_UNIT)) + +struct mlx5e_ktls_rx_resync_buf { + union { + struct mlx5_wqe_tls_progress_params_seg progress; + u8 pad[PROGRESS_PARAMS_PADDED_SIZE]; + } ____cacheline_aligned_in_smp; + dma_addr_t dma_addr; + struct mlx5e_ktls_offload_context_rx *priv_rx; +}; + +enum { + MLX5E_PRIV_RX_FLAG_DELETING, + MLX5E_NUM_PRIV_RX_FLAGS, +}; + +struct mlx5e_ktls_rx_resync_ctx { + struct tls_offload_resync_async core; + struct work_struct work; + struct mlx5e_priv *priv; + refcount_t refcnt; + __be64 sw_rcd_sn_be; + u32 seq; +}; + +struct mlx5e_ktls_offload_context_rx { + struct tls12_crypto_info_aes_gcm_128 crypto_info; + struct accel_rule rule; + struct sock *sk; + struct mlx5e_rq_stats *stats; + struct completion add_ctx; + u32 tirn; + u32 key_id; + u32 rxq; + DECLARE_BITMAP(flags, MLX5E_NUM_PRIV_RX_FLAGS); + + /* resync */ + struct mlx5e_ktls_rx_resync_ctx resync; +}; + +static int mlx5e_ktls_create_tir(struct mlx5_core_dev *mdev, u32 *tirn, u32 rqtn) +{ + int err, inlen; + void *tirc; + u32 *in; + + inlen = MLX5_ST_SZ_BYTES(create_tir_in); + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) + return -ENOMEM; + + tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); + + MLX5_SET(tirc, tirc, transport_domain, mdev->mlx5e_res.td.tdn); + MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT); + MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8); + MLX5_SET(tirc, tirc, indirect_table, rqtn); + MLX5_SET(tirc, tirc, tls_en, 1); + MLX5_SET(tirc, tirc, self_lb_block, + MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST | + MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST); + + err = mlx5_core_create_tir(mdev, in, tirn); + + kvfree(in); + return err; +} + +static void accel_rule_handle_work(struct work_struct *work) +{ + struct mlx5e_ktls_offload_context_rx *priv_rx; + struct accel_rule *accel_rule; + struct mlx5_flow_handle *rule; + + accel_rule = container_of(work, struct accel_rule, work); + priv_rx = container_of(accel_rule, struct mlx5e_ktls_offload_context_rx, rule); + if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) + goto out; + + rule = mlx5e_accel_fs_add_sk(accel_rule->priv, priv_rx->sk, + priv_rx->tirn, MLX5_FS_DEFAULT_FLOW_TAG); + if (!IS_ERR_OR_NULL(rule)) + accel_rule->rule = rule; +out: + complete(&priv_rx->add_ctx); +} + +static void accel_rule_init(struct accel_rule *rule, struct mlx5e_priv *priv, + struct sock *sk) +{ + INIT_WORK(&rule->work, accel_rule_handle_work); + rule->priv = priv; +} + +static void icosq_fill_wi(struct mlx5e_icosq *sq, u16 pi, + struct mlx5e_icosq_wqe_info *wi) +{ + sq->db.wqe_info[pi] = *wi; +} + +static struct mlx5_wqe_ctrl_seg * +post_static_params(struct mlx5e_icosq *sq, + struct mlx5e_ktls_offload_context_rx *priv_rx) +{ + struct mlx5e_set_tls_static_params_wqe *wqe; + struct mlx5e_icosq_wqe_info wi; + u16 pi, num_wqebbs, room; + + num_wqebbs = MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS; + room = mlx5e_stop_room_for_wqe(num_wqebbs); + if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, room))) + return ERR_PTR(-ENOSPC); + + pi = mlx5e_icosq_get_next_pi(sq, num_wqebbs); + wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi); + mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_rx->crypto_info, + priv_rx->tirn, priv_rx->key_id, + priv_rx->resync.seq, false, + TLS_OFFLOAD_CTX_DIR_RX); + wi = (struct mlx5e_icosq_wqe_info) { + .wqe_type = MLX5E_ICOSQ_WQE_UMR_TLS, + .num_wqebbs = num_wqebbs, + .tls_set_params.priv_rx = priv_rx, + }; + icosq_fill_wi(sq, pi, &wi); + sq->pc += num_wqebbs; + + return &wqe->ctrl; +} + +static struct mlx5_wqe_ctrl_seg * +post_progress_params(struct mlx5e_icosq *sq, + struct mlx5e_ktls_offload_context_rx *priv_rx, + u32 next_record_tcp_sn) +{ + struct mlx5e_set_tls_progress_params_wqe *wqe; + struct mlx5e_icosq_wqe_info wi; + u16 pi, num_wqebbs, room; + + num_wqebbs = MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS; + room = mlx5e_stop_room_for_wqe(num_wqebbs); + if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, room))) + return ERR_PTR(-ENOSPC); + + pi = mlx5e_icosq_get_next_pi(sq, num_wqebbs); + wqe = MLX5E_TLS_FETCH_SET_PROGRESS_PARAMS_WQE(sq, pi); + mlx5e_ktls_build_progress_params(wqe, sq->pc, sq->sqn, priv_rx->tirn, false, + next_record_tcp_sn, + TLS_OFFLOAD_CTX_DIR_RX); + wi = (struct mlx5e_icosq_wqe_info) { + .wqe_type = MLX5E_ICOSQ_WQE_SET_PSV_TLS, + .num_wqebbs = num_wqebbs, + .tls_set_params.priv_rx = priv_rx, + }; + + icosq_fill_wi(sq, pi, &wi); + sq->pc += num_wqebbs; + + return &wqe->ctrl; +} + +static int post_rx_param_wqes(struct mlx5e_channel *c, + struct mlx5e_ktls_offload_context_rx *priv_rx, + u32 next_record_tcp_sn) +{ + struct mlx5_wqe_ctrl_seg *cseg; + struct mlx5e_icosq *sq; + int err; + + err = 0; + sq = &c->async_icosq; + spin_lock(&c->async_icosq_lock); + + cseg = post_static_params(sq, priv_rx); + if (IS_ERR(cseg)) + goto err_out; + cseg = post_progress_params(sq, priv_rx, next_record_tcp_sn); + if (IS_ERR(cseg)) + goto err_out; + + mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg); +unlock: + spin_unlock(&c->async_icosq_lock); + + return err; + +err_out: + priv_rx->stats->tls_resync_req_skip++; + err = PTR_ERR(cseg); + complete(&priv_rx->add_ctx); + goto unlock; +} + +static void +mlx5e_set_ktls_rx_priv_ctx(struct tls_context *tls_ctx, + struct mlx5e_ktls_offload_context_rx *priv_rx) +{ + struct mlx5e_ktls_offload_context_rx **ctx = + __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_RX); + + BUILD_BUG_ON(sizeof(struct mlx5e_ktls_offload_context_rx *) > + TLS_OFFLOAD_CONTEXT_SIZE_RX); + + *ctx = priv_rx; +} + +static struct mlx5e_ktls_offload_context_rx * +mlx5e_get_ktls_rx_priv_ctx(struct tls_context *tls_ctx) +{ + struct mlx5e_ktls_offload_context_rx **ctx = + __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_RX); + + return *ctx; +} + +/* Re-sync */ +/* Runs in work context */ +static struct mlx5_wqe_ctrl_seg * +resync_post_get_progress_params(struct mlx5e_icosq *sq, + struct mlx5e_ktls_offload_context_rx *priv_rx) +{ + struct mlx5e_get_tls_progress_params_wqe *wqe; + struct mlx5e_ktls_rx_resync_buf *buf; + struct mlx5e_icosq_wqe_info wi; + struct mlx5_wqe_ctrl_seg *cseg; + struct mlx5_seg_get_psv *psv; + struct device *pdev; + int err; + u16 pi; + + buf = kzalloc(sizeof(*buf), GFP_KERNEL); + if (unlikely(!buf)) { + err = -ENOMEM; + goto err_out; + } + + pdev = sq->channel->priv->mdev->device; + buf->dma_addr = dma_map_single(pdev, &buf->progress, + PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE); + if (unlikely(dma_mapping_error(pdev, buf->dma_addr))) { + err = -ENOMEM; + goto err_out; + } + + buf->priv_rx = priv_rx; + + BUILD_BUG_ON(MLX5E_KTLS_GET_PROGRESS_WQEBBS != 1); + if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, 1))) { + err = -ENOSPC; + goto err_out; + } + + pi = mlx5e_icosq_get_next_pi(sq, 1); + wqe = MLX5E_TLS_FETCH_GET_PROGRESS_PARAMS_WQE(sq, pi); + +#define GET_PSV_DS_CNT (DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS)) + + cseg = &wqe->ctrl; + cseg->opmod_idx_opcode = + cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_GET_PSV | + (MLX5_OPC_MOD_TLS_TIR_PROGRESS_PARAMS << 24)); + cseg->qpn_ds = + cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | GET_PSV_DS_CNT); + + psv = &wqe->psv; + psv->num_psv = 1 << 4; + psv->l_key = sq->channel->mkey_be; + psv->psv_index[0] = cpu_to_be32(priv_rx->tirn); + psv->va = cpu_to_be64(buf->dma_addr); + + wi = (struct mlx5e_icosq_wqe_info) { + .wqe_type = MLX5E_ICOSQ_WQE_GET_PSV_TLS, + .num_wqebbs = 1, + .tls_get_params.buf = buf, + }; + icosq_fill_wi(sq, pi, &wi); + sq->pc++; + + return cseg; + +err_out: + priv_rx->stats->tls_resync_req_skip++; + return ERR_PTR(err); +} + +/* Function is called with elevated refcount. + * It decreases it only if no WQE is posted. + */ +static void resync_handle_work(struct work_struct *work) +{ + struct mlx5e_ktls_offload_context_rx *priv_rx; + struct mlx5e_ktls_rx_resync_ctx *resync; + struct mlx5_wqe_ctrl_seg *cseg; + struct mlx5e_channel *c; + struct mlx5e_icosq *sq; + struct mlx5_wq_cyc *wq; + + resync = container_of(work, struct mlx5e_ktls_rx_resync_ctx, work); + priv_rx = container_of(resync, struct mlx5e_ktls_offload_context_rx, resync); + + if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) { + refcount_dec(&resync->refcnt); + return; + } + + c = resync->priv->channels.c[priv_rx->rxq]; + sq = &c->async_icosq; + wq = &sq->wq; + + spin_lock(&c->async_icosq_lock); + + cseg = resync_post_get_progress_params(sq, priv_rx); + if (IS_ERR(cseg)) { + refcount_dec(&resync->refcnt); + goto unlock; + } + mlx5e_notify_hw(wq, sq->pc, sq->uar_map, cseg); +unlock: + spin_unlock(&c->async_icosq_lock); +} + +static void resync_init(struct mlx5e_ktls_rx_resync_ctx *resync, + struct mlx5e_priv *priv) +{ + INIT_WORK(&resync->work, resync_handle_work); + resync->priv = priv; + refcount_set(&resync->refcnt, 1); +} + +/* Function can be called with the refcount being either elevated or not. + * It does not affect the refcount. + */ +static int resync_handle_seq_match(struct mlx5e_ktls_offload_context_rx *priv_rx, + struct mlx5e_channel *c) +{ + struct tls12_crypto_info_aes_gcm_128 *info = &priv_rx->crypto_info; + struct mlx5_wqe_ctrl_seg *cseg; + struct mlx5e_icosq *sq; + int err; + + memcpy(info->rec_seq, &priv_rx->resync.sw_rcd_sn_be, sizeof(info->rec_seq)); + err = 0; + + sq = &c->async_icosq; + spin_lock(&c->async_icosq_lock); + + cseg = post_static_params(sq, priv_rx); + if (IS_ERR(cseg)) { + priv_rx->stats->tls_resync_res_skip++; + err = PTR_ERR(cseg); + goto unlock; + } + /* Do not increment priv_rx refcnt, CQE handling is empty */ + mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg); + priv_rx->stats->tls_resync_res_ok++; +unlock: + spin_unlock(&c->async_icosq_lock); + + return err; +} + +/* Function is called with elevated refcount, it decreases it. */ +void mlx5e_ktls_handle_get_psv_completion(struct mlx5e_icosq_wqe_info *wi, + struct mlx5e_icosq *sq) +{ + struct mlx5e_ktls_rx_resync_buf *buf = wi->tls_get_params.buf; + struct mlx5e_ktls_offload_context_rx *priv_rx; + struct mlx5e_ktls_rx_resync_ctx *resync; + u8 tracker_state, auth_state, *ctx; + u32 hw_seq; + + priv_rx = buf->priv_rx; + resync = &priv_rx->resync; + + if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) + goto out; + + dma_sync_single_for_cpu(resync->priv->mdev->device, buf->dma_addr, + PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE); + + ctx = buf->progress.ctx; + tracker_state = MLX5_GET(tls_progress_params, ctx, record_tracker_state); + auth_state = MLX5_GET(tls_progress_params, ctx, auth_state); + if (tracker_state != MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_TRACKING || + auth_state != MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD) { + priv_rx->stats->tls_resync_req_skip++; + goto out; + } + + hw_seq = MLX5_GET(tls_progress_params, ctx, hw_resync_tcp_sn); + tls_offload_rx_resync_async_request_end(priv_rx->sk, cpu_to_be32(hw_seq)); + priv_rx->stats->tls_resync_req_end++; +out: + refcount_dec(&resync->refcnt); + kfree(buf); +} + +/* Runs in NAPI. + * Function elevates the refcount, unless no work is queued. + */ +static bool resync_queue_get_psv(struct sock *sk) +{ + struct mlx5e_ktls_offload_context_rx *priv_rx; + struct mlx5e_ktls_rx_resync_ctx *resync; + + priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_get_ctx(sk)); + if (unlikely(!priv_rx)) + return false; + + if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) + return false; + + resync = &priv_rx->resync; + refcount_inc(&resync->refcnt); + if (unlikely(!queue_work(resync->priv->tls->rx_wq, &resync->work))) + refcount_dec(&resync->refcnt); + + return true; +} + +/* Runs in NAPI */ +static void resync_update_sn(struct mlx5e_rq *rq, struct sk_buff *skb) +{ + struct ethhdr *eth = (struct ethhdr *)(skb->data); + struct net_device *netdev = rq->netdev; + struct sock *sk = NULL; + unsigned int datalen; + struct iphdr *iph; + struct tcphdr *th; + __be32 seq; + int depth = 0; + + __vlan_get_protocol(skb, eth->h_proto, &depth); + iph = (struct iphdr *)(skb->data + depth); + + if (iph->version == 4) { + depth += sizeof(struct iphdr); + th = (void *)iph + sizeof(struct iphdr); + + sk = inet_lookup_established(dev_net(netdev), &tcp_hashinfo, + iph->saddr, th->source, iph->daddr, + th->dest, netdev->ifindex); +#if IS_ENABLED(CONFIG_IPV6) + } else { + struct ipv6hdr *ipv6h = (struct ipv6hdr *)iph; + + depth += sizeof(struct ipv6hdr); + th = (void *)ipv6h + sizeof(struct ipv6hdr); + + sk = __inet6_lookup_established(dev_net(netdev), &tcp_hashinfo, + &ipv6h->saddr, th->source, + &ipv6h->daddr, ntohs(th->dest), + netdev->ifindex, 0); +#endif + } + + depth += sizeof(struct tcphdr); + + if (unlikely(!sk || sk->sk_state == TCP_TIME_WAIT)) + return; + + if (unlikely(!resync_queue_get_psv(sk))) + return; + + skb->sk = sk; + skb->destructor = sock_edemux; + + seq = th->seq; + datalen = skb->len - depth; + tls_offload_rx_resync_async_request_start(sk, seq, datalen); + rq->stats->tls_resync_req_start++; +} + +void mlx5e_ktls_rx_resync(struct net_device *netdev, struct sock *sk, + u32 seq, u8 *rcd_sn) +{ + struct mlx5e_ktls_offload_context_rx *priv_rx; + struct mlx5e_ktls_rx_resync_ctx *resync; + struct mlx5e_priv *priv; + struct mlx5e_channel *c; + + priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_get_ctx(sk)); + if (unlikely(!priv_rx)) + return; + + resync = &priv_rx->resync; + resync->sw_rcd_sn_be = *(__be64 *)rcd_sn; + resync->seq = seq; + + priv = netdev_priv(netdev); + c = priv->channels.c[priv_rx->rxq]; + + resync_handle_seq_match(priv_rx, c); +} + +/* End of resync section */ + +void mlx5e_ktls_handle_rx_skb(struct mlx5e_rq *rq, struct sk_buff *skb, + struct mlx5_cqe64 *cqe, u32 *cqe_bcnt) +{ + struct mlx5e_rq_stats *stats = rq->stats; + + switch (get_cqe_tls_offload(cqe)) { + case CQE_TLS_OFFLOAD_DECRYPTED: + skb->decrypted = 1; + stats->tls_decrypted_packets++; + stats->tls_decrypted_bytes += *cqe_bcnt; + break; + case CQE_TLS_OFFLOAD_RESYNC: + stats->tls_resync_req_pkt++; + resync_update_sn(rq, skb); + break; + default: /* CQE_TLS_OFFLOAD_ERROR: */ + stats->tls_err++; + break; + } +} + +void mlx5e_ktls_handle_ctx_completion(struct mlx5e_icosq_wqe_info *wi) +{ + struct mlx5e_ktls_offload_context_rx *priv_rx = wi->tls_set_params.priv_rx; + struct accel_rule *rule = &priv_rx->rule; + + if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) { + complete(&priv_rx->add_ctx); + return; + } + queue_work(rule->priv->tls->rx_wq, &rule->work); +} + +static int mlx5e_ktls_sk_get_rxq(struct sock *sk) +{ + int rxq = sk_rx_queue_get(sk); + + if (unlikely(rxq == -1)) + rxq = 0; + + return rxq; +} + +int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk, + struct tls_crypto_info *crypto_info, + u32 start_offload_tcp_sn) +{ + struct mlx5e_ktls_offload_context_rx *priv_rx; + struct mlx5e_ktls_rx_resync_ctx *resync; + struct tls_context *tls_ctx; + struct mlx5_core_dev *mdev; + struct mlx5e_priv *priv; + int rxq, err; + u32 rqtn; + + tls_ctx = tls_get_ctx(sk); + priv = netdev_priv(netdev); + mdev = priv->mdev; + priv_rx = kzalloc(sizeof(*priv_rx), GFP_KERNEL); + if (unlikely(!priv_rx)) + return -ENOMEM; + + err = mlx5_ktls_create_key(mdev, crypto_info, &priv_rx->key_id); + if (err) + goto err_create_key; + + priv_rx->crypto_info = + *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info; + + rxq = mlx5e_ktls_sk_get_rxq(sk); + priv_rx->rxq = rxq; + priv_rx->sk = sk; + + priv_rx->stats = &priv->channel_stats[rxq].rq; + mlx5e_set_ktls_rx_priv_ctx(tls_ctx, priv_rx); + + rqtn = priv->direct_tir[rxq].rqt.rqtn; + + err = mlx5e_ktls_create_tir(mdev, &priv_rx->tirn, rqtn); + if (err) + goto err_create_tir; + + init_completion(&priv_rx->add_ctx); + + accel_rule_init(&priv_rx->rule, priv, sk); + resync = &priv_rx->resync; + resync_init(resync, priv); + tls_offload_ctx_rx(tls_ctx)->resync_async = &resync->core; + tls_offload_rx_resync_set_type(sk, TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ_ASYNC); + + err = post_rx_param_wqes(priv->channels.c[rxq], priv_rx, start_offload_tcp_sn); + if (err) + goto err_post_wqes; + + priv_rx->stats->tls_ctx++; + + return 0; + +err_post_wqes: + mlx5_core_destroy_tir(mdev, priv_rx->tirn); +err_create_tir: + mlx5_ktls_destroy_key(mdev, priv_rx->key_id); +err_create_key: + kfree(priv_rx); + return err; +} + +/* Elevated refcount on the resync object means there are + * outstanding operations (uncompleted GET_PSV WQEs) that + * will read the resync / priv_rx objects once completed. + * Wait for them to avoid use-after-free. + */ +static void wait_for_resync(struct net_device *netdev, + struct mlx5e_ktls_rx_resync_ctx *resync) +{ +#define MLX5E_KTLS_RX_RESYNC_TIMEOUT 20000 /* msecs */ + unsigned long exp_time = jiffies + msecs_to_jiffies(MLX5E_KTLS_RX_RESYNC_TIMEOUT); + unsigned int refcnt; + + do { + refcnt = refcount_read(&resync->refcnt); + if (refcnt == 1) + return; + + msleep(20); + } while (time_before(jiffies, exp_time)); + + netdev_warn(netdev, + "Failed waiting for kTLS RX resync refcnt to be released (%u).\n", + refcnt); +} + +void mlx5e_ktls_del_rx(struct net_device *netdev, struct tls_context *tls_ctx) +{ + struct mlx5e_ktls_offload_context_rx *priv_rx; + struct mlx5e_ktls_rx_resync_ctx *resync; + struct mlx5_core_dev *mdev; + struct mlx5e_priv *priv; + + priv = netdev_priv(netdev); + mdev = priv->mdev; + + priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_ctx); + set_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags); + mlx5e_set_ktls_rx_priv_ctx(tls_ctx, NULL); + napi_synchronize(&priv->channels.c[priv_rx->rxq]->napi); + if (!cancel_work_sync(&priv_rx->rule.work)) + /* completion is needed, as the priv_rx in the add flow + * is maintained on the wqe info (wi), not on the socket. + */ + wait_for_completion(&priv_rx->add_ctx); + resync = &priv_rx->resync; + if (cancel_work_sync(&resync->work)) + refcount_dec(&resync->refcnt); + wait_for_resync(netdev, resync); + + priv_rx->stats->tls_del++; + if (priv_rx->rule.rule) + mlx5e_accel_fs_del_sk(priv_rx->rule.rule); + + mlx5_core_destroy_tir(mdev, priv_rx->tirn); + mlx5_ktls_destroy_key(mdev, priv_rx->key_id); + kfree(priv_rx); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c index 3cd78d9503c1..f4861545b236 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c @@ -1,105 +1,139 @@ // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB // Copyright (c) 2019 Mellanox Technologies. -#include <linux/tls.h> -#include "en.h" -#include "en/txrx.h" -#include "en_accel/ktls.h" +#include "en_accel/ktls_txrx.h" +#include "en_accel/ktls_utils.h" -enum { - MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2 = 0x2, +struct mlx5e_dump_wqe { + struct mlx5_wqe_ctrl_seg ctrl; + struct mlx5_wqe_data_seg data; }; -enum { - MLX5E_ENCRYPTION_STANDARD_TLS = 0x1, -}; +#define MLX5E_KTLS_DUMP_WQEBBS \ + (DIV_ROUND_UP(sizeof(struct mlx5e_dump_wqe), MLX5_SEND_WQE_BB)) -#define EXTRACT_INFO_FIELDS do { \ - salt = info->salt; \ - rec_seq = info->rec_seq; \ - salt_sz = sizeof(info->salt); \ - rec_seq_sz = sizeof(info->rec_seq); \ -} while (0) +static u8 +mlx5e_ktls_dumps_num_wqes(struct mlx5e_txqsq *sq, unsigned int nfrags, + unsigned int sync_len) +{ + /* Given the MTU and sync_len, calculates an upper bound for the + * number of DUMP WQEs needed for the TX resync of a record. + */ + return nfrags + DIV_ROUND_UP(sync_len, sq->hw_mtu); +} -static void -fill_static_params_ctx(void *ctx, struct mlx5e_ktls_offload_context_tx *priv_tx) +u16 mlx5e_ktls_get_stop_room(struct mlx5e_txqsq *sq) { - struct tls12_crypto_info_aes_gcm_128 *info = &priv_tx->crypto_info; - char *initial_rn, *gcm_iv; - u16 salt_sz, rec_seq_sz; - char *salt, *rec_seq; - u8 tls_version; + u16 num_dumps, stop_room = 0; + + num_dumps = mlx5e_ktls_dumps_num_wqes(sq, MAX_SKB_FRAGS, TLS_MAX_PAYLOAD_SIZE); - EXTRACT_INFO_FIELDS; + stop_room += mlx5e_stop_room_for_wqe(MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS); + stop_room += mlx5e_stop_room_for_wqe(MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS); + stop_room += num_dumps * mlx5e_stop_room_for_wqe(MLX5E_KTLS_DUMP_WQEBBS); - gcm_iv = MLX5_ADDR_OF(tls_static_params, ctx, gcm_iv); - initial_rn = MLX5_ADDR_OF(tls_static_params, ctx, initial_record_number); + return stop_room; +} + +static int mlx5e_ktls_create_tis(struct mlx5_core_dev *mdev, u32 *tisn) +{ + u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; + void *tisc; - memcpy(gcm_iv, salt, salt_sz); - memcpy(initial_rn, rec_seq, rec_seq_sz); + tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); - tls_version = MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2; + MLX5_SET(tisc, tisc, tls_en, 1); - MLX5_SET(tls_static_params, ctx, tls_version, tls_version); - MLX5_SET(tls_static_params, ctx, const_1, 1); - MLX5_SET(tls_static_params, ctx, const_2, 2); - MLX5_SET(tls_static_params, ctx, encryption_standard, - MLX5E_ENCRYPTION_STANDARD_TLS); - MLX5_SET(tls_static_params, ctx, dek_index, priv_tx->key_id); + return mlx5e_create_tis(mdev, in, tisn); } +struct mlx5e_ktls_offload_context_tx { + struct tls_offload_context_tx *tx_ctx; + struct tls12_crypto_info_aes_gcm_128 crypto_info; + u32 expected_seq; + u32 tisn; + u32 key_id; + bool ctx_post_pending; +}; + static void -build_static_params(struct mlx5e_umr_wqe *wqe, u16 pc, u32 sqn, - struct mlx5e_ktls_offload_context_tx *priv_tx, - bool fence) +mlx5e_set_ktls_tx_priv_ctx(struct tls_context *tls_ctx, + struct mlx5e_ktls_offload_context_tx *priv_tx) { - struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; - struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl; + struct mlx5e_ktls_offload_context_tx **ctx = + __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_TX); -#define STATIC_PARAMS_DS_CNT \ - DIV_ROUND_UP(MLX5E_KTLS_STATIC_UMR_WQE_SZ, MLX5_SEND_WQE_DS) + BUILD_BUG_ON(sizeof(struct mlx5e_ktls_offload_context_tx *) > + TLS_OFFLOAD_CONTEXT_SIZE_TX); - cseg->opmod_idx_opcode = cpu_to_be32((pc << 8) | MLX5_OPCODE_UMR | - (MLX5_OPC_MOD_TLS_TIS_STATIC_PARAMS << 24)); - cseg->qpn_ds = cpu_to_be32((sqn << MLX5_WQE_CTRL_QPN_SHIFT) | - STATIC_PARAMS_DS_CNT); - cseg->fm_ce_se = fence ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0; - cseg->tisn = cpu_to_be32(priv_tx->tisn << 8); + *ctx = priv_tx; +} - ucseg->flags = MLX5_UMR_INLINE; - ucseg->bsf_octowords = cpu_to_be16(MLX5_ST_SZ_BYTES(tls_static_params) / 16); +static struct mlx5e_ktls_offload_context_tx * +mlx5e_get_ktls_tx_priv_ctx(struct tls_context *tls_ctx) +{ + struct mlx5e_ktls_offload_context_tx **ctx = + __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_TX); - fill_static_params_ctx(wqe->tls_static_params_ctx, priv_tx); + return *ctx; } -static void -fill_progress_params_ctx(void *ctx, struct mlx5e_ktls_offload_context_tx *priv_tx) +int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk, + struct tls_crypto_info *crypto_info, u32 start_offload_tcp_sn) { - MLX5_SET(tls_progress_params, ctx, tisn, priv_tx->tisn); - MLX5_SET(tls_progress_params, ctx, record_tracker_state, - MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START); - MLX5_SET(tls_progress_params, ctx, auth_state, - MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD); + struct mlx5e_ktls_offload_context_tx *priv_tx; + struct tls_context *tls_ctx; + struct mlx5_core_dev *mdev; + struct mlx5e_priv *priv; + int err; + + tls_ctx = tls_get_ctx(sk); + priv = netdev_priv(netdev); + mdev = priv->mdev; + + priv_tx = kzalloc(sizeof(*priv_tx), GFP_KERNEL); + if (!priv_tx) + return -ENOMEM; + + err = mlx5_ktls_create_key(mdev, crypto_info, &priv_tx->key_id); + if (err) + goto err_create_key; + + priv_tx->expected_seq = start_offload_tcp_sn; + priv_tx->crypto_info = + *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info; + priv_tx->tx_ctx = tls_offload_ctx_tx(tls_ctx); + + mlx5e_set_ktls_tx_priv_ctx(tls_ctx, priv_tx); + + err = mlx5e_ktls_create_tis(mdev, &priv_tx->tisn); + if (err) + goto err_create_tis; + + priv_tx->ctx_post_pending = true; + + return 0; + +err_create_tis: + mlx5_ktls_destroy_key(mdev, priv_tx->key_id); +err_create_key: + kfree(priv_tx); + return err; } -static void -build_progress_params(struct mlx5e_tx_wqe *wqe, u16 pc, u32 sqn, - struct mlx5e_ktls_offload_context_tx *priv_tx, - bool fence) +void mlx5e_ktls_del_tx(struct net_device *netdev, struct tls_context *tls_ctx) { - struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; - -#define PROGRESS_PARAMS_DS_CNT \ - DIV_ROUND_UP(MLX5E_KTLS_PROGRESS_WQE_SZ, MLX5_SEND_WQE_DS) + struct mlx5e_ktls_offload_context_tx *priv_tx; + struct mlx5_core_dev *mdev; + struct mlx5e_priv *priv; - cseg->opmod_idx_opcode = - cpu_to_be32((pc << 8) | MLX5_OPCODE_SET_PSV | - (MLX5_OPC_MOD_TLS_TIS_PROGRESS_PARAMS << 24)); - cseg->qpn_ds = cpu_to_be32((sqn << MLX5_WQE_CTRL_QPN_SHIFT) | - PROGRESS_PARAMS_DS_CNT); - cseg->fm_ce_se = fence ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0; + priv_tx = mlx5e_get_ktls_tx_priv_ctx(tls_ctx); + priv = netdev_priv(netdev); + mdev = priv->mdev; - fill_progress_params_ctx(wqe->tls_progress_params_ctx, priv_tx); + mlx5e_destroy_tis(mdev, priv_tx->tisn); + mlx5_ktls_destroy_key(mdev, priv_tx->key_id); + kfree(priv_tx); } static void tx_fill_wi(struct mlx5e_txqsq *sq, @@ -115,11 +149,6 @@ static void tx_fill_wi(struct mlx5e_txqsq *sq, }; } -void mlx5e_ktls_tx_offload_set_pending(struct mlx5e_ktls_offload_context_tx *priv_tx) -{ - priv_tx->ctx_post_pending = true; -} - static bool mlx5e_ktls_tx_offload_test_and_clear_pending(struct mlx5e_ktls_offload_context_tx *priv_tx) { @@ -135,12 +164,15 @@ post_static_params(struct mlx5e_txqsq *sq, struct mlx5e_ktls_offload_context_tx *priv_tx, bool fence) { - u16 pi, num_wqebbs = MLX5E_KTLS_STATIC_WQEBBS; - struct mlx5e_umr_wqe *umr_wqe; + struct mlx5e_set_tls_static_params_wqe *wqe; + u16 pi, num_wqebbs; + num_wqebbs = MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS; pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs); - umr_wqe = MLX5E_TLS_FETCH_UMR_WQE(sq, pi); - build_static_params(umr_wqe, sq->pc, sq->sqn, priv_tx, fence); + wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi); + mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_tx->crypto_info, + priv_tx->tisn, priv_tx->key_id, 0, fence, + TLS_OFFLOAD_CTX_DIR_TX); tx_fill_wi(sq, pi, num_wqebbs, 0, NULL); sq->pc += num_wqebbs; } @@ -150,12 +182,14 @@ post_progress_params(struct mlx5e_txqsq *sq, struct mlx5e_ktls_offload_context_tx *priv_tx, bool fence) { - u16 pi, num_wqebbs = MLX5E_KTLS_PROGRESS_WQEBBS; - struct mlx5e_tx_wqe *wqe; + struct mlx5e_set_tls_progress_params_wqe *wqe; + u16 pi, num_wqebbs; + num_wqebbs = MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS; pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs); - wqe = MLX5E_TLS_FETCH_PROGRESS_WQE(sq, pi); - build_progress_params(wqe, sq->pc, sq->sqn, priv_tx, fence); + wqe = MLX5E_TLS_FETCH_SET_PROGRESS_PARAMS_WQE(sq, pi); + mlx5e_ktls_build_progress_params(wqe, sq->pc, sq->sqn, priv_tx->tisn, fence, 0, + TLS_OFFLOAD_CTX_DIR_TX); tx_fill_wi(sq, pi, num_wqebbs, 0, NULL); sq->pc += num_wqebbs; } @@ -284,7 +318,7 @@ tx_post_resync_dump(struct mlx5e_txqsq *sq, skb_frag_t *frag, u32 tisn, bool fir cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_DUMP); cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); - cseg->tisn = cpu_to_be32(tisn << 8); + cseg->tis_tir_num = cpu_to_be32(tisn << 8); cseg->fm_ce_se = first ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0; fsz = skb_frag_size(frag); @@ -436,7 +470,7 @@ bool mlx5e_ktls_handle_tx_skb(struct tls_context *tls_ctx, struct mlx5e_txqsq *s if (likely(!skb->decrypted)) goto out; WARN_ON_ONCE(1); - /* fall-through */ + fallthrough; case MLX5E_KTLS_SYNC_FAIL: goto err_out; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c new file mode 100644 index 000000000000..ac29aeb8af49 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ + +#include "en_accel/ktls_txrx.h" +#include "en_accel/ktls_utils.h" + +enum { + MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2 = 0x2, +}; + +enum { + MLX5E_ENCRYPTION_STANDARD_TLS = 0x1, +}; + +#define EXTRACT_INFO_FIELDS do { \ + salt = info->salt; \ + rec_seq = info->rec_seq; \ + salt_sz = sizeof(info->salt); \ + rec_seq_sz = sizeof(info->rec_seq); \ +} while (0) + +static void +fill_static_params(struct mlx5_wqe_tls_static_params_seg *params, + struct tls12_crypto_info_aes_gcm_128 *info, + u32 key_id, u32 resync_tcp_sn) +{ + char *initial_rn, *gcm_iv; + u16 salt_sz, rec_seq_sz; + char *salt, *rec_seq; + u8 tls_version; + u8 *ctx; + + ctx = params->ctx; + + EXTRACT_INFO_FIELDS; + + gcm_iv = MLX5_ADDR_OF(tls_static_params, ctx, gcm_iv); + initial_rn = MLX5_ADDR_OF(tls_static_params, ctx, initial_record_number); + + memcpy(gcm_iv, salt, salt_sz); + memcpy(initial_rn, rec_seq, rec_seq_sz); + + tls_version = MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2; + + MLX5_SET(tls_static_params, ctx, tls_version, tls_version); + MLX5_SET(tls_static_params, ctx, const_1, 1); + MLX5_SET(tls_static_params, ctx, const_2, 2); + MLX5_SET(tls_static_params, ctx, encryption_standard, + MLX5E_ENCRYPTION_STANDARD_TLS); + MLX5_SET(tls_static_params, ctx, resync_tcp_sn, resync_tcp_sn); + MLX5_SET(tls_static_params, ctx, dek_index, key_id); +} + +void +mlx5e_ktls_build_static_params(struct mlx5e_set_tls_static_params_wqe *wqe, + u16 pc, u32 sqn, + struct tls12_crypto_info_aes_gcm_128 *info, + u32 tis_tir_num, u32 key_id, u32 resync_tcp_sn, + bool fence, enum tls_offload_ctx_dir direction) +{ + struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl; + struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; + u8 opmod = direction == TLS_OFFLOAD_CTX_DIR_TX ? + MLX5_OPC_MOD_TLS_TIS_STATIC_PARAMS : + MLX5_OPC_MOD_TLS_TIR_STATIC_PARAMS; + +#define STATIC_PARAMS_DS_CNT DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS) + + cseg->opmod_idx_opcode = cpu_to_be32((pc << 8) | MLX5_OPCODE_UMR | (opmod << 24)); + cseg->qpn_ds = cpu_to_be32((sqn << MLX5_WQE_CTRL_QPN_SHIFT) | + STATIC_PARAMS_DS_CNT); + cseg->fm_ce_se = fence ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0; + cseg->tis_tir_num = cpu_to_be32(tis_tir_num << 8); + + ucseg->flags = MLX5_UMR_INLINE; + ucseg->bsf_octowords = cpu_to_be16(MLX5_ST_SZ_BYTES(tls_static_params) / 16); + + fill_static_params(&wqe->params, info, key_id, resync_tcp_sn); +} + +static void +fill_progress_params(struct mlx5_wqe_tls_progress_params_seg *params, u32 tis_tir_num, + u32 next_record_tcp_sn) +{ + u8 *ctx = params->ctx; + + params->tis_tir_num = cpu_to_be32(tis_tir_num); + + MLX5_SET(tls_progress_params, ctx, next_record_tcp_sn, + next_record_tcp_sn); + MLX5_SET(tls_progress_params, ctx, record_tracker_state, + MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START); + MLX5_SET(tls_progress_params, ctx, auth_state, + MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD); +} + +void +mlx5e_ktls_build_progress_params(struct mlx5e_set_tls_progress_params_wqe *wqe, + u16 pc, u32 sqn, + u32 tis_tir_num, bool fence, + u32 next_record_tcp_sn, + enum tls_offload_ctx_dir direction) +{ + struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; + u8 opmod = direction == TLS_OFFLOAD_CTX_DIR_TX ? + MLX5_OPC_MOD_TLS_TIS_PROGRESS_PARAMS : + MLX5_OPC_MOD_TLS_TIR_PROGRESS_PARAMS; + +#define PROGRESS_PARAMS_DS_CNT DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS) + + cseg->opmod_idx_opcode = + cpu_to_be32((pc << 8) | MLX5_OPCODE_SET_PSV | (opmod << 24)); + cseg->qpn_ds = cpu_to_be32((sqn << MLX5_WQE_CTRL_QPN_SHIFT) | + PROGRESS_PARAMS_DS_CNT); + cseg->fm_ce_se = fence ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0; + + fill_progress_params(&wqe->params, tis_tir_num, next_record_tcp_sn); +} + diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h new file mode 100644 index 000000000000..ff4c740af10b --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ + +#ifndef __MLX5E_KTLS_TXRX_H__ +#define __MLX5E_KTLS_TXRX_H__ + +#ifdef CONFIG_MLX5_EN_TLS + +#include <net/tls.h> +#include "en.h" +#include "en/txrx.h" + +struct mlx5e_accel_tx_tls_state { + u32 tls_tisn; +}; + +u16 mlx5e_ktls_get_stop_room(struct mlx5e_txqsq *sq); + +bool mlx5e_ktls_handle_tx_skb(struct tls_context *tls_ctx, struct mlx5e_txqsq *sq, + struct sk_buff *skb, int datalen, + struct mlx5e_accel_tx_tls_state *state); +void mlx5e_ktls_handle_rx_skb(struct mlx5e_rq *rq, struct sk_buff *skb, + struct mlx5_cqe64 *cqe, u32 *cqe_bcnt); + +void mlx5e_ktls_handle_ctx_completion(struct mlx5e_icosq_wqe_info *wi); +void mlx5e_ktls_handle_get_psv_completion(struct mlx5e_icosq_wqe_info *wi, + struct mlx5e_icosq *sq); + +void mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq, + struct mlx5e_tx_wqe_info *wi, + u32 *dma_fifo_cc); +#else +static inline void +mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq, + struct mlx5e_tx_wqe_info *wi, + u32 *dma_fifo_cc) +{ +} + +#endif /* CONFIG_MLX5_EN_TLS */ + +#endif /* __MLX5E_TLS_TXRX_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h new file mode 100644 index 000000000000..e5c180f2403b --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ + +#ifndef __MLX5E_KTLS_UTILS_H__ +#define __MLX5E_KTLS_UTILS_H__ + +#include <net/tls.h> +#include "en.h" +#include "accel/tls.h" + +enum { + MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD = 0, + MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_OFFLOAD = 1, + MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_AUTHENTICATION = 2, +}; + +enum { + MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START = 0, + MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_TRACKING = 1, + MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_SEARCHING = 2, +}; + +int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk, + struct tls_crypto_info *crypto_info, u32 start_offload_tcp_sn); +void mlx5e_ktls_del_tx(struct net_device *netdev, struct tls_context *tls_ctx); +int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk, + struct tls_crypto_info *crypto_info, u32 start_offload_tcp_sn); +void mlx5e_ktls_del_rx(struct net_device *netdev, struct tls_context *tls_ctx); +void mlx5e_ktls_rx_resync(struct net_device *netdev, struct sock *sk, u32 seq, u8 *rcd_sn); + +struct mlx5e_set_tls_static_params_wqe { + struct mlx5_wqe_ctrl_seg ctrl; + struct mlx5_wqe_umr_ctrl_seg uctrl; + struct mlx5_mkey_seg mkc; + struct mlx5_wqe_tls_static_params_seg params; +}; + +struct mlx5e_set_tls_progress_params_wqe { + struct mlx5_wqe_ctrl_seg ctrl; + struct mlx5_wqe_tls_progress_params_seg params; +}; + +struct mlx5e_get_tls_progress_params_wqe { + struct mlx5_wqe_ctrl_seg ctrl; + struct mlx5_seg_get_psv psv; +}; + +#define MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS \ + (DIV_ROUND_UP(sizeof(struct mlx5e_set_tls_static_params_wqe), MLX5_SEND_WQE_BB)) + +#define MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS \ + (DIV_ROUND_UP(sizeof(struct mlx5e_set_tls_progress_params_wqe), MLX5_SEND_WQE_BB)) + +#define MLX5E_KTLS_GET_PROGRESS_WQEBBS \ + (DIV_ROUND_UP(sizeof(struct mlx5e_get_tls_progress_params_wqe), MLX5_SEND_WQE_BB)) + +#define MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi) \ + ((struct mlx5e_set_tls_static_params_wqe *)\ + mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_set_tls_static_params_wqe))) + +#define MLX5E_TLS_FETCH_SET_PROGRESS_PARAMS_WQE(sq, pi) \ + ((struct mlx5e_set_tls_progress_params_wqe *)\ + mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_set_tls_progress_params_wqe))) + +#define MLX5E_TLS_FETCH_GET_PROGRESS_PARAMS_WQE(sq, pi) \ + ((struct mlx5e_get_tls_progress_params_wqe *)\ + mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_get_tls_progress_params_wqe))) + +#define MLX5E_TLS_FETCH_DUMP_WQE(sq, pi) \ + ((struct mlx5e_dump_wqe *)\ + mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_dump_wqe))) + +void +mlx5e_ktls_build_static_params(struct mlx5e_set_tls_static_params_wqe *wqe, + u16 pc, u32 sqn, + struct tls12_crypto_info_aes_gcm_128 *info, + u32 tis_tir_num, u32 key_id, u32 resync_tcp_sn, + bool fence, enum tls_offload_ctx_dir direction); +void +mlx5e_ktls_build_progress_params(struct mlx5e_set_tls_progress_params_wqe *wqe, + u16 pc, u32 sqn, + u32 tis_tir_num, bool fence, + u32 next_record_tcp_sn, + enum tls_offload_ctx_dir direction); + +#endif /* __MLX5E_TLS_UTILS_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c index 1fbb5a90cb38..fee991f5ee7c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c @@ -197,6 +197,7 @@ void mlx5e_tls_build_netdev(struct mlx5e_priv *priv) return; } + /* FPGA */ if (!mlx5_accel_is_tls_device(priv->mdev)) return; @@ -221,11 +222,21 @@ void mlx5e_tls_build_netdev(struct mlx5e_priv *priv) int mlx5e_tls_init(struct mlx5e_priv *priv) { - struct mlx5e_tls *tls = kzalloc(sizeof(*tls), GFP_KERNEL); + struct mlx5e_tls *tls; + if (!mlx5_accel_is_tls_device(priv->mdev)) + return 0; + + tls = kzalloc(sizeof(*tls), GFP_KERNEL); if (!tls) return -ENOMEM; + tls->rx_wq = create_singlethread_workqueue("mlx5e_tls_rx"); + if (!tls->rx_wq) { + kfree(tls); + return -ENOMEM; + } + priv->tls = tls; return 0; } @@ -237,20 +248,7 @@ void mlx5e_tls_cleanup(struct mlx5e_priv *priv) if (!tls) return; + destroy_workqueue(tls->rx_wq); kfree(tls); priv->tls = NULL; } - -u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq) -{ - struct mlx5_core_dev *mdev = sq->channel->mdev; - - if (!mlx5_accel_is_tls_device(mdev)) - return 0; - - if (MLX5_CAP_GEN(mdev, tls_tx)) - return mlx5e_ktls_get_stop_room(sq); - - /* Resync SKB. */ - return mlx5e_stop_room_for_wqe(MLX5_SEND_WQE_MAX_WQEBBS); -} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h index 9219bdb2786e..bd270a85c804 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h @@ -53,6 +53,7 @@ struct mlx5e_tls_sw_stats { struct mlx5e_tls { struct mlx5e_tls_sw_stats sw_stats; + struct workqueue_struct *rx_wq; }; struct mlx5e_tls_offload_context_tx { @@ -86,6 +87,11 @@ mlx5e_get_tls_rx_context(struct tls_context *tls_ctx) base); } +static inline bool mlx5e_is_tls_on(struct mlx5e_priv *priv) +{ + return priv->tls; +} + void mlx5e_tls_build_netdev(struct mlx5e_priv *priv); int mlx5e_tls_init(struct mlx5e_priv *priv); void mlx5e_tls_cleanup(struct mlx5e_priv *priv); @@ -94,8 +100,6 @@ int mlx5e_tls_get_count(struct mlx5e_priv *priv); int mlx5e_tls_get_strings(struct mlx5e_priv *priv, uint8_t *data); int mlx5e_tls_get_stats(struct mlx5e_priv *priv, u64 *data); -u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq); - #else static inline void mlx5e_tls_build_netdev(struct mlx5e_priv *priv) @@ -104,17 +108,13 @@ static inline void mlx5e_tls_build_netdev(struct mlx5e_priv *priv) mlx5e_ktls_build_netdev(priv); } +static inline bool mlx5e_is_tls_on(struct mlx5e_priv *priv) { return false; } static inline int mlx5e_tls_init(struct mlx5e_priv *priv) { return 0; } static inline void mlx5e_tls_cleanup(struct mlx5e_priv *priv) { } static inline int mlx5e_tls_get_count(struct mlx5e_priv *priv) { return 0; } static inline int mlx5e_tls_get_strings(struct mlx5e_priv *priv, uint8_t *data) { return 0; } static inline int mlx5e_tls_get_stats(struct mlx5e_priv *priv, u64 *data) { return 0; } -static inline u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq) -{ - return 0; -} - #endif #endif /* __MLX5E_TLS_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c index 05454a843b28..b0c31d49ff8d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c @@ -278,9 +278,10 @@ bool mlx5e_tls_handle_tx_skb(struct net_device *netdev, struct mlx5e_txqsq *sq, if (WARN_ON_ONCE(tls_ctx->netdev != netdev)) goto err_out; - if (MLX5_CAP_GEN(sq->channel->mdev, tls_tx)) + if (mlx5_accel_is_ktls_tx(sq->channel->mdev)) return mlx5e_ktls_handle_tx_skb(tls_ctx, sq, skb, datalen, state); + /* FPGA */ skb_seq = ntohl(tcp_hdr(skb)->seq); context = mlx5e_get_tls_tx_context(tls_ctx); expected_seq = context->expected_seq; @@ -305,7 +306,7 @@ err_out: void mlx5e_tls_handle_tx_wqe(struct mlx5e_txqsq *sq, struct mlx5_wqe_ctrl_seg *cseg, struct mlx5e_accel_tx_tls_state *state) { - cseg->tisn = cpu_to_be32(state->tls_tisn << 8); + cseg->tis_tir_num = cpu_to_be32(state->tls_tisn << 8); } static int tls_update_resync_sn(struct net_device *netdev, @@ -354,15 +355,13 @@ out: return 0; } -void mlx5e_tls_handle_rx_skb(struct net_device *netdev, struct sk_buff *skb, - u32 *cqe_bcnt) +/* FPGA tls rx handler */ +void mlx5e_tls_handle_rx_skb_metadata(struct mlx5e_rq *rq, struct sk_buff *skb, + u32 *cqe_bcnt) { struct mlx5e_tls_metadata *mdata; struct mlx5e_priv *priv; - if (!is_metadata_hdr_valid(skb)) - return; - /* Use the metadata */ mdata = (struct mlx5e_tls_metadata *)(skb->data + ETH_HLEN); switch (mdata->content.recv.syndrome) { @@ -370,13 +369,13 @@ void mlx5e_tls_handle_rx_skb(struct net_device *netdev, struct sk_buff *skb, skb->decrypted = 1; break; case SYNDROM_RESYNC_REQUEST: - tls_update_resync_sn(netdev, skb, mdata); - priv = netdev_priv(netdev); + tls_update_resync_sn(rq->netdev, skb, mdata); + priv = netdev_priv(rq->netdev); atomic64_inc(&priv->tls->sw_stats.rx_tls_resync_request); break; case SYNDROM_AUTH_FAILED: /* Authentication failure will be observed and verified by kTLS */ - priv = netdev_priv(netdev); + priv = netdev_priv(rq->netdev); atomic64_inc(&priv->tls->sw_stats.rx_tls_auth_fail); break; default: @@ -387,3 +386,18 @@ void mlx5e_tls_handle_rx_skb(struct net_device *netdev, struct sk_buff *skb, remove_metadata_hdr(skb); *cqe_bcnt -= MLX5E_METADATA_ETHER_LEN; } + +u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq) +{ + struct mlx5_core_dev *mdev = sq->channel->mdev; + + if (!mlx5_accel_is_tls_device(mdev)) + return 0; + + if (mlx5_accel_is_ktls_device(mdev)) + return mlx5e_ktls_get_stop_room(sq); + + /* FPGA */ + /* Resync SKB. */ + return mlx5e_stop_room_for_wqe(MLX5_SEND_WQE_MAX_WQEBBS); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h index a50d0394df0a..5f162ad2ee8f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h @@ -34,23 +34,47 @@ #ifndef __MLX5E_TLS_RXTX_H__ #define __MLX5E_TLS_RXTX_H__ +#include "accel/accel.h" +#include "en_accel/ktls_txrx.h" + #ifdef CONFIG_MLX5_EN_TLS #include <linux/skbuff.h> #include "en.h" #include "en/txrx.h" -struct mlx5e_accel_tx_tls_state { - u32 tls_tisn; -}; +u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq); bool mlx5e_tls_handle_tx_skb(struct net_device *netdev, struct mlx5e_txqsq *sq, struct sk_buff *skb, struct mlx5e_accel_tx_tls_state *state); void mlx5e_tls_handle_tx_wqe(struct mlx5e_txqsq *sq, struct mlx5_wqe_ctrl_seg *cseg, struct mlx5e_accel_tx_tls_state *state); -void mlx5e_tls_handle_rx_skb(struct net_device *netdev, struct sk_buff *skb, - u32 *cqe_bcnt); +void mlx5e_tls_handle_rx_skb_metadata(struct mlx5e_rq *rq, struct sk_buff *skb, + u32 *cqe_bcnt); + +static inline void +mlx5e_tls_handle_rx_skb(struct mlx5e_rq *rq, struct sk_buff *skb, + struct mlx5_cqe64 *cqe, u32 *cqe_bcnt) +{ + if (unlikely(get_cqe_tls_offload(cqe))) /* cqe bit indicates a TLS device */ + return mlx5e_ktls_handle_rx_skb(rq, skb, cqe, cqe_bcnt); + + if (unlikely(test_bit(MLX5E_RQ_STATE_FPGA_TLS, &rq->state) && is_metadata_hdr_valid(skb))) + return mlx5e_tls_handle_rx_skb_metadata(rq, skb, cqe_bcnt); +} + +#else + +static inline bool +mlx5e_accel_is_tls(struct mlx5_cqe64 *cqe, struct sk_buff *skb) { return false; } +static inline void +mlx5e_tls_handle_rx_skb(struct mlx5e_rq *rq, struct sk_buff *skb, + struct mlx5_cqe64 *cqe, u32 *cqe_bcnt) {} +static inline u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq) +{ + return 0; +} #endif /* CONFIG_MLX5_EN_TLS */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c index 014639ea06e3..39475f6565c7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c @@ -90,23 +90,15 @@ static enum mlx5e_traffic_types arfs_get_tt(enum arfs_type type) static int arfs_disable(struct mlx5e_priv *priv) { - struct mlx5_flow_destination dest = {}; - struct mlx5e_tir *tir = priv->indir_tir; - int err = 0; - int tt; - int i; + int err, i; - dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; for (i = 0; i < ARFS_NUM_TYPES; i++) { - dest.tir_num = tir[i].tirn; - tt = arfs_get_tt(i); - /* Modify ttc rules destination to bypass the aRFS tables*/ - err = mlx5_modify_rule_destination(priv->fs.ttc.rules[tt], - &dest, NULL); + /* Modify ttc rules destination back to their default */ + err = mlx5e_ttc_fwd_default_dest(priv, arfs_get_tt(i)); if (err) { netdev_err(priv->netdev, - "%s: modify ttc destination failed\n", - __func__); + "%s: modify ttc[%d] default destination failed, err(%d)\n", + __func__, arfs_get_tt(i), err); return err; } } @@ -125,21 +117,17 @@ int mlx5e_arfs_disable(struct mlx5e_priv *priv) int mlx5e_arfs_enable(struct mlx5e_priv *priv) { struct mlx5_flow_destination dest = {}; - int err = 0; - int tt; - int i; + int err, i; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; for (i = 0; i < ARFS_NUM_TYPES; i++) { dest.ft = priv->fs.arfs.arfs_tables[i].ft.t; - tt = arfs_get_tt(i); /* Modify ttc rules destination to point on the aRFS FTs */ - err = mlx5_modify_rule_destination(priv->fs.ttc.rules[tt], - &dest, NULL); + err = mlx5e_ttc_fwd_dest(priv, arfs_get_tt(i), &dest); if (err) { netdev_err(priv->netdev, - "%s: modify ttc destination failed err=%d\n", - __func__, err); + "%s: modify ttc[%d] dest to arfs, failed err(%d)\n", + __func__, arfs_get_tt(i), err); arfs_disable(priv); return err; } @@ -186,8 +174,10 @@ static int arfs_add_default_rule(struct mlx5e_priv *priv, return -EINVAL; } + /* FIXME: Must use mlx5e_ttc_get_default_dest(), + * but can't since TTC default is not setup yet ! + */ dest.tir_num = tir[tt].tirn; - arfs_t->default_rule = mlx5_add_flow_rules(arfs_t->ft.t, NULL, &flow_act, &dest, 1); @@ -220,7 +210,7 @@ static int arfs_create_groups(struct mlx5e_flow_table *ft, sizeof(*ft->g), GFP_KERNEL); in = kvzalloc(inlen, GFP_KERNEL); if (!in || !ft->g) { - kvfree(ft->g); + kfree(ft->g); kvfree(in); return -ENOMEM; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c index 1e42c7ae621b..a6cf008057b5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c @@ -60,6 +60,16 @@ void mlx5e_destroy_tir(struct mlx5_core_dev *mdev, mutex_unlock(&mdev->mlx5e_res.td.list_lock); } +void mlx5e_mkey_set_relaxed_ordering(struct mlx5_core_dev *mdev, void *mkc) +{ + bool ro_pci_enable = pcie_relaxed_ordering_enabled(mdev->pdev); + bool ro_write = MLX5_CAP_GEN(mdev, relaxed_ordering_write); + bool ro_read = MLX5_CAP_GEN(mdev, relaxed_ordering_read); + + MLX5_SET(mkc, mkc, relaxed_ordering_read, ro_pci_enable && ro_read); + MLX5_SET(mkc, mkc, relaxed_ordering_write, ro_pci_enable && ro_write); +} + static int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, struct mlx5_core_mkey *mkey) { @@ -76,7 +86,7 @@ static int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA); MLX5_SET(mkc, mkc, lw, 1); MLX5_SET(mkc, mkc, lr, 1); - + mlx5e_mkey_set_relaxed_ordering(mdev, mkc); MLX5_SET(mkc, mkc, pd, pdn); MLX5_SET(mkc, mkc, length64, 1); MLX5_SET(mkc, mkc, qpn, 0xffffff); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index c2464c349117..08270987c506 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -194,6 +194,24 @@ void mlx5e_build_ptys2ethtool_map(void) ETHTOOL_LINK_MODE_200000baseLR4_ER4_FR4_Full_BIT, ETHTOOL_LINK_MODE_200000baseDR4_Full_BIT, ETHTOOL_LINK_MODE_200000baseCR4_Full_BIT); + MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_100GAUI_1_100GBASE_CR_KR, ext, + ETHTOOL_LINK_MODE_100000baseKR_Full_BIT, + ETHTOOL_LINK_MODE_100000baseSR_Full_BIT, + ETHTOOL_LINK_MODE_100000baseLR_ER_FR_Full_BIT, + ETHTOOL_LINK_MODE_100000baseDR_Full_BIT, + ETHTOOL_LINK_MODE_100000baseCR_Full_BIT); + MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_200GAUI_2_200GBASE_CR2_KR2, ext, + ETHTOOL_LINK_MODE_200000baseKR2_Full_BIT, + ETHTOOL_LINK_MODE_200000baseSR2_Full_BIT, + ETHTOOL_LINK_MODE_200000baseLR2_ER2_FR2_Full_BIT, + ETHTOOL_LINK_MODE_200000baseDR2_Full_BIT, + ETHTOOL_LINK_MODE_200000baseCR2_Full_BIT); + MLX5_BUILD_PTYS2ETHTOOL_CONFIG(MLX5E_400GAUI_4_400GBASE_CR4_KR4, ext, + ETHTOOL_LINK_MODE_400000baseKR4_Full_BIT, + ETHTOOL_LINK_MODE_400000baseSR4_Full_BIT, + ETHTOOL_LINK_MODE_400000baseLR4_ER4_FR4_Full_BIT, + ETHTOOL_LINK_MODE_400000baseDR4_Full_BIT, + ETHTOOL_LINK_MODE_400000baseCR4_Full_BIT); } static void mlx5e_ethtool_get_speed_arr(struct mlx5_core_dev *mdev, @@ -225,7 +243,7 @@ int mlx5e_ethtool_get_sset_count(struct mlx5e_priv *priv, int sset) return MLX5E_NUM_PFLAGS; case ETH_SS_TEST: return mlx5e_self_test_num(priv); - /* fallthrough */ + fallthrough; default: return -EOPNOTSUPP; } @@ -1012,7 +1030,8 @@ static u32 mlx5e_ethtool2ptys_ext_adver_link(const unsigned long *link_modes) unsigned long modes[2]; for (i = 0; i < MLX5E_EXT_LINK_MODES_NUMBER; ++i) { - if (*ptys2ext_ethtool_table[i].advertised == 0) + if (ptys2ext_ethtool_table[i].advertised[0] == 0 && + ptys2ext_ethtool_table[i].advertised[1] == 0) continue; memset(modes, 0, sizeof(modes)); bitmap_and(modes, ptys2ext_ethtool_table[i].advertised, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c index 73d3dc07331f..64d002d92250 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c @@ -672,9 +672,9 @@ static void mlx5e_cleanup_ttc_rules(struct mlx5e_ttc_table *ttc) int i; for (i = 0; i < MLX5E_NUM_TT; i++) { - if (!IS_ERR_OR_NULL(ttc->rules[i])) { - mlx5_del_flow_rules(ttc->rules[i]); - ttc->rules[i] = NULL; + if (!IS_ERR_OR_NULL(ttc->rules[i].rule)) { + mlx5_del_flow_rules(ttc->rules[i].rule); + ttc->rules[i].rule = NULL; } } @@ -857,7 +857,8 @@ static int mlx5e_generate_ttc_table_rules(struct mlx5e_priv *priv, struct mlx5e_ttc_table *ttc) { struct mlx5_flow_destination dest = {}; - struct mlx5_flow_handle **rules; + struct mlx5_flow_handle **trules; + struct mlx5e_ttc_rule *rules; struct mlx5_flow_table *ft; int tt; int err; @@ -867,39 +868,47 @@ static int mlx5e_generate_ttc_table_rules(struct mlx5e_priv *priv, dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; for (tt = 0; tt < MLX5E_NUM_TT; tt++) { + struct mlx5e_ttc_rule *rule = &rules[tt]; + if (tt == MLX5E_TT_ANY) dest.tir_num = params->any_tt_tirn; else dest.tir_num = params->indir_tirn[tt]; - rules[tt] = mlx5e_generate_ttc_rule(priv, ft, &dest, - ttc_rules[tt].etype, - ttc_rules[tt].proto); - if (IS_ERR(rules[tt])) + + rule->rule = mlx5e_generate_ttc_rule(priv, ft, &dest, + ttc_rules[tt].etype, + ttc_rules[tt].proto); + if (IS_ERR(rule->rule)) { + err = PTR_ERR(rule->rule); + rule->rule = NULL; goto del_rules; + } + rule->default_dest = dest; } if (!params->inner_ttc || !mlx5e_tunnel_inner_ft_supported(priv->mdev)) return 0; - rules = ttc->tunnel_rules; + trules = ttc->tunnel_rules; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; dest.ft = params->inner_ttc->ft.t; for (tt = 0; tt < MLX5E_NUM_TUNNEL_TT; tt++) { if (!mlx5e_tunnel_proto_supported(priv->mdev, ttc_tunnel_rules[tt].proto)) continue; - rules[tt] = mlx5e_generate_ttc_rule(priv, ft, &dest, - ttc_tunnel_rules[tt].etype, - ttc_tunnel_rules[tt].proto); - if (IS_ERR(rules[tt])) + trules[tt] = mlx5e_generate_ttc_rule(priv, ft, &dest, + ttc_tunnel_rules[tt].etype, + ttc_tunnel_rules[tt].proto); + if (IS_ERR(trules[tt])) { + err = PTR_ERR(trules[tt]); + trules[tt] = NULL; goto del_rules; + } } return 0; del_rules: - err = PTR_ERR(rules[tt]); - rules[tt] = NULL; mlx5e_cleanup_ttc_rules(ttc); return err; } @@ -1015,33 +1024,38 @@ static int mlx5e_generate_inner_ttc_table_rules(struct mlx5e_priv *priv, struct mlx5e_ttc_table *ttc) { struct mlx5_flow_destination dest = {}; - struct mlx5_flow_handle **rules; + struct mlx5e_ttc_rule *rules; struct mlx5_flow_table *ft; int err; int tt; ft = ttc->ft.t; rules = ttc->rules; - dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; + for (tt = 0; tt < MLX5E_NUM_TT; tt++) { + struct mlx5e_ttc_rule *rule = &rules[tt]; + if (tt == MLX5E_TT_ANY) dest.tir_num = params->any_tt_tirn; else dest.tir_num = params->indir_tirn[tt]; - rules[tt] = mlx5e_generate_inner_ttc_rule(priv, ft, &dest, - ttc_rules[tt].etype, - ttc_rules[tt].proto); - if (IS_ERR(rules[tt])) + rule->rule = mlx5e_generate_inner_ttc_rule(priv, ft, &dest, + ttc_rules[tt].etype, + ttc_rules[tt].proto); + if (IS_ERR(rule->rule)) { + err = PTR_ERR(rule->rule); + rule->rule = NULL; goto del_rules; + } + rule->default_dest = dest; } return 0; del_rules: - err = PTR_ERR(rules[tt]); - rules[tt] = NULL; + mlx5e_cleanup_ttc_rules(ttc); return err; } @@ -1210,6 +1224,30 @@ err: return err; } +int mlx5e_ttc_fwd_dest(struct mlx5e_priv *priv, enum mlx5e_traffic_types type, + struct mlx5_flow_destination *new_dest) +{ + return mlx5_modify_rule_destination(priv->fs.ttc.rules[type].rule, new_dest, NULL); +} + +struct mlx5_flow_destination +mlx5e_ttc_get_default_dest(struct mlx5e_priv *priv, enum mlx5e_traffic_types type) +{ + struct mlx5_flow_destination *dest = &priv->fs.ttc.rules[type].default_dest; + + WARN_ONCE(dest->type != MLX5_FLOW_DESTINATION_TYPE_TIR, + "TTC[%d] default dest is not setup yet", type); + + return *dest; +} + +int mlx5e_ttc_fwd_default_dest(struct mlx5e_priv *priv, enum mlx5e_traffic_types type) +{ + struct mlx5_flow_destination dest = mlx5e_ttc_get_default_dest(priv, type); + + return mlx5e_ttc_fwd_dest(priv, type, &dest); +} + static void mlx5e_del_l2_flow_rule(struct mlx5e_priv *priv, struct mlx5e_l2_rule *ai) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 081f15074cac..aebcf73f8546 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -45,7 +45,6 @@ #include "en_tc.h" #include "en_rep.h" #include "en_accel/ipsec.h" -#include "en_accel/ipsec_rxtx.h" #include "en_accel/en_accel.h" #include "en_accel/tls.h" #include "accel/ipsec.h" @@ -231,7 +230,6 @@ static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq, cseg->qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | ds_cnt); - cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; cseg->umr_mkey = rq->mkey_be; ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE; @@ -276,7 +274,7 @@ static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev, MLX5_SET(mkc, mkc, lw, 1); MLX5_SET(mkc, mkc, lr, 1); MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); - + mlx5e_mkey_set_relaxed_ordering(mdev, mkc); MLX5_SET(mkc, mkc, qpn, 0xffffff); MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.pdn); MLX5_SET64(mkc, mkc, len, npages << page_shift); @@ -419,7 +417,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->mpwqe.wq, &rq->wq_ctrl); if (err) - return err; + goto err_rq_wq_destroy; rq->mpwqe.wq.db = &rq->mpwqe.wq.db[MLX5_RCV_DBR]; @@ -428,29 +426,6 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, pool_size = MLX5_MPWRQ_PAGES_PER_WQE << mlx5e_mpwqe_get_log_rq_size(params, xsk); - rq->post_wqes = mlx5e_post_rx_mpwqes; - rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe; - - rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe_mpwqe; -#ifdef CONFIG_MLX5_EN_IPSEC - if (MLX5_IPSEC_DEV(mdev)) { - err = -EINVAL; - netdev_err(c->netdev, "MPWQE RQ with IPSec offload not supported\n"); - goto err_rq_wq_destroy; - } -#endif - if (!rq->handle_rx_cqe) { - err = -EINVAL; - netdev_err(c->netdev, "RX handler of MPWQE RQ is not set, err %d\n", err); - goto err_rq_wq_destroy; - } - - rq->mpwqe.skb_from_cqe_mpwrq = xsk ? - mlx5e_xsk_skb_from_cqe_mpwrq_linear : - mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL) ? - mlx5e_skb_from_cqe_mpwrq_linear : - mlx5e_skb_from_cqe_mpwrq_nonlinear; - rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); rq->mpwqe.num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk)); @@ -470,7 +445,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, err = mlx5_wq_cyc_create(mdev, &rqp->wq, rqc_wq, &rq->wqe.wq, &rq->wq_ctrl); if (err) - return err; + goto err_rq_wq_destroy; rq->wqe.wq.db = &rq->wqe.wq.db[MLX5_RCV_DBR]; @@ -492,29 +467,13 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, if (err) goto err_free; - rq->post_wqes = mlx5e_post_rx_wqes; - rq->dealloc_wqe = mlx5e_dealloc_rx_wqe; - -#ifdef CONFIG_MLX5_EN_IPSEC - if (c->priv->ipsec) - rq->handle_rx_cqe = mlx5e_ipsec_handle_rx_cqe; - else -#endif - rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe; - if (!rq->handle_rx_cqe) { - err = -EINVAL; - netdev_err(c->netdev, "RX handler of RQ is not set, err %d\n", err); - goto err_free; - } - - rq->wqe.skb_from_cqe = xsk ? - mlx5e_xsk_skb_from_cqe_linear : - mlx5e_rx_is_linear_skb(params, NULL) ? - mlx5e_skb_from_cqe_linear : - mlx5e_skb_from_cqe_nonlinear; rq->mkey_be = c->mkey_be; } + err = mlx5e_rq_set_handlers(rq, params, xsk); + if (err) + goto err_free; + if (xsk) { err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, MEM_TYPE_XSK_BUFF_POOL, NULL); @@ -873,6 +832,9 @@ int mlx5e_open_rq(struct mlx5e_channel *c, struct mlx5e_params *params, if (err) goto err_destroy_rq; + if (mlx5e_is_tls_on(c->priv) && !mlx5_accel_is_ktls_device(c->mdev)) + __set_bit(MLX5E_RQ_STATE_FPGA_TLS, &c->rq.state); /* must be FPGA */ + if (MLX5_CAP_ETH(c->mdev, cqe_checksum_full)) __set_bit(MLX5E_RQ_STATE_CSUM_FULL, &c->rq.state); @@ -1441,6 +1403,7 @@ void mlx5e_close_icosq(struct mlx5e_icosq *sq) struct mlx5e_channel *c = sq->channel; mlx5e_destroy_sq(c->mdev, sq->sqn); + mlx5e_free_icosq_descs(sq); mlx5e_free_icosq(sq); } @@ -1675,7 +1638,7 @@ static int mlx5e_open_tx_cqs(struct mlx5e_channel *c, for (tc = 0; tc < c->num_tc; tc++) { err = mlx5e_open_cq(c, params->tx_cq_moderation, - &cparam->tx_cq, &c->sq[tc].cq); + &cparam->txq_sq.cqp, &c->sq[tc].cq); if (err) goto err_close_tx_cqs; } @@ -1707,7 +1670,7 @@ static int mlx5e_open_sqs(struct mlx5e_channel *c, int txq_ix = c->ix + tc * params->num_channels; err = mlx5e_open_txqsq(c, c->priv->tisn[c->lag_port][tc], txq_ix, - params, &cparam->sq, &c->sq[tc], tc); + params, &cparam->txq_sq, &c->sq[tc], tc); if (err) goto err_close_sqs; } @@ -1817,34 +1780,43 @@ static int mlx5e_open_queues(struct mlx5e_channel *c, struct dim_cq_moder icocq_moder = {0, 0}; int err; - err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->icosq.cq); + err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq.cqp, &c->async_icosq.cq); if (err) return err; + err = mlx5e_open_cq(c, icocq_moder, &cparam->async_icosq.cqp, &c->icosq.cq); + if (err) + goto err_close_async_icosq_cq; + err = mlx5e_open_tx_cqs(c, params, cparam); if (err) goto err_close_icosq_cq; - err = mlx5e_open_cq(c, params->tx_cq_moderation, &cparam->tx_cq, &c->xdpsq.cq); + err = mlx5e_open_cq(c, params->tx_cq_moderation, &cparam->xdp_sq.cqp, &c->xdpsq.cq); if (err) goto err_close_tx_cqs; - err = mlx5e_open_cq(c, params->rx_cq_moderation, &cparam->rx_cq, &c->rq.cq); + err = mlx5e_open_cq(c, params->rx_cq_moderation, &cparam->rq.cqp, &c->rq.cq); if (err) goto err_close_xdp_tx_cqs; - /* XDP SQ CQ params are same as normal TXQ sq CQ params */ err = c->xdp ? mlx5e_open_cq(c, params->tx_cq_moderation, - &cparam->tx_cq, &c->rq_xdpsq.cq) : 0; + &cparam->xdp_sq.cqp, &c->rq_xdpsq.cq) : 0; if (err) goto err_close_rx_cq; napi_enable(&c->napi); - err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq); + spin_lock_init(&c->async_icosq_lock); + + err = mlx5e_open_icosq(c, params, &cparam->async_icosq, &c->async_icosq); if (err) goto err_disable_napi; + err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq); + if (err) + goto err_close_async_icosq; + err = mlx5e_open_sqs(c, params, cparam); if (err) goto err_close_icosq; @@ -1879,6 +1851,9 @@ err_close_sqs: err_close_icosq: mlx5e_close_icosq(&c->icosq); +err_close_async_icosq: + mlx5e_close_icosq(&c->async_icosq); + err_disable_napi: napi_disable(&c->napi); @@ -1897,6 +1872,9 @@ err_close_tx_cqs: err_close_icosq_cq: mlx5e_close_cq(&c->icosq.cq); +err_close_async_icosq_cq: + mlx5e_close_cq(&c->async_icosq.cq); + return err; } @@ -1908,6 +1886,7 @@ static void mlx5e_close_queues(struct mlx5e_channel *c) mlx5e_close_xdpsq(&c->rq_xdpsq); mlx5e_close_sqs(c); mlx5e_close_icosq(&c->icosq); + mlx5e_close_icosq(&c->async_icosq); napi_disable(&c->napi); if (c->xdp) mlx5e_close_cq(&c->rq_xdpsq.cq); @@ -1915,6 +1894,7 @@ static void mlx5e_close_queues(struct mlx5e_channel *c) mlx5e_close_cq(&c->xdpsq.cq); mlx5e_close_tx_cqs(c); mlx5e_close_cq(&c->icosq.cq); + mlx5e_close_cq(&c->async_icosq.cq); } static u8 mlx5e_enumerate_lag_port(struct mlx5_core_dev *mdev, int ix) @@ -1995,6 +1975,7 @@ static void mlx5e_activate_channel(struct mlx5e_channel *c) for (tc = 0; tc < c->num_tc; tc++) mlx5e_activate_txqsq(&c->sq[tc]); mlx5e_activate_icosq(&c->icosq); + mlx5e_activate_icosq(&c->async_icosq); mlx5e_activate_rq(&c->rq); if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) @@ -2009,6 +1990,7 @@ static void mlx5e_deactivate_channel(struct mlx5e_channel *c) mlx5e_deactivate_xsk(c); mlx5e_deactivate_rq(&c->rq); + mlx5e_deactivate_icosq(&c->async_icosq); mlx5e_deactivate_icosq(&c->icosq); for (tc = 0; tc < c->num_tc; tc++) mlx5e_deactivate_txqsq(&c->sq[tc]); @@ -2138,6 +2120,7 @@ void mlx5e_build_rq_param(struct mlx5e_priv *priv, MLX5_SET(rqc, rqc, scatter_fcs, params->scatter_fcs_en); param->wq.buf_numa_node = dev_to_node(mdev->device); + mlx5e_build_rx_cq_param(priv, params, xsk, ¶m->cqp); } static void mlx5e_build_drop_rq_param(struct mlx5e_priv *priv, @@ -2180,6 +2163,7 @@ static void mlx5e_build_sq_param(struct mlx5e_priv *priv, mlx5e_build_sq_param_common(priv, param); MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size); MLX5_SET(sqc, sqc, allow_swp, allow_swp); + mlx5e_build_tx_cq_param(priv, params, ¶m->cqp); } static void mlx5e_build_common_cq_param(struct mlx5e_priv *priv, @@ -2256,6 +2240,7 @@ void mlx5e_build_icosq_param(struct mlx5e_priv *priv, MLX5_SET(wq, wq, log_wq_sz, log_wq_size); MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(priv->mdev, reg_umr_sq)); + mlx5e_build_ico_cq_param(priv, log_wq_size, ¶m->cqp); } void mlx5e_build_xdpsq_param(struct mlx5e_priv *priv, @@ -2268,6 +2253,7 @@ void mlx5e_build_xdpsq_param(struct mlx5e_priv *priv, mlx5e_build_sq_param_common(priv, param); MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size); param->is_mpw = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE); + mlx5e_build_tx_cq_param(priv, params, ¶m->cqp); } static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5e_params *params, @@ -2282,22 +2268,29 @@ static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5e_params *params, } } +static u8 mlx5e_build_async_icosq_log_wq_sz(struct net_device *netdev) +{ + if (netdev->hw_features & NETIF_F_HW_TLS_RX) + return MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE; + + return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; +} + static void mlx5e_build_channel_param(struct mlx5e_priv *priv, struct mlx5e_params *params, struct mlx5e_channel_param *cparam) { - u8 icosq_log_wq_sz; + u8 icosq_log_wq_sz, async_icosq_log_wq_sz; mlx5e_build_rq_param(priv, params, NULL, &cparam->rq); icosq_log_wq_sz = mlx5e_build_icosq_log_wq_sz(params, &cparam->rq); + async_icosq_log_wq_sz = mlx5e_build_async_icosq_log_wq_sz(priv->netdev); - mlx5e_build_sq_param(priv, params, &cparam->sq); + mlx5e_build_sq_param(priv, params, &cparam->txq_sq); mlx5e_build_xdpsq_param(priv, params, &cparam->xdp_sq); mlx5e_build_icosq_param(priv, icosq_log_wq_sz, &cparam->icosq); - mlx5e_build_rx_cq_param(priv, params, NULL, &cparam->rx_cq); - mlx5e_build_tx_cq_param(priv, params, &cparam->tx_cq); - mlx5e_build_ico_cq_param(priv, icosq_log_wq_sz, &cparam->icosq_cq); + mlx5e_build_icosq_param(priv, async_icosq_log_wq_sz, &cparam->async_icosq); } int mlx5e_open_channels(struct mlx5e_priv *priv, @@ -3069,6 +3062,25 @@ void mlx5e_timestamp_init(struct mlx5e_priv *priv) priv->tstamp.rx_filter = HWTSTAMP_FILTER_NONE; } +static void mlx5e_modify_admin_state(struct mlx5_core_dev *mdev, + enum mlx5_port_status state) +{ + struct mlx5_eswitch *esw = mdev->priv.eswitch; + int vport_admin_state; + + mlx5_set_port_admin_status(mdev, state); + + if (!MLX5_ESWITCH_MANAGER(mdev) || mlx5_eswitch_mode(esw) == MLX5_ESWITCH_OFFLOADS) + return; + + if (state == MLX5_PORT_UP) + vport_admin_state = MLX5_VPORT_ADMIN_STATE_AUTO; + else + vport_admin_state = MLX5_VPORT_ADMIN_STATE_DOWN; + + mlx5_eswitch_set_vport_state(esw, MLX5_VPORT_UPLINK, vport_admin_state); +} + int mlx5e_open_locked(struct net_device *netdev) { struct mlx5e_priv *priv = netdev_priv(netdev); @@ -3101,7 +3113,7 @@ int mlx5e_open(struct net_device *netdev) mutex_lock(&priv->state_lock); err = mlx5e_open_locked(netdev); if (!err) - mlx5_set_port_admin_status(priv->mdev, MLX5_PORT_UP); + mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_UP); mutex_unlock(&priv->state_lock); return err; @@ -3135,7 +3147,7 @@ int mlx5e_close(struct net_device *netdev) return -ENODEV; mutex_lock(&priv->state_lock); - mlx5_set_port_admin_status(priv->mdev, MLX5_PORT_DOWN); + mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_DOWN); err = mlx5e_close_locked(netdev); mutex_unlock(&priv->state_lock); @@ -3828,6 +3840,7 @@ int mlx5e_set_features(struct net_device *netdev, netdev_features_t features) #ifdef CONFIG_MLX5_EN_ARFS err |= MLX5E_HANDLE_FEATURE(NETIF_F_NTUPLE, set_feature_arfs); #endif + err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TLS_RX, mlx5e_ktls_set_feature_rx); if (err) { netdev->features = oper_features; @@ -4178,83 +4191,6 @@ int mlx5e_get_vf_stats(struct net_device *dev, } #endif -struct mlx5e_vxlan_work { - struct work_struct work; - struct mlx5e_priv *priv; - u16 port; -}; - -static void mlx5e_vxlan_add_work(struct work_struct *work) -{ - struct mlx5e_vxlan_work *vxlan_work = - container_of(work, struct mlx5e_vxlan_work, work); - struct mlx5e_priv *priv = vxlan_work->priv; - u16 port = vxlan_work->port; - - mutex_lock(&priv->state_lock); - mlx5_vxlan_add_port(priv->mdev->vxlan, port); - mutex_unlock(&priv->state_lock); - - kfree(vxlan_work); -} - -static void mlx5e_vxlan_del_work(struct work_struct *work) -{ - struct mlx5e_vxlan_work *vxlan_work = - container_of(work, struct mlx5e_vxlan_work, work); - struct mlx5e_priv *priv = vxlan_work->priv; - u16 port = vxlan_work->port; - - mutex_lock(&priv->state_lock); - mlx5_vxlan_del_port(priv->mdev->vxlan, port); - mutex_unlock(&priv->state_lock); - kfree(vxlan_work); -} - -static void mlx5e_vxlan_queue_work(struct mlx5e_priv *priv, u16 port, int add) -{ - struct mlx5e_vxlan_work *vxlan_work; - - vxlan_work = kmalloc(sizeof(*vxlan_work), GFP_ATOMIC); - if (!vxlan_work) - return; - - if (add) - INIT_WORK(&vxlan_work->work, mlx5e_vxlan_add_work); - else - INIT_WORK(&vxlan_work->work, mlx5e_vxlan_del_work); - - vxlan_work->priv = priv; - vxlan_work->port = port; - queue_work(priv->wq, &vxlan_work->work); -} - -void mlx5e_add_vxlan_port(struct net_device *netdev, struct udp_tunnel_info *ti) -{ - struct mlx5e_priv *priv = netdev_priv(netdev); - - if (ti->type != UDP_TUNNEL_TYPE_VXLAN) - return; - - if (!mlx5_vxlan_allowed(priv->mdev->vxlan)) - return; - - mlx5e_vxlan_queue_work(priv, be16_to_cpu(ti->port), 1); -} - -void mlx5e_del_vxlan_port(struct net_device *netdev, struct udp_tunnel_info *ti) -{ - struct mlx5e_priv *priv = netdev_priv(netdev); - - if (ti->type != UDP_TUNNEL_TYPE_VXLAN) - return; - - if (!mlx5_vxlan_allowed(priv->mdev->vxlan)) - return; - - mlx5e_vxlan_queue_work(priv, be16_to_cpu(ti->port), 0); -} - static netdev_features_t mlx5e_tunnel_features_check(struct mlx5e_priv *priv, struct sk_buff *skb, netdev_features_t features) @@ -4329,8 +4265,6 @@ static void mlx5e_tx_timeout_work(struct work_struct *work) { struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, tx_timeout_work); - bool report_failed = false; - int err; int i; rtnl_lock(); @@ -4348,18 +4282,10 @@ static void mlx5e_tx_timeout_work(struct work_struct *work) continue; if (mlx5e_reporter_tx_timeout(sq)) - report_failed = true; + /* break if tried to reopened channels */ + break; } - if (!report_failed) - goto unlock; - - err = mlx5e_safe_reopen_channels(priv); - if (err) - netdev_err(priv->netdev, - "mlx5e_safe_reopen_channels failed recovering from a tx_timeout, err(%d).\n", - err); - unlock: mutex_unlock(&priv->state_lock); rtnl_unlock(); @@ -4492,29 +4418,11 @@ unlock: return err; } -static u32 mlx5e_xdp_query(struct net_device *dev) -{ - struct mlx5e_priv *priv = netdev_priv(dev); - const struct bpf_prog *xdp_prog; - u32 prog_id = 0; - - mutex_lock(&priv->state_lock); - xdp_prog = priv->channels.params.xdp_prog; - if (xdp_prog) - prog_id = xdp_prog->aux->id; - mutex_unlock(&priv->state_lock); - - return prog_id; -} - static int mlx5e_xdp(struct net_device *dev, struct netdev_bpf *xdp) { switch (xdp->command) { case XDP_SETUP_PROG: return mlx5e_xdp_set(dev, xdp->prog); - case XDP_QUERY_PROG: - xdp->prog_id = mlx5e_xdp_query(dev); - return 0; case XDP_SETUP_XSK_UMEM: return mlx5e_xsk_setup_umem(dev, xdp->xsk.umem, xdp->xsk.queue_id); @@ -4594,8 +4502,8 @@ const struct net_device_ops mlx5e_netdev_ops = { .ndo_change_mtu = mlx5e_change_nic_mtu, .ndo_do_ioctl = mlx5e_ioctl, .ndo_set_tx_maxrate = mlx5e_set_tx_maxrate, - .ndo_udp_tunnel_add = mlx5e_add_vxlan_port, - .ndo_udp_tunnel_del = mlx5e_del_vxlan_port, + .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, + .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_features_check = mlx5e_features_check, .ndo_tx_timeout = mlx5e_tx_timeout, .ndo_bpf = mlx5e_xdp, @@ -4866,6 +4774,39 @@ static void mlx5e_set_netdev_dev_addr(struct net_device *netdev) } } +static int mlx5e_vxlan_set_port(struct net_device *netdev, unsigned int table, + unsigned int entry, struct udp_tunnel_info *ti) +{ + struct mlx5e_priv *priv = netdev_priv(netdev); + + return mlx5_vxlan_add_port(priv->mdev->vxlan, ntohs(ti->port)); +} + +static int mlx5e_vxlan_unset_port(struct net_device *netdev, unsigned int table, + unsigned int entry, struct udp_tunnel_info *ti) +{ + struct mlx5e_priv *priv = netdev_priv(netdev); + + return mlx5_vxlan_del_port(priv->mdev->vxlan, ntohs(ti->port)); +} + +void mlx5e_vxlan_set_netdev_info(struct mlx5e_priv *priv) +{ + if (!mlx5_vxlan_allowed(priv->mdev->vxlan)) + return; + + priv->nic_info.set_port = mlx5e_vxlan_set_port; + priv->nic_info.unset_port = mlx5e_vxlan_unset_port; + priv->nic_info.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP | + UDP_TUNNEL_NIC_INFO_STATIC_IANA_VXLAN; + priv->nic_info.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN; + /* Don't count the space hard-coded to the IANA port */ + priv->nic_info.tables[0].n_entries = + mlx5_vxlan_max_udp_ports(priv->mdev) - 1; + + priv->netdev->udp_tunnel_nic_info = &priv->nic_info; +} + static void mlx5e_build_nic_netdev(struct net_device *netdev) { struct mlx5e_priv *priv = netdev_priv(netdev); @@ -4909,6 +4850,8 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev) netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER; netdev->hw_features |= NETIF_F_HW_VLAN_STAG_TX; + mlx5e_vxlan_set_netdev_info(priv); + if (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev) || mlx5e_any_tunnel_proto_supported(mdev)) { netdev->hw_enc_features |= NETIF_F_HW_CSUM; @@ -5058,6 +5001,9 @@ static int mlx5e_nic_init(struct mlx5_core_dev *mdev, if (err) mlx5_core_err(mdev, "TLS initialization failed, %d\n", err); mlx5e_build_nic_netdev(netdev); + err = mlx5e_devlink_port_register(priv); + if (err) + mlx5_core_err(mdev, "mlx5e_devlink_port_register failed, %d\n", err); mlx5e_health_create_reporters(priv); return 0; @@ -5066,6 +5012,7 @@ static int mlx5e_nic_init(struct mlx5_core_dev *mdev, static void mlx5e_nic_cleanup(struct mlx5e_priv *priv) { mlx5e_health_destroy_reporters(priv); + mlx5e_devlink_port_unregister(priv); mlx5e_tls_cleanup(priv); mlx5e_ipsec_cleanup(priv); mlx5e_netdev_cleanup(priv->netdev, priv); @@ -5118,12 +5065,18 @@ static int mlx5e_init_nic_rx(struct mlx5e_priv *priv) if (err) goto err_destroy_flow_steering; + err = mlx5e_accel_init_rx(priv); + if (err) + goto err_tc_nic_cleanup; + #ifdef CONFIG_MLX5_EN_ARFS priv->netdev->rx_cpu_rmap = mlx5_eq_table_get_rmap(priv->mdev); #endif return 0; +err_tc_nic_cleanup: + mlx5e_tc_nic_cleanup(priv); err_destroy_flow_steering: mlx5e_destroy_flow_steering(priv); err_destroy_xsk_tirs: @@ -5147,6 +5100,7 @@ err_destroy_q_counters: static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv) { + mlx5e_accel_cleanup_rx(priv); mlx5e_tc_nic_cleanup(priv); mlx5e_destroy_flow_steering(priv); mlx5e_destroy_direct_tirs(priv, priv->xsk_tir); @@ -5182,7 +5136,7 @@ static void mlx5e_nic_enable(struct mlx5e_priv *priv) /* Marking the link as currently not needed by the Driver */ if (!netif_running(netdev)) - mlx5_set_port_admin_status(mdev, MLX5_PORT_DOWN); + mlx5e_modify_admin_state(mdev, MLX5_PORT_DOWN); mlx5e_set_netdev_mtu_boundaries(priv); mlx5e_set_dev_port_mtu(priv); @@ -5203,8 +5157,7 @@ static void mlx5e_nic_enable(struct mlx5e_priv *priv) rtnl_lock(); if (netif_running(netdev)) mlx5e_open(netdev); - if (mlx5_vxlan_allowed(priv->mdev->vxlan)) - udp_tunnel_get_rx_info(netdev); + udp_tunnel_nic_reset_ntf(priv->netdev); netif_device_attach(netdev); rtnl_unlock(); } @@ -5219,8 +5172,6 @@ static void mlx5e_nic_disable(struct mlx5e_priv *priv) rtnl_lock(); if (netif_running(priv->netdev)) mlx5e_close(priv->netdev); - if (mlx5_vxlan_allowed(priv->mdev->vxlan)) - udp_tunnel_drop_rx_info(priv->netdev); netif_device_detach(priv->netdev); rtnl_unlock(); @@ -5251,8 +5202,7 @@ static const struct mlx5e_profile mlx5e_nic_profile = { .update_rx = mlx5e_update_nic_rx, .update_stats = mlx5e_update_ndo_stats, .update_carrier = mlx5e_update_carrier, - .rx_handlers.handle_rx_cqe = mlx5e_handle_rx_cqe, - .rx_handlers.handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq, + .rx_handlers = &mlx5e_rx_handlers_nic, .max_tc = MLX5E_MAX_NUM_TC, .rq_groups = MLX5E_NUM_RQ_GROUPS(XSK), .stats_grps = mlx5e_nic_stats_grps, @@ -5390,6 +5340,8 @@ err_cleanup_tx: profile->cleanup_tx(priv); out: + set_bit(MLX5E_STATE_DESTROYING, &priv->state); + cancel_work_sync(&priv->update_stats_work); return err; } @@ -5495,16 +5447,10 @@ static void *mlx5e_add(struct mlx5_core_dev *mdev) goto err_destroy_netdev; } - err = mlx5e_devlink_port_register(priv); - if (err) { - mlx5_core_err(mdev, "mlx5e_devlink_port_register failed, %d\n", err); - goto err_detach; - } - err = register_netdev(netdev); if (err) { mlx5_core_err(mdev, "register_netdev failed, %d\n", err); - goto err_devlink_port_unregister; + goto err_detach; } mlx5e_devlink_port_type_eth_set(priv); @@ -5512,8 +5458,6 @@ static void *mlx5e_add(struct mlx5_core_dev *mdev) mlx5e_dcbnl_init_app(priv); return priv; -err_devlink_port_unregister: - mlx5e_devlink_port_unregister(priv); err_detach: mlx5e_detach(mdev, priv); err_destroy_netdev: @@ -5534,7 +5478,6 @@ static void mlx5e_remove(struct mlx5_core_dev *mdev, void *vpriv) priv = vpriv; mlx5e_dcbnl_delete_app(priv); unregister_netdev(priv->netdev); - mlx5e_devlink_port_unregister(priv); mlx5e_detach(mdev, vpriv); mlx5e_destroy_netdev(priv); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index 006807e04eda..e13e5d1b3eae 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -35,7 +35,6 @@ #include <net/switchdev.h> #include <net/pkt_cls.h> #include <net/act_api.h> -#include <net/arp.h> #include <net/devlink.h> #include <net/ipv6_stubs.h> @@ -43,6 +42,7 @@ #include "esw/chains.h" #include "en.h" #include "en_rep.h" +#include "en/txrx.h" #include "en_tc.h" #include "en/rep/tc.h" #include "en/rep/neigh.h" @@ -611,6 +611,29 @@ static struct devlink_port *mlx5e_rep_get_devlink_port(struct net_device *dev) return &rpriv->dl_port; } +static int mlx5e_rep_change_carrier(struct net_device *dev, bool new_carrier) +{ + struct mlx5e_priv *priv = netdev_priv(dev); + struct mlx5e_rep_priv *rpriv = priv->ppriv; + struct mlx5_eswitch_rep *rep = rpriv->rep; + int err; + + if (new_carrier) { + err = mlx5_modify_vport_admin_state(priv->mdev, MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, + rep->vport, 1, MLX5_VPORT_ADMIN_STATE_UP); + if (err) + return err; + netif_carrier_on(dev); + } else { + err = mlx5_modify_vport_admin_state(priv->mdev, MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, + rep->vport, 1, MLX5_VPORT_ADMIN_STATE_DOWN); + if (err) + return err; + netif_carrier_off(dev); + } + return 0; +} + static const struct net_device_ops mlx5e_netdev_ops_rep = { .ndo_open = mlx5e_rep_open, .ndo_stop = mlx5e_rep_close, @@ -621,6 +644,7 @@ static const struct net_device_ops mlx5e_netdev_ops_rep = { .ndo_has_offload_stats = mlx5e_rep_has_offload_stats, .ndo_get_offload_stats = mlx5e_rep_get_offload_stats, .ndo_change_mtu = mlx5e_rep_change_mtu, + .ndo_change_carrier = mlx5e_rep_change_carrier, }; static const struct net_device_ops mlx5e_netdev_ops_uplink_rep = { @@ -634,8 +658,8 @@ static const struct net_device_ops mlx5e_netdev_ops_uplink_rep = { .ndo_has_offload_stats = mlx5e_rep_has_offload_stats, .ndo_get_offload_stats = mlx5e_rep_get_offload_stats, .ndo_change_mtu = mlx5e_uplink_rep_change_mtu, - .ndo_udp_tunnel_add = mlx5e_add_vxlan_port, - .ndo_udp_tunnel_del = mlx5e_del_vxlan_port, + .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, + .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_features_check = mlx5e_features_check, .ndo_set_vf_mac = mlx5e_set_vf_mac, .ndo_set_vf_rate = mlx5e_set_vf_rate, @@ -700,12 +724,13 @@ static void mlx5e_build_rep_netdev(struct net_device *netdev) struct mlx5_eswitch_rep *rep = rpriv->rep; struct mlx5_core_dev *mdev = priv->mdev; + SET_NETDEV_DEV(netdev, mdev->device); if (rep->vport == MLX5_VPORT_UPLINK) { - SET_NETDEV_DEV(netdev, mdev->device); netdev->netdev_ops = &mlx5e_netdev_ops_uplink_rep; /* we want a persistent mac for the uplink rep */ mlx5_query_mac_address(mdev, netdev->dev_addr); netdev->ethtool_ops = &mlx5e_uplink_rep_ethtool_ops; + mlx5e_vxlan_set_netdev_info(priv); mlx5e_dcbnl_build_rep_netdev(netdev); } else { netdev->netdev_ops = &mlx5e_netdev_ops_rep; @@ -936,6 +961,7 @@ err_close_drop_rq: static void mlx5e_cleanup_rep_rx(struct mlx5e_priv *priv) { + mlx5e_ethtool_cleanup_steering(priv); rep_vport_rx_rule_destroy(priv); mlx5e_destroy_rep_root_ft(priv); mlx5e_destroy_ttc_table(priv, &priv->fs.ttc); @@ -1080,6 +1106,8 @@ static void mlx5e_uplink_rep_enable(struct mlx5e_priv *priv) mlx5e_rep_tc_enable(priv); + mlx5_modify_vport_admin_state(mdev, MLX5_VPORT_STATE_OP_MOD_UPLINK, + 0, 0, MLX5_VPORT_ADMIN_STATE_AUTO); mlx5_lag_add(mdev, netdev); priv->events_nb.notifier_call = uplink_rep_async_event; mlx5_notifier_register(mdev, &priv->events_nb); @@ -1144,8 +1172,7 @@ static const struct mlx5e_profile mlx5e_rep_profile = { .enable = mlx5e_rep_enable, .update_rx = mlx5e_update_rep_rx, .update_stats = mlx5e_update_ndo_stats, - .rx_handlers.handle_rx_cqe = mlx5e_handle_rx_cqe_rep, - .rx_handlers.handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq_rep, + .rx_handlers = &mlx5e_rx_handlers_rep, .max_tc = 1, .rq_groups = MLX5E_NUM_RQ_GROUPS(REGULAR), .stats_grps = mlx5e_rep_stats_grps, @@ -1164,8 +1191,7 @@ static const struct mlx5e_profile mlx5e_uplink_rep_profile = { .update_rx = mlx5e_update_rep_rx, .update_stats = mlx5e_update_ndo_stats, .update_carrier = mlx5e_update_carrier, - .rx_handlers.handle_rx_cqe = mlx5e_handle_rx_cqe_rep, - .rx_handlers.handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq_rep, + .rx_handlers = &mlx5e_rx_handlers_rep, .max_tc = MLX5E_MAX_NUM_TC, .rq_groups = MLX5E_NUM_RQ_GROUPS(REGULAR), .stats_grps = mlx5e_ul_rep_stats_grps, @@ -1181,17 +1207,12 @@ is_devlink_port_supported(const struct mlx5_core_dev *dev, mlx5_eswitch_is_vf_vport(dev->priv.eswitch, rpriv->rep->vport); } -static unsigned int -vport_to_devlink_port_index(const struct mlx5_core_dev *dev, u16 vport_num) -{ - return (MLX5_CAP_GEN(dev, vhca_id) << 16) | vport_num; -} - static int register_devlink_port(struct mlx5_core_dev *dev, struct mlx5e_rep_priv *rpriv) { struct devlink *devlink = priv_to_devlink(dev); struct mlx5_eswitch_rep *rep = rpriv->rep; + struct devlink_port_attrs attrs = {}; struct netdev_phys_item_id ppid = {}; unsigned int dl_port_index = 0; u16 pfnum; @@ -1200,23 +1221,24 @@ static int register_devlink_port(struct mlx5_core_dev *dev, return 0; mlx5e_rep_get_port_parent_id(rpriv->netdev, &ppid); - dl_port_index = vport_to_devlink_port_index(dev, rep->vport); + dl_port_index = mlx5_esw_vport_to_devlink_port_index(dev, rep->vport); pfnum = PCI_FUNC(dev->pdev->devfn); - - if (rep->vport == MLX5_VPORT_UPLINK) - devlink_port_attrs_set(&rpriv->dl_port, - DEVLINK_PORT_FLAVOUR_PHYSICAL, - pfnum, false, 0, - &ppid.id[0], ppid.id_len); - else if (rep->vport == MLX5_VPORT_PF) - devlink_port_attrs_pci_pf_set(&rpriv->dl_port, - &ppid.id[0], ppid.id_len, - pfnum); - else if (mlx5_eswitch_is_vf_vport(dev->priv.eswitch, rpriv->rep->vport)) + if (rep->vport == MLX5_VPORT_UPLINK) { + attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL; + attrs.phys.port_number = pfnum; + memcpy(attrs.switch_id.id, &ppid.id[0], ppid.id_len); + attrs.switch_id.id_len = ppid.id_len; + devlink_port_attrs_set(&rpriv->dl_port, &attrs); + } else if (rep->vport == MLX5_VPORT_PF) { + memcpy(rpriv->dl_port.attrs.switch_id.id, &ppid.id[0], ppid.id_len); + rpriv->dl_port.attrs.switch_id.id_len = ppid.id_len; + devlink_port_attrs_pci_pf_set(&rpriv->dl_port, pfnum); + } else if (mlx5_eswitch_is_vf_vport(dev->priv.eswitch, rpriv->rep->vport)) { + memcpy(rpriv->dl_port.attrs.switch_id.id, &ppid.id[0], ppid.id_len); + rpriv->dl_port.attrs.switch_id.id_len = ppid.id_len; devlink_port_attrs_pci_vf_set(&rpriv->dl_port, - &ppid.id[0], ppid.id_len, pfnum, rep->vport - 1); - + } return devlink_port_register(devlink, &rpriv->dl_port, dl_port_index); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h index 1d5669801484..622c27ae4ac7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h @@ -41,6 +41,8 @@ #include "lib/port_tun.h" #ifdef CONFIG_MLX5_ESWITCH +extern const struct mlx5e_rx_handlers mlx5e_rx_handlers_rep; + struct mlx5e_neigh_update_table { struct rhashtable neigh_ht; /* Save the neigh hash entries in a list in addition to the hash table @@ -223,10 +225,6 @@ bool mlx5e_is_uplink_rep(struct mlx5e_priv *priv); int mlx5e_add_sqs_fwd_rules(struct mlx5e_priv *priv); void mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv *priv); -void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe); -void mlx5e_handle_rx_cqe_mpwrq_rep(struct mlx5e_rq *rq, - struct mlx5_cqe64 *cqe); - void mlx5e_rep_queue_neigh_stats_work(struct mlx5e_priv *priv); bool mlx5e_eswitch_vf_rep(struct net_device *netdev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index dbb1c6323967..65828af120b7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -34,22 +34,39 @@ #include <linux/ip.h> #include <linux/ipv6.h> #include <linux/tcp.h> -#include <linux/indirect_call_wrapper.h> #include <net/ip6_checksum.h> #include <net/page_pool.h> #include <net/inet_ecn.h> #include "en.h" +#include "en/txrx.h" #include "en_tc.h" #include "eswitch.h" #include "en_rep.h" #include "en/rep/tc.h" #include "ipoib/ipoib.h" +#include "accel/ipsec.h" +#include "fpga/ipsec.h" #include "en_accel/ipsec_rxtx.h" #include "en_accel/tls_rxtx.h" #include "lib/clock.h" #include "en/xdp.h" #include "en/xsk/rx.h" #include "en/health.h" +#include "en/params.h" + +static struct sk_buff * +mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, + u16 cqe_bcnt, u32 head_offset, u32 page_idx); +static struct sk_buff * +mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, + u16 cqe_bcnt, u32 head_offset, u32 page_idx); +static void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe); +static void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe); + +const struct mlx5e_rx_handlers mlx5e_rx_handlers_nic = { + .handle_rx_cqe = mlx5e_handle_rx_cqe, + .handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq, +}; static inline bool mlx5e_rx_hw_stamp(struct hwtstamp_config *config) { @@ -370,7 +387,7 @@ static inline void mlx5e_free_rx_wqe(struct mlx5e_rq *rq, mlx5e_put_rx_frag(rq, wi, recycle); } -void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix) +static void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix) { struct mlx5e_wqe_frag_info *wi = get_frag(rq, ix); @@ -537,14 +554,14 @@ err: return err; } -void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) +static void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) { struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix]; /* Don't recycle, this function is called on rq/netdev close */ mlx5e_free_rx_mpwqe(rq, wi, false); } -bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq) +INDIRECT_CALLABLE_SCOPE bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq) { struct mlx5_wq_cyc *wq = &rq->wqe.wq; u8 wqe_bulk; @@ -578,6 +595,33 @@ bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq) return !!err; } +void mlx5e_free_icosq_descs(struct mlx5e_icosq *sq) +{ + u16 sqcc; + + sqcc = sq->cc; + + while (sqcc != sq->pc) { + struct mlx5e_icosq_wqe_info *wi; + u16 ci; + + ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc); + wi = &sq->db.wqe_info[ci]; + sqcc += wi->num_wqebbs; +#ifdef CONFIG_MLX5_EN_TLS + switch (wi->wqe_type) { + case MLX5E_ICOSQ_WQE_SET_PSV_TLS: + mlx5e_ktls_handle_ctx_completion(wi); + break; + case MLX5E_ICOSQ_WQE_GET_PSV_TLS: + mlx5e_ktls_handle_get_psv_completion(wi, sq); + break; + } +#endif + } + sq->cc = sqcc; +} + int mlx5e_poll_ico_cq(struct mlx5e_cq *cq) { struct mlx5e_icosq *sq = container_of(cq, struct mlx5e_icosq, cq); @@ -633,6 +677,16 @@ int mlx5e_poll_ico_cq(struct mlx5e_cq *cq) break; case MLX5E_ICOSQ_WQE_NOP: break; +#ifdef CONFIG_MLX5_EN_TLS + case MLX5E_ICOSQ_WQE_UMR_TLS: + break; + case MLX5E_ICOSQ_WQE_SET_PSV_TLS: + mlx5e_ktls_handle_ctx_completion(wi); + break; + case MLX5E_ICOSQ_WQE_GET_PSV_TLS: + mlx5e_ktls_handle_get_psv_completion(wi, sq); + break; +#endif default: netdev_WARN_ONCE(cq->channel->netdev, "Bad WQE type in ICOSQ WQE info: 0x%x\n", @@ -648,7 +702,7 @@ int mlx5e_poll_ico_cq(struct mlx5e_cq *cq) return i; } -bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq) +INDIRECT_CALLABLE_SCOPE bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq) { struct mlx5e_icosq *sq = &rq->channel->icosq; struct mlx5_wq_ll *wq = &rq->mpwqe.wq; @@ -936,9 +990,14 @@ static inline void mlx5e_handle_csum(struct net_device *netdev, goto csum_unnecessary; if (likely(is_last_ethertype_ip(skb, &network_depth, &proto))) { - if (unlikely(get_ip_proto(skb, network_depth, proto) == IPPROTO_SCTP)) + u8 ipproto = get_ip_proto(skb, network_depth, proto); + + if (unlikely(ipproto == IPPROTO_SCTP)) goto csum_unnecessary; + if (unlikely(mlx5_ipsec_is_rx_flow(cqe))) + goto csum_none; + stats->csum_complete++; skb->ip_summed = CHECKSUM_COMPLETE; skb->csum = csum_unfold((__force __sum16)cqe->check_sum); @@ -982,9 +1041,10 @@ static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe, skb->mac_len = ETH_HLEN; -#ifdef CONFIG_MLX5_EN_TLS - mlx5e_tls_handle_rx_skb(netdev, skb, &cqe_bcnt); -#endif + mlx5e_tls_handle_rx_skb(rq, skb, cqe, &cqe_bcnt); + + if (unlikely(mlx5_ipsec_is_rx_flow(cqe))) + mlx5e_ipsec_offload_handle_rx_skb(netdev, skb, cqe); if (lro_num_seg > 1) { mlx5e_lro_update_hdr(skb, cqe, cqe_bcnt); @@ -1063,7 +1123,7 @@ static void mlx5e_fill_xdp_buff(struct mlx5e_rq *rq, void *va, u16 headroom, xdp->frame_sz = rq->buff.frame0_sz; } -struct sk_buff * +static struct sk_buff * mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt) { @@ -1103,7 +1163,7 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, return skb; } -struct sk_buff * +static struct sk_buff * mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt) { @@ -1152,11 +1212,13 @@ static void trigger_report(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) struct mlx5_err_cqe *err_cqe = (struct mlx5_err_cqe *)cqe; if (cqe_syndrome_needs_recover(err_cqe->syndrome) && - !test_and_set_bit(MLX5E_RQ_STATE_RECOVERING, &rq->state)) + !test_and_set_bit(MLX5E_RQ_STATE_RECOVERING, &rq->state)) { + mlx5e_dump_error_cqe(&rq->cq, rq->rqn, err_cqe); queue_work(rq->channel->priv->wq, &rq->recover_work); + } } -void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) +static void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) { struct mlx5_wq_cyc *wq = &rq->wqe.wq; struct mlx5e_wqe_frag_info *wi; @@ -1199,7 +1261,7 @@ wq_cyc_pop: } #ifdef CONFIG_MLX5_ESWITCH -void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) +static void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) { struct net_device *netdev = rq->netdev; struct mlx5e_priv *priv = netdev_priv(netdev); @@ -1221,7 +1283,10 @@ void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) goto free_wqe; } - skb = rq->wqe.skb_from_cqe(rq, cqe, wi, cqe_bcnt); + skb = INDIRECT_CALL_2(rq->wqe.skb_from_cqe, + mlx5e_skb_from_cqe_linear, + mlx5e_skb_from_cqe_nonlinear, + rq, cqe, wi, cqe_bcnt); if (!skb) { /* probably for XDP */ if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) { @@ -1251,8 +1316,7 @@ wq_cyc_pop: mlx5_wq_cyc_pop(wq); } -void mlx5e_handle_rx_cqe_mpwrq_rep(struct mlx5e_rq *rq, - struct mlx5_cqe64 *cqe) +static void mlx5e_handle_rx_cqe_mpwrq_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) { u16 cstrides = mpwrq_get_cqe_consumed_strides(cqe); u16 wqe_id = be16_to_cpu(cqe->wqe_id); @@ -1310,9 +1374,14 @@ mpwrq_cqe_out: mlx5e_free_rx_mpwqe(rq, wi, true); mlx5_wq_ll_pop(wq, cqe->wqe_id, &wqe->next.next_wqe_index); } + +const struct mlx5e_rx_handlers mlx5e_rx_handlers_rep = { + .handle_rx_cqe = mlx5e_handle_rx_cqe_rep, + .handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq_rep, +}; #endif -struct sk_buff * +static struct sk_buff * mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, u16 cqe_bcnt, u32 head_offset, u32 page_idx) { @@ -1358,7 +1427,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w return skb; } -struct sk_buff * +static struct sk_buff * mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, u16 cqe_bcnt, u32 head_offset, u32 page_idx) { @@ -1408,7 +1477,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, return skb; } -void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) +static void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) { u16 cstrides = mpwrq_get_cqe_consumed_strides(cqe); u16 wqe_id = be16_to_cpu(cqe->wqe_id); @@ -1604,7 +1673,7 @@ static inline void mlx5i_complete_rx_cqe(struct mlx5e_rq *rq, stats->bytes += cqe_bcnt; } -void mlx5i_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) +static void mlx5i_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) { struct mlx5_wq_cyc *wq = &rq->wqe.wq; struct mlx5e_wqe_frag_info *wi; @@ -1640,11 +1709,15 @@ wq_free_wqe: mlx5_wq_cyc_pop(wq); } +const struct mlx5e_rx_handlers mlx5i_rx_handlers = { + .handle_rx_cqe = mlx5i_handle_rx_cqe, + .handle_rx_cqe_mpwqe = NULL, /* Not supported */ +}; #endif /* CONFIG_MLX5_CORE_IPOIB */ #ifdef CONFIG_MLX5_EN_IPSEC -void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) +static void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) { struct mlx5_wq_cyc *wq = &rq->wqe.wq; struct mlx5e_wqe_frag_info *wi; @@ -1681,3 +1754,55 @@ wq_free_wqe: } #endif /* CONFIG_MLX5_EN_IPSEC */ + +int mlx5e_rq_set_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params, bool xsk) +{ + struct mlx5_core_dev *mdev = rq->mdev; + struct mlx5e_channel *c = rq->channel; + + switch (rq->wq_type) { + case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: + rq->mpwqe.skb_from_cqe_mpwrq = xsk ? + mlx5e_xsk_skb_from_cqe_mpwrq_linear : + mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL) ? + mlx5e_skb_from_cqe_mpwrq_linear : + mlx5e_skb_from_cqe_mpwrq_nonlinear; + rq->post_wqes = mlx5e_post_rx_mpwqes; + rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe; + + rq->handle_rx_cqe = c->priv->profile->rx_handlers->handle_rx_cqe_mpwqe; +#ifdef CONFIG_MLX5_EN_IPSEC + if (MLX5_IPSEC_DEV(mdev)) { + netdev_err(c->netdev, "MPWQE RQ with IPSec offload not supported\n"); + return -EINVAL; + } +#endif + if (!rq->handle_rx_cqe) { + netdev_err(c->netdev, "RX handler of MPWQE RQ is not set\n"); + return -EINVAL; + } + break; + default: /* MLX5_WQ_TYPE_CYCLIC */ + rq->wqe.skb_from_cqe = xsk ? + mlx5e_xsk_skb_from_cqe_linear : + mlx5e_rx_is_linear_skb(params, NULL) ? + mlx5e_skb_from_cqe_linear : + mlx5e_skb_from_cqe_nonlinear; + rq->post_wqes = mlx5e_post_rx_wqes; + rq->dealloc_wqe = mlx5e_dealloc_rx_wqe; + +#ifdef CONFIG_MLX5_EN_IPSEC + if ((mlx5_fpga_ipsec_device_caps(mdev) & MLX5_ACCEL_IPSEC_CAP_DEVICE) && + c->priv->ipsec) + rq->handle_rx_cqe = mlx5e_ipsec_handle_rx_cqe; + else +#endif + rq->handle_rx_cqe = c->priv->profile->rx_handlers->handle_rx_cqe; + if (!rq->handle_rx_cqe) { + netdev_err(c->netdev, "RX handler of RQ is not set\n"); + return -EINVAL; + } + } + + return 0; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index f009fe09e99b..e3b2f59408e6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -163,6 +163,19 @@ static const struct counter_desc sw_stats_desc[] = { { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_congst_umr) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_arfs_err) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_recover) }, +#ifdef CONFIG_MLX5_EN_TLS + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_decrypted_packets) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_decrypted_bytes) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_ctx) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_del) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_resync_req_pkt) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_resync_req_start) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_resync_req_end) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_resync_req_skip) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_resync_res_ok) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_resync_res_skip) }, + { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_err) }, +#endif { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_events) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_poll) }, { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_arm) }, @@ -275,6 +288,19 @@ static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(sw) s->rx_congst_umr += rq_stats->congst_umr; s->rx_arfs_err += rq_stats->arfs_err; s->rx_recover += rq_stats->recover; +#ifdef CONFIG_MLX5_EN_TLS + s->rx_tls_decrypted_packets += rq_stats->tls_decrypted_packets; + s->rx_tls_decrypted_bytes += rq_stats->tls_decrypted_bytes; + s->rx_tls_ctx += rq_stats->tls_ctx; + s->rx_tls_del += rq_stats->tls_del; + s->rx_tls_resync_req_pkt += rq_stats->tls_resync_req_pkt; + s->rx_tls_resync_req_start += rq_stats->tls_resync_req_start; + s->rx_tls_resync_req_end += rq_stats->tls_resync_req_end; + s->rx_tls_resync_req_skip += rq_stats->tls_resync_req_skip; + s->rx_tls_resync_res_ok += rq_stats->tls_resync_res_ok; + s->rx_tls_resync_res_skip += rq_stats->tls_resync_res_skip; + s->rx_tls_err += rq_stats->tls_err; +#endif s->ch_events += ch_stats->events; s->ch_poll += ch_stats->poll; s->ch_arm += ch_stats->arm; @@ -1475,6 +1501,19 @@ static const struct counter_desc rq_stats_desc[] = { { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, congst_umr) }, { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, arfs_err) }, { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, recover) }, +#ifdef CONFIG_MLX5_EN_TLS + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_decrypted_packets) }, + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_decrypted_bytes) }, + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_ctx) }, + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_del) }, + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_resync_req_pkt) }, + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_resync_req_start) }, + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_resync_req_end) }, + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_resync_req_skip) }, + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_resync_res_ok) }, + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_resync_res_skip) }, + { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_err) }, +#endif }; static const struct counter_desc sq_stats_desc[] = { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h index 2b83ba990714..2e1cca1923b9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h @@ -186,6 +186,18 @@ struct mlx5e_sw_stats { u64 tx_tls_skip_no_sync_data; u64 tx_tls_drop_no_sync_data; u64 tx_tls_drop_bypass_req; + + u64 rx_tls_decrypted_packets; + u64 rx_tls_decrypted_bytes; + u64 rx_tls_ctx; + u64 rx_tls_del; + u64 rx_tls_resync_req_pkt; + u64 rx_tls_resync_req_start; + u64 rx_tls_resync_req_end; + u64 rx_tls_resync_req_skip; + u64 rx_tls_resync_res_ok; + u64 rx_tls_resync_res_skip; + u64 rx_tls_err; #endif u64 rx_xsk_packets; @@ -305,6 +317,19 @@ struct mlx5e_rq_stats { u64 congst_umr; u64 arfs_err; u64 recover; +#ifdef CONFIG_MLX5_EN_TLS + u64 tls_decrypted_packets; + u64 tls_decrypted_bytes; + u64 tls_ctx; + u64 tls_del; + u64 tls_resync_req_pkt; + u64 tls_resync_req_start; + u64 tls_resync_req_end; + u64 tls_resync_req_skip; + u64 tls_resync_res_ok; + u64 tls_resync_res_skip; + u64 tls_err; +#endif }; struct mlx5e_sq_stats { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index cc8412151ca0..fd53d101d8fd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -63,6 +63,7 @@ #include "en/tc_tun.h" #include "en/mapping.h" #include "en/tc_ct.h" +#include "en/mod_hdr.h" #include "lib/devcom.h" #include "lib/geneve.h" #include "diag/en_tc_tracepoint.h" @@ -140,8 +141,7 @@ struct mlx5e_tc_flow { */ struct encap_flow_item encaps[MLX5_MAX_FLOW_FWD_VPORTS]; struct mlx5e_tc_flow *peer_flow; - struct mlx5e_mod_hdr_entry *mh; /* attached mod header instance */ - struct list_head mod_hdr; /* flows sharing the same mod hdr ID */ + struct mlx5e_mod_hdr_handle *mh; /* attached mod header instance */ struct mlx5e_hairpin_entry *hpe; /* attached hairpin instance */ struct list_head hairpin; /* flows sharing the same hairpin */ struct list_head peer; /* flows with peer flow */ @@ -180,17 +180,17 @@ struct mlx5e_tc_attr_to_reg_mapping mlx5e_tc_attr_to_reg_mappings[] = { }, [TUNNEL_TO_REG] = { .mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_1, - .moffset = 3, - .mlen = 1, + .moffset = 1, + .mlen = 3, .soffset = MLX5_BYTE_OFF(fte_match_param, misc_parameters_2.metadata_reg_c_1), }, [ZONE_TO_REG] = zone_to_reg_ct, + [ZONE_RESTORE_TO_REG] = zone_restore_to_reg_ct, [CTSTATE_TO_REG] = ctstate_to_reg_ct, [MARK_TO_REG] = mark_to_reg_ct, [LABELS_TO_REG] = labels_to_reg_ct, [FTEID_TO_REG] = fteid_to_reg_ct, - [TUPLEID_TO_REG] = tupleid_to_reg_ct, }; static void mlx5e_put_flow_tunnel_id(struct mlx5e_tc_flow *flow); @@ -219,6 +219,28 @@ mlx5e_tc_match_to_reg_match(struct mlx5_flow_spec *spec, spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; } +void +mlx5e_tc_match_to_reg_get_match(struct mlx5_flow_spec *spec, + enum mlx5e_tc_attr_to_reg type, + u32 *data, + u32 *mask) +{ + int soffset = mlx5e_tc_attr_to_reg_mappings[type].soffset; + int match_len = mlx5e_tc_attr_to_reg_mappings[type].mlen; + void *headers_c = spec->match_criteria; + void *headers_v = spec->match_value; + void *fmask, *fval; + + fmask = headers_c + soffset; + fval = headers_v + soffset; + + memcpy(mask, fmask, match_len); + memcpy(data, fval, match_len); + + *mask = be32_to_cpu((__force __be32)(*mask << (32 - (match_len * 8)))); + *data = be32_to_cpu((__force __be32)(*data << (32 - (match_len * 8)))); +} + int mlx5e_tc_match_to_reg_set(struct mlx5_core_dev *mdev, struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts, @@ -287,29 +309,6 @@ struct mlx5e_hairpin_entry { struct completion res_ready; }; -struct mod_hdr_key { - int num_actions; - void *actions; -}; - -struct mlx5e_mod_hdr_entry { - /* a node of a hash table which keeps all the mod_hdr entries */ - struct hlist_node mod_hdr_hlist; - - /* protects flows list */ - spinlock_t flows_lock; - /* flows sharing the same mod_hdr entry */ - struct list_head flows; - - struct mod_hdr_key key; - - struct mlx5_modify_hdr *modify_hdr; - - refcount_t refcnt; - struct completion res_ready; - int compl_result; -}; - static void mlx5e_tc_del_flow(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow); @@ -386,148 +385,43 @@ static bool mlx5e_is_offloaded_flow(struct mlx5e_tc_flow *flow) return flow_flag_test(flow, OFFLOADED); } -static inline u32 hash_mod_hdr_info(struct mod_hdr_key *key) -{ - return jhash(key->actions, - key->num_actions * MLX5_MH_ACT_SZ, 0); -} - -static inline int cmp_mod_hdr_info(struct mod_hdr_key *a, - struct mod_hdr_key *b) +static int get_flow_name_space(struct mlx5e_tc_flow *flow) { - if (a->num_actions != b->num_actions) - return 1; - - return memcmp(a->actions, b->actions, a->num_actions * MLX5_MH_ACT_SZ); + return mlx5e_is_eswitch_flow(flow) ? + MLX5_FLOW_NAMESPACE_FDB : MLX5_FLOW_NAMESPACE_KERNEL; } static struct mod_hdr_tbl * -get_mod_hdr_table(struct mlx5e_priv *priv, int namespace) +get_mod_hdr_table(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow) { struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; - return namespace == MLX5_FLOW_NAMESPACE_FDB ? &esw->offloads.mod_hdr : + return get_flow_name_space(flow) == MLX5_FLOW_NAMESPACE_FDB ? + &esw->offloads.mod_hdr : &priv->fs.tc.mod_hdr; } -static struct mlx5e_mod_hdr_entry * -mlx5e_mod_hdr_get(struct mod_hdr_tbl *tbl, struct mod_hdr_key *key, u32 hash_key) -{ - struct mlx5e_mod_hdr_entry *mh, *found = NULL; - - hash_for_each_possible(tbl->hlist, mh, mod_hdr_hlist, hash_key) { - if (!cmp_mod_hdr_info(&mh->key, key)) { - refcount_inc(&mh->refcnt); - found = mh; - break; - } - } - - return found; -} - -static void mlx5e_mod_hdr_put(struct mlx5e_priv *priv, - struct mlx5e_mod_hdr_entry *mh, - int namespace) -{ - struct mod_hdr_tbl *tbl = get_mod_hdr_table(priv, namespace); - - if (!refcount_dec_and_mutex_lock(&mh->refcnt, &tbl->lock)) - return; - hash_del(&mh->mod_hdr_hlist); - mutex_unlock(&tbl->lock); - - WARN_ON(!list_empty(&mh->flows)); - if (mh->compl_result > 0) - mlx5_modify_header_dealloc(priv->mdev, mh->modify_hdr); - - kfree(mh); -} - -static int get_flow_name_space(struct mlx5e_tc_flow *flow) -{ - return mlx5e_is_eswitch_flow(flow) ? - MLX5_FLOW_NAMESPACE_FDB : MLX5_FLOW_NAMESPACE_KERNEL; -} static int mlx5e_attach_mod_hdr(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow, struct mlx5e_tc_flow_parse_attr *parse_attr) { - int num_actions, actions_size, namespace, err; - struct mlx5e_mod_hdr_entry *mh; - struct mod_hdr_tbl *tbl; - struct mod_hdr_key key; - u32 hash_key; - - num_actions = parse_attr->mod_hdr_acts.num_actions; - actions_size = MLX5_MH_ACT_SZ * num_actions; - - key.actions = parse_attr->mod_hdr_acts.actions; - key.num_actions = num_actions; - - hash_key = hash_mod_hdr_info(&key); - - namespace = get_flow_name_space(flow); - tbl = get_mod_hdr_table(priv, namespace); - - mutex_lock(&tbl->lock); - mh = mlx5e_mod_hdr_get(tbl, &key, hash_key); - if (mh) { - mutex_unlock(&tbl->lock); - wait_for_completion(&mh->res_ready); - - if (mh->compl_result < 0) { - err = -EREMOTEIO; - goto attach_header_err; - } - goto attach_flow; - } - - mh = kzalloc(sizeof(*mh) + actions_size, GFP_KERNEL); - if (!mh) { - mutex_unlock(&tbl->lock); - return -ENOMEM; - } - - mh->key.actions = (void *)mh + sizeof(*mh); - memcpy(mh->key.actions, key.actions, actions_size); - mh->key.num_actions = num_actions; - spin_lock_init(&mh->flows_lock); - INIT_LIST_HEAD(&mh->flows); - refcount_set(&mh->refcnt, 1); - init_completion(&mh->res_ready); - - hash_add(tbl->hlist, &mh->mod_hdr_hlist, hash_key); - mutex_unlock(&tbl->lock); + struct mlx5_modify_hdr *modify_hdr; + struct mlx5e_mod_hdr_handle *mh; - mh->modify_hdr = mlx5_modify_header_alloc(priv->mdev, namespace, - mh->key.num_actions, - mh->key.actions); - if (IS_ERR(mh->modify_hdr)) { - err = PTR_ERR(mh->modify_hdr); - mh->compl_result = err; - goto alloc_header_err; - } - mh->compl_result = 1; - complete_all(&mh->res_ready); + mh = mlx5e_mod_hdr_attach(priv->mdev, get_mod_hdr_table(priv, flow), + get_flow_name_space(flow), + &parse_attr->mod_hdr_acts); + if (IS_ERR(mh)) + return PTR_ERR(mh); -attach_flow: - flow->mh = mh; - spin_lock(&mh->flows_lock); - list_add(&flow->mod_hdr, &mh->flows); - spin_unlock(&mh->flows_lock); + modify_hdr = mlx5e_mod_hdr_get(mh); if (mlx5e_is_eswitch_flow(flow)) - flow->esw_attr->modify_hdr = mh->modify_hdr; + flow->esw_attr->modify_hdr = modify_hdr; else - flow->nic_attr->modify_hdr = mh->modify_hdr; + flow->nic_attr->modify_hdr = modify_hdr; + flow->mh = mh; return 0; - -alloc_header_err: - complete_all(&mh->res_ready); -attach_header_err: - mlx5e_mod_hdr_put(priv, mh, namespace); - return err; } static void mlx5e_detach_mod_hdr(struct mlx5e_priv *priv, @@ -537,11 +431,8 @@ static void mlx5e_detach_mod_hdr(struct mlx5e_priv *priv, if (!flow->mh) return; - spin_lock(&flow->mh->flows_lock); - list_del(&flow->mod_hdr); - spin_unlock(&flow->mh->flows_lock); - - mlx5e_mod_hdr_put(priv, flow->mh, get_flow_name_space(flow)); + mlx5e_mod_hdr_detach(priv->mdev, get_mod_hdr_table(priv, flow), + flow->mh); flow->mh = NULL; } @@ -2356,6 +2247,7 @@ static int __parse_cls_flower(struct mlx5e_priv *priv, match.key->vlan_priority); *match_level = MLX5_MATCH_L2; + spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; } } @@ -3086,6 +2978,7 @@ struct ipv6_hoplimit_word { static int is_action_keys_supported(const struct flow_action_entry *act, bool ct_flow, bool *modify_ip_header, + bool *modify_tuple, struct netlink_ext_ack *extack) { u32 mask, offset; @@ -3108,7 +3001,10 @@ static int is_action_keys_supported(const struct flow_action_entry *act, *modify_ip_header = true; } - if (ct_flow && offset >= offsetof(struct iphdr, saddr)) { + if (offset >= offsetof(struct iphdr, saddr)) + *modify_tuple = true; + + if (ct_flow && *modify_tuple) { NL_SET_ERR_MSG_MOD(extack, "can't offload re-write of ipv4 address with action ct"); return -EOPNOTSUPP; @@ -3123,28 +3019,36 @@ static int is_action_keys_supported(const struct flow_action_entry *act, *modify_ip_header = true; } - if (ct_flow && offset >= offsetof(struct ipv6hdr, saddr)) { + if (ct_flow && offset >= offsetof(struct ipv6hdr, saddr)) + *modify_tuple = true; + + if (ct_flow && *modify_tuple) { NL_SET_ERR_MSG_MOD(extack, "can't offload re-write of ipv6 address with action ct"); return -EOPNOTSUPP; } - } else if (ct_flow && (htype == FLOW_ACT_MANGLE_HDR_TYPE_TCP || - htype == FLOW_ACT_MANGLE_HDR_TYPE_UDP)) { - NL_SET_ERR_MSG_MOD(extack, - "can't offload re-write of transport header ports with action ct"); - return -EOPNOTSUPP; + } else if (htype == FLOW_ACT_MANGLE_HDR_TYPE_TCP || + htype == FLOW_ACT_MANGLE_HDR_TYPE_UDP) { + *modify_tuple = true; + if (ct_flow) { + NL_SET_ERR_MSG_MOD(extack, + "can't offload re-write of transport header ports with action ct"); + return -EOPNOTSUPP; + } } return 0; } -static bool modify_header_match_supported(struct mlx5_flow_spec *spec, +static bool modify_header_match_supported(struct mlx5e_priv *priv, + struct mlx5_flow_spec *spec, struct flow_action *flow_action, u32 actions, bool ct_flow, + bool ct_clear, struct netlink_ext_ack *extack) { const struct flow_action_entry *act; - bool modify_ip_header; + bool modify_ip_header, modify_tuple; void *headers_c; void *headers_v; u16 ethertype; @@ -3161,23 +3065,39 @@ static bool modify_header_match_supported(struct mlx5_flow_spec *spec, goto out_ok; modify_ip_header = false; + modify_tuple = false; flow_action_for_each(i, act, flow_action) { if (act->id != FLOW_ACTION_MANGLE && act->id != FLOW_ACTION_ADD) continue; err = is_action_keys_supported(act, ct_flow, - &modify_ip_header, extack); + &modify_ip_header, + &modify_tuple, extack); if (err) return err; } + /* Add ct_state=-trk match so it will be offloaded for non ct flows + * (or after clear action), as otherwise, since the tuple is changed, + * we can't restore ct state + */ + if (!ct_clear && modify_tuple && + mlx5_tc_ct_add_no_trk_match(priv, spec)) { + NL_SET_ERR_MSG_MOD(extack, + "can't offload tuple modify header with ct matches"); + netdev_info(priv->netdev, + "can't offload tuple modify header with ct matches"); + return false; + } + ip_proto = MLX5_GET(fte_match_set_lyr_2_4, headers_v, ip_protocol); if (modify_ip_header && ip_proto != IPPROTO_TCP && ip_proto != IPPROTO_UDP && ip_proto != IPPROTO_ICMP) { NL_SET_ERR_MSG_MOD(extack, "can't offload re-write of non TCP/UDP"); - pr_info("can't offload re-write of ip proto %d\n", ip_proto); + netdev_info(priv->netdev, "can't offload re-write of ip proto %d\n", + ip_proto); return false; } @@ -3191,13 +3111,14 @@ static bool actions_match_supported(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow, struct netlink_ext_ack *extack) { - bool ct_flow; + bool ct_flow = false, ct_clear = false; u32 actions; - ct_flow = flow_flag_test(flow, CT); if (mlx5e_is_eswitch_flow(flow)) { actions = flow->esw_attr->action; - + ct_clear = flow->esw_attr->ct_attr.ct_action & + TCA_CT_ACT_CLEAR; + ct_flow = flow_flag_test(flow, CT) && !ct_clear; if (flow->esw_attr->split_count && ct_flow) { /* All registers used by ct are cleared when using * split rules. @@ -3211,9 +3132,10 @@ static bool actions_match_supported(struct mlx5e_priv *priv, } if (actions & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) - return modify_header_match_supported(&parse_attr->spec, + return modify_header_match_supported(priv, &parse_attr->spec, flow_action, actions, - ct_flow, extack); + ct_flow, ct_clear, + extack); return true; } @@ -4408,7 +4330,6 @@ mlx5e_alloc_flow(struct mlx5e_priv *priv, int attr_size, flow->priv = priv; for (out_index = 0; out_index < MLX5_MAX_FLOW_FWD_VPORTS; out_index++) INIT_LIST_HEAD(&flow->encaps[out_index].list); - INIT_LIST_HEAD(&flow->mod_hdr); INIT_LIST_HEAD(&flow->hairpin); INIT_LIST_HEAD(&flow->l3_to_l2_reformat); refcount_set(&flow->refcnt, 1); @@ -4480,11 +4401,13 @@ __mlx5e_add_fdb_flow(struct mlx5e_priv *priv, if (err) goto err_free; - err = parse_tc_fdb_actions(priv, &rule->action, flow, extack, filter_dev); + /* actions validation depends on parsing the ct matches first */ + err = mlx5_tc_ct_parse_match(priv, &parse_attr->spec, f, + &flow->esw_attr->ct_attr, extack); if (err) goto err_free; - err = mlx5_tc_ct_parse_match(priv, &parse_attr->spec, f, extack); + err = parse_tc_fdb_actions(priv, &rule->action, flow, extack, filter_dev); if (err) goto err_free; @@ -4832,7 +4755,7 @@ int mlx5e_stats_flower(struct net_device *dev, struct mlx5e_priv *priv, no_peer_counter: mlx5_devcom_release_peer_data(devcom, MLX5_DEVCOM_ESW_OFFLOADS); out: - flow_stats_update(&f->stats, bytes, packets, lastuse, + flow_stats_update(&f->stats, bytes, packets, 0, lastuse, FLOW_ACTION_HW_STATS_DELAYED); trace_mlx5e_stats_flower(f); errout: @@ -4950,7 +4873,7 @@ void mlx5e_tc_stats_matchall(struct mlx5e_priv *priv, dpkts = cur_stats.rx_packets - rpriv->prev_vf_vport_stats.rx_packets; dbytes = cur_stats.rx_bytes - rpriv->prev_vf_vport_stats.rx_bytes; rpriv->prev_vf_vport_stats = cur_stats; - flow_stats_update(&ma->stats, dbytes, dpkts, jiffies, + flow_stats_update(&ma->stats, dbytes, dpkts, 0, jiffies, FLOW_ACTION_HW_STATS_DELAYED); } @@ -5015,9 +4938,8 @@ int mlx5e_tc_nic_init(struct mlx5e_priv *priv) struct mlx5e_tc_table *tc = &priv->fs.tc; int err; + mlx5e_mod_hdr_tbl_init(&tc->mod_hdr); mutex_init(&tc->t_lock); - mutex_init(&tc->mod_hdr.lock); - hash_init(tc->mod_hdr.hlist); mutex_init(&tc->hairpin_tbl_lock); hash_init(tc->hairpin_tbl); @@ -5055,7 +4977,7 @@ void mlx5e_tc_nic_cleanup(struct mlx5e_priv *priv) &tc->netdevice_nb, &tc->netdevice_nn); - mutex_destroy(&tc->mod_hdr.lock); + mlx5e_mod_hdr_tbl_destroy(&tc->mod_hdr); mutex_destroy(&tc->hairpin_tbl_lock); rhashtable_destroy(&tc->ht); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h index 5c330b0cae21..437f680728fd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h @@ -40,6 +40,14 @@ #ifdef CONFIG_MLX5_ESWITCH +int mlx5e_tc_num_filters(struct mlx5e_priv *priv, unsigned long flags); + +struct mlx5e_tc_update_priv { + struct net_device *tun_dev; +}; + +#if IS_ENABLED(CONFIG_MLX5_CLS_ACT) + struct tunnel_match_key { struct flow_dissector_key_control enc_control; struct flow_dissector_key_keyid enc_key_id; @@ -62,9 +70,9 @@ struct tunnel_match_enc_opts { * Upper TUNNEL_INFO_BITS for general tunnel info. * Lower ENC_OPTS_BITS bits for enc_opts. */ -#define TUNNEL_INFO_BITS 6 +#define TUNNEL_INFO_BITS 12 #define TUNNEL_INFO_BITS_MASK GENMASK(TUNNEL_INFO_BITS - 1, 0) -#define ENC_OPTS_BITS 2 +#define ENC_OPTS_BITS 12 #define ENC_OPTS_BITS_MASK GENMASK(ENC_OPTS_BITS - 1, 0) #define TUNNEL_ID_BITS (TUNNEL_INFO_BITS + ENC_OPTS_BITS) #define TUNNEL_ID_MASK GENMASK(TUNNEL_ID_BITS - 1, 0) @@ -114,8 +122,6 @@ void mlx5e_put_encap_flow_list(struct mlx5e_priv *priv, struct list_head *flow_l struct mlx5e_neigh_hash_entry; void mlx5e_tc_update_neigh_used_value(struct mlx5e_neigh_hash_entry *nhe); -int mlx5e_tc_num_filters(struct mlx5e_priv *priv, unsigned long flags); - void mlx5e_tc_reoffload_flows_work(struct work_struct *work); enum mlx5e_tc_attr_to_reg { @@ -123,10 +129,10 @@ enum mlx5e_tc_attr_to_reg { TUNNEL_TO_REG, CTSTATE_TO_REG, ZONE_TO_REG, + ZONE_RESTORE_TO_REG, MARK_TO_REG, LABELS_TO_REG, FTEID_TO_REG, - TUPLEID_TO_REG, }; struct mlx5e_tc_attr_to_reg_mapping { @@ -142,16 +148,6 @@ extern struct mlx5e_tc_attr_to_reg_mapping mlx5e_tc_attr_to_reg_mappings[]; bool mlx5e_is_valid_eswitch_fwd_dev(struct mlx5e_priv *priv, struct net_device *out_dev); -struct mlx5e_tc_update_priv { - struct net_device *tun_dev; -}; - -struct mlx5e_tc_mod_hdr_acts { - int num_actions; - int max_actions; - void *actions; -}; - int mlx5e_tc_match_to_reg_set(struct mlx5_core_dev *mdev, struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts, enum mlx5e_tc_attr_to_reg type, @@ -162,6 +158,11 @@ void mlx5e_tc_match_to_reg_match(struct mlx5_flow_spec *spec, u32 data, u32 mask); +void mlx5e_tc_match_to_reg_get_match(struct mlx5_flow_spec *spec, + enum mlx5e_tc_attr_to_reg type, + u32 *data, + u32 *mask); + int alloc_mod_hdr_actions(struct mlx5_core_dev *mdev, int namespace, struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts); @@ -174,8 +175,6 @@ void mlx5e_tc_set_ethertype(struct mlx5_core_dev *mdev, struct flow_match_basic *match, bool outer, void *headers_c, void *headers_v); -#if IS_ENABLED(CONFIG_MLX5_CLS_ACT) - int mlx5e_tc_nic_init(struct mlx5e_priv *priv); void mlx5e_tc_nic_cleanup(struct mlx5e_priv *priv); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c index 6d406063aca4..da596de3abba 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c @@ -38,7 +38,6 @@ #include "en/txrx.h" #include "ipoib/ipoib.h" #include "en_accel/en_accel.h" -#include "en_accel/ktls.h" #include "lib/clock.h" static void mlx5e_dma_unmap_wqe_err(struct mlx5e_txqsq *sq, u8 num_dma) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c index 8480278f2ee2..de10b06bade5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c @@ -31,8 +31,8 @@ */ #include <linux/irq.h> -#include <linux/indirect_call_wrapper.h> #include "en.h" +#include "en/txrx.h" #include "en/xdp.h" #include "en/xsk/rx.h" #include "en/xsk/tx.h" @@ -149,17 +149,17 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget) } mlx5e_poll_ico_cq(&c->icosq.cq); + if (mlx5e_poll_ico_cq(&c->async_icosq.cq)) + /* Don't clear the flag if nothing was polled to prevent + * queueing more WQEs and overflowing the async ICOSQ. + */ + clear_bit(MLX5E_SQ_STATE_PENDING_XSK_TX, &c->async_icosq.state); busy |= INDIRECT_CALL_2(rq->post_wqes, mlx5e_post_rx_mpwqes, mlx5e_post_rx_wqes, rq); if (xsk_open) { - if (mlx5e_poll_ico_cq(&c->xskicosq.cq)) - /* Don't clear the flag if nothing was polled to prevent - * queueing more WQEs and overflowing XSKICOSQ. - */ - clear_bit(MLX5E_SQ_STATE_PENDING_XSK_TX, &c->xskicosq.state); busy |= mlx5e_poll_xdpsq_cq(&xsksq->cq); busy_xsk |= mlx5e_napi_xsk_post(xsksq, xskrq); } @@ -189,11 +189,11 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget) mlx5e_cq_arm(&rq->cq); mlx5e_cq_arm(&c->icosq.cq); + mlx5e_cq_arm(&c->async_icosq.cq); mlx5e_cq_arm(&c->xdpsq.cq); if (xsk_open) { mlx5e_handle_rx_dim(xskrq); - mlx5e_cq_arm(&c->xskicosq.cq); mlx5e_cq_arm(&xsksq->cq); mlx5e_cq_arm(&xskrq->cq); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index 1116ab9bea6c..6e6a9a563992 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -42,6 +42,7 @@ #include "fs_core.h" #include "devlink.h" #include "ecpf.h" +#include "en/mod_hdr.h" enum { MLX5_ACTION_NONE = 0, @@ -63,6 +64,29 @@ struct vport_addr { static void esw_destroy_legacy_fdb_table(struct mlx5_eswitch *esw); static void esw_cleanup_vepa_rules(struct mlx5_eswitch *esw); +static int mlx5_eswitch_check(const struct mlx5_core_dev *dev) +{ + if (MLX5_CAP_GEN(dev, port_type) != MLX5_CAP_PORT_TYPE_ETH) + return -EOPNOTSUPP; + + if (!MLX5_ESWITCH_MANAGER(dev)) + return -EOPNOTSUPP; + + return 0; +} + +struct mlx5_eswitch *mlx5_devlink_eswitch_get(struct devlink *devlink) +{ + struct mlx5_core_dev *dev = devlink_priv(devlink); + int err; + + err = mlx5_eswitch_check(dev); + if (err) + return ERR_PTR(err); + + return dev->priv.eswitch; +} + struct mlx5_vport *__must_check mlx5_eswitch_get_vport(struct mlx5_eswitch *esw, u16 vport_num) { @@ -1127,7 +1151,7 @@ int mlx5_esw_modify_vport_rate(struct mlx5_eswitch *esw, u16 vport_num, MODIFY_SCHEDULING_ELEMENT_IN_MODIFY_BITMASK_MAX_AVERAGE_BW); } -static void node_guid_gen_from_mac(u64 *node_guid, u8 mac[ETH_ALEN]) +static void node_guid_gen_from_mac(u64 *node_guid, const u8 *mac) { ((u8 *)node_guid)[7] = mac[0]; ((u8 *)node_guid)[6] = mac[1]; @@ -1608,7 +1632,7 @@ abort: mlx5_reload_interface(esw->dev, MLX5_INTERFACE_PROTOCOL_IB); mlx5_reload_interface(esw->dev, MLX5_INTERFACE_PROTOCOL_ETH); } - + esw_destroy_tsar(esw); return err; } @@ -1628,7 +1652,17 @@ int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs) return 0; mutex_lock(&esw->mode_lock); - ret = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY, num_vfs); + if (esw->mode == MLX5_ESWITCH_NONE) { + ret = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY, num_vfs); + } else { + enum mlx5_eswitch_vport_event vport_events; + + vport_events = (esw->mode == MLX5_ESWITCH_LEGACY) ? + MLX5_LEGACY_SRIOV_VPORT_EVENTS : MLX5_VPORT_UC_ADDR_CHANGE; + ret = mlx5_eswitch_load_vf_vports(esw, num_vfs, vport_events); + if (!ret) + esw->esw_funcs.num_vfs = num_vfs; + } mutex_unlock(&esw->mode_lock); return ret; } @@ -1653,8 +1687,6 @@ void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw, bool clear_vf) else if (esw->mode == MLX5_ESWITCH_OFFLOADS) esw_offloads_disable(esw); - esw_destroy_tsar(esw); - old_mode = esw->mode; esw->mode = MLX5_ESWITCH_NONE; @@ -1664,6 +1696,8 @@ void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw, bool clear_vf) mlx5_reload_interface(esw->dev, MLX5_INTERFACE_PROTOCOL_IB); mlx5_reload_interface(esw->dev, MLX5_INTERFACE_PROTOCOL_ETH); } + esw_destroy_tsar(esw); + if (clear_vf) mlx5_eswitch_clear_vf_vports_info(esw); } @@ -1675,6 +1709,7 @@ void mlx5_eswitch_disable(struct mlx5_eswitch *esw, bool clear_vf) mutex_lock(&esw->mode_lock); mlx5_eswitch_disable_locked(esw, clear_vf); + esw->esw_funcs.num_vfs = 0; mutex_unlock(&esw->mode_lock); } @@ -1725,10 +1760,9 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev) mutex_init(&esw->offloads.encap_tbl_lock); hash_init(esw->offloads.encap_tbl); - mutex_init(&esw->offloads.mod_hdr.lock); - hash_init(esw->offloads.mod_hdr.hlist); mutex_init(&esw->offloads.decap_tbl_lock); hash_init(esw->offloads.decap_tbl); + mlx5e_mod_hdr_tbl_init(&esw->offloads.mod_hdr); atomic64_set(&esw->offloads.num_flows, 0); ida_init(&esw->offloads.vport_metadata_ida); mutex_init(&esw->state_lock); @@ -1770,7 +1804,7 @@ void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw) mutex_destroy(&esw->mode_lock); mutex_destroy(&esw->state_lock); ida_destroy(&esw->offloads.vport_metadata_ida); - mutex_destroy(&esw->offloads.mod_hdr.lock); + mlx5e_mod_hdr_tbl_destroy(&esw->offloads.mod_hdr); mutex_destroy(&esw->offloads.encap_tbl_lock); mutex_destroy(&esw->offloads.decap_tbl_lock); kfree(esw->vports); @@ -1778,46 +1812,135 @@ void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw) } /* Vport Administration */ -int mlx5_eswitch_set_vport_mac(struct mlx5_eswitch *esw, - u16 vport, u8 mac[ETH_ALEN]) +static int +mlx5_esw_set_vport_mac_locked(struct mlx5_eswitch *esw, + struct mlx5_vport *evport, const u8 *mac) { - struct mlx5_vport *evport = mlx5_eswitch_get_vport(esw, vport); + u16 vport_num = evport->vport; u64 node_guid; int err = 0; - if (IS_ERR(evport)) - return PTR_ERR(evport); if (is_multicast_ether_addr(mac)) return -EINVAL; - mutex_lock(&esw->state_lock); - if (evport->info.spoofchk && !is_valid_ether_addr(mac)) mlx5_core_warn(esw->dev, "Set invalid MAC while spoofchk is on, vport(%d)\n", - vport); + vport_num); - err = mlx5_modify_nic_vport_mac_address(esw->dev, vport, mac); + err = mlx5_modify_nic_vport_mac_address(esw->dev, vport_num, mac); if (err) { mlx5_core_warn(esw->dev, "Failed to mlx5_modify_nic_vport_mac vport(%d) err=(%d)\n", - vport, err); - goto unlock; + vport_num, err); + return err; } node_guid_gen_from_mac(&node_guid, mac); - err = mlx5_modify_nic_vport_node_guid(esw->dev, vport, node_guid); + err = mlx5_modify_nic_vport_node_guid(esw->dev, vport_num, node_guid); if (err) mlx5_core_warn(esw->dev, "Failed to set vport %d node guid, err = %d. RDMA_CM will not function properly for this VF.\n", - vport, err); + vport_num, err); ether_addr_copy(evport->info.mac, mac); evport->info.node_guid = node_guid; if (evport->enabled && esw->mode == MLX5_ESWITCH_LEGACY) err = esw_acl_ingress_lgcy_setup(esw, evport); -unlock: + return err; +} + +int mlx5_eswitch_set_vport_mac(struct mlx5_eswitch *esw, + u16 vport, const u8 *mac) +{ + struct mlx5_vport *evport = mlx5_eswitch_get_vport(esw, vport); + int err = 0; + + if (IS_ERR(evport)) + return PTR_ERR(evport); + + mutex_lock(&esw->state_lock); + err = mlx5_esw_set_vport_mac_locked(esw, evport, mac); + mutex_unlock(&esw->state_lock); + return err; +} + +static bool +is_port_function_supported(const struct mlx5_eswitch *esw, u16 vport_num) +{ + return vport_num == MLX5_VPORT_PF || + mlx5_eswitch_is_vf_vport(esw, vport_num); +} + +int mlx5_devlink_port_function_hw_addr_get(struct devlink *devlink, + struct devlink_port *port, + u8 *hw_addr, int *hw_addr_len, + struct netlink_ext_ack *extack) +{ + struct mlx5_eswitch *esw; + struct mlx5_vport *vport; + int err = -EOPNOTSUPP; + u16 vport_num; + + esw = mlx5_devlink_eswitch_get(devlink); + if (IS_ERR(esw)) + return PTR_ERR(esw); + + vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); + if (!is_port_function_supported(esw, vport_num)) + return -EOPNOTSUPP; + + vport = mlx5_eswitch_get_vport(esw, vport_num); + if (IS_ERR(vport)) { + NL_SET_ERR_MSG_MOD(extack, "Invalid port"); + return PTR_ERR(vport); + } + + mutex_lock(&esw->state_lock); + if (vport->enabled) { + ether_addr_copy(hw_addr, vport->info.mac); + *hw_addr_len = ETH_ALEN; + err = 0; + } else { + NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled"); + } + mutex_unlock(&esw->state_lock); + return err; +} + +int mlx5_devlink_port_function_hw_addr_set(struct devlink *devlink, + struct devlink_port *port, + const u8 *hw_addr, int hw_addr_len, + struct netlink_ext_ack *extack) +{ + struct mlx5_eswitch *esw; + struct mlx5_vport *vport; + int err = -EOPNOTSUPP; + u16 vport_num; + + esw = mlx5_devlink_eswitch_get(devlink); + if (IS_ERR(esw)) { + NL_SET_ERR_MSG_MOD(extack, "Eswitch doesn't support set hw_addr"); + return PTR_ERR(esw); + } + + vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); + if (!is_port_function_supported(esw, vport_num)) { + NL_SET_ERR_MSG_MOD(extack, "Port doesn't support set hw_addr"); + return -EINVAL; + } + vport = mlx5_eswitch_get_vport(esw, vport_num); + if (IS_ERR(vport)) { + NL_SET_ERR_MSG_MOD(extack, "Invalid port"); + return PTR_ERR(vport); + } + + mutex_lock(&esw->state_lock); + if (vport->enabled) + err = mlx5_esw_set_vport_mac_locked(esw, vport, hw_addr); + else + NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled"); mutex_unlock(&esw->state_lock); return err; } @@ -1826,6 +1949,8 @@ int mlx5_eswitch_set_vport_state(struct mlx5_eswitch *esw, u16 vport, int link_state) { struct mlx5_vport *evport = mlx5_eswitch_get_vport(esw, vport); + int opmod = MLX5_VPORT_STATE_OP_MOD_ESW_VPORT; + int other_vport = 1; int err = 0; if (!ESW_ALLOWED(esw)) @@ -1833,15 +1958,17 @@ int mlx5_eswitch_set_vport_state(struct mlx5_eswitch *esw, if (IS_ERR(evport)) return PTR_ERR(evport); + if (vport == MLX5_VPORT_UPLINK) { + opmod = MLX5_VPORT_STATE_OP_MOD_UPLINK; + other_vport = 0; + vport = 0; + } mutex_lock(&esw->state_lock); - err = mlx5_modify_vport_admin_state(esw->dev, - MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, - vport, 1, link_state); + err = mlx5_modify_vport_admin_state(esw->dev, opmod, vport, other_vport, link_state); if (err) { - mlx5_core_warn(esw->dev, - "Failed to set vport %d link state, err = %d", - vport, err); + mlx5_core_warn(esw->dev, "Failed to set vport %d link state, opmod = %d, err = %d", + vport, opmod, err); goto unlock; } @@ -1883,8 +2010,6 @@ int __mlx5_eswitch_set_vport_vlan(struct mlx5_eswitch *esw, struct mlx5_vport *evport = mlx5_eswitch_get_vport(esw, vport); int err = 0; - if (!ESW_ALLOWED(esw)) - return -EPERM; if (IS_ERR(evport)) return PTR_ERR(evport); if (vlan > 4095 || qos > 7) @@ -1912,6 +2037,9 @@ int mlx5_eswitch_set_vport_vlan(struct mlx5_eswitch *esw, u8 set_flags = 0; int err; + if (!ESW_ALLOWED(esw)) + return -EPERM; + if (vlan || qos) set_flags = SET_VLAN_STRIP | SET_VLAN_INSERT; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index a5175e98c0b3..867d8120b8a5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -44,16 +44,6 @@ #include "lib/mpfs.h" #include "en/tc_ct.h" -#define FDB_TC_MAX_CHAIN 3 -#define FDB_FT_CHAIN (FDB_TC_MAX_CHAIN + 1) -#define FDB_TC_SLOW_PATH_CHAIN (FDB_FT_CHAIN + 1) - -/* The index of the last real chain (FT) + 1 as chain zero is valid as well */ -#define FDB_NUM_CHAINS (FDB_FT_CHAIN + 1) - -#define FDB_TC_MAX_PRIO 16 -#define FDB_TC_LEVELS_PER_PRIO 2 - #ifdef CONFIG_MLX5_ESWITCH #define ESW_OFFLOADS_DEFAULT_NUM_GROUPS 15 @@ -281,7 +271,6 @@ struct mlx5_eswitch { struct mlx5_esw_offload offloads; int mode; - int nvports; u16 manager_vport; u16 first_host_vport; struct mlx5_esw_functions esw_funcs; @@ -311,7 +300,7 @@ int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs); void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw, bool clear_vf); void mlx5_eswitch_disable(struct mlx5_eswitch *esw, bool clear_vf); int mlx5_eswitch_set_vport_mac(struct mlx5_eswitch *esw, - u16 vport, u8 mac[ETH_ALEN]); + u16 vport, const u8 *mac); int mlx5_eswitch_set_vport_state(struct mlx5_eswitch *esw, u16 vport, int link_state); int mlx5_eswitch_set_vport_vlan(struct mlx5_eswitch *esw, @@ -450,6 +439,15 @@ int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink, struct netlink_ext_ack *extack); int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink, enum devlink_eswitch_encap_mode *encap); +int mlx5_devlink_port_function_hw_addr_get(struct devlink *devlink, + struct devlink_port *port, + u8 *hw_addr, int *hw_addr_len, + struct netlink_ext_ack *extack); +int mlx5_devlink_port_function_hw_addr_set(struct devlink *devlink, + struct devlink_port *port, + const u8 *hw_addr, int hw_addr_len, + struct netlink_ext_ack *extack); + void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type); int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch *esw, @@ -514,16 +512,9 @@ static inline u16 mlx5_eswitch_first_host_vport_num(struct mlx5_core_dev *dev) MLX5_VPORT_PF : MLX5_VPORT_FIRST_VF; } -static inline bool mlx5_eswitch_is_funcs_handler(struct mlx5_core_dev *dev) +static inline bool mlx5_eswitch_is_funcs_handler(const struct mlx5_core_dev *dev) { - /* Ideally device should have the functions changed supported - * capability regardless of it being ECPF or PF wherever such - * event should be processed such as on eswitch manager device. - * However, some ECPF based device might not have this capability - * set. Hence OR for ECPF check to cover such device. - */ - return MLX5_CAP_ESW(dev, esw_functions_changed) || - mlx5_core_is_ecpf_esw_manager(dev); + return mlx5_core_is_ecpf_esw_manager(dev); } static inline int mlx5_eswitch_uplink_idx(struct mlx5_eswitch *esw) @@ -565,6 +556,19 @@ static inline u16 mlx5_eswitch_index_to_vport_num(struct mlx5_eswitch *esw, return index; } +static inline unsigned int +mlx5_esw_vport_to_devlink_port_index(const struct mlx5_core_dev *dev, + u16 vport_num) +{ + return (MLX5_CAP_GEN(dev, vhca_id) << 16) | vport_num; +} + +static inline u16 +mlx5_esw_devlink_port_index_to_vport_num(unsigned int dl_port_index) +{ + return dl_port_index & 0xffff; +} + /* TODO: This mlx5e_tc function shouldn't be called by eswitch */ void mlx5e_tc_clean_fdb_peer_flows(struct mlx5_eswitch *esw); @@ -634,6 +638,7 @@ void mlx5e_tc_clean_fdb_peer_flows(struct mlx5_eswitch *esw); for ((vport) = (nvfs); \ (vport) >= (esw)->first_host_vport; (vport)--) +struct mlx5_eswitch *mlx5_devlink_eswitch_get(struct devlink *devlink); struct mlx5_vport *__must_check mlx5_eswitch_get_vport(struct mlx5_eswitch *esw, u16 vport_num); @@ -680,6 +685,8 @@ static inline int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs) { r static inline void mlx5_eswitch_disable(struct mlx5_eswitch *esw, bool clear_vf) {} static inline bool mlx5_esw_lag_prereq(struct mlx5_core_dev *dev0, struct mlx5_core_dev *dev1) { return true; } static inline bool mlx5_eswitch_is_funcs_handler(struct mlx5_core_dev *dev) { return false; } +static inline +int mlx5_eswitch_set_vport_state(struct mlx5_eswitch *esw, u16 vport, int link_state) { return 0; } static inline const u32 *mlx5_esw_query_functions(struct mlx5_core_dev *dev) { return ERR_PTR(-EOPNOTSUPP); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index 060354bb211a..d2516922d867 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -236,6 +236,15 @@ static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw, return &esw->offloads.vport_reps[idx]; } +static void +mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch *esw, + struct mlx5_flow_spec *spec, + struct mlx5_esw_flow_attr *attr) +{ + if (MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) && + attr && attr->in_rep && attr->in_rep->vport == MLX5_VPORT_UPLINK) + spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK; +} static void mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw, @@ -259,9 +268,6 @@ mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw, mlx5_eswitch_get_vport_metadata_mask()); spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; - misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); - if (memchr_inv(misc, 0, MLX5_ST_SZ_BYTES(fte_match_set_misc))) - spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; } else { misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); MLX5_SET(fte_match_set_misc, misc, source_port, attr->in_rep->vport); @@ -279,10 +285,6 @@ mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw, spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; } - - if (MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) && - attr->in_rep->vport == MLX5_VPORT_UPLINK) - spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK; } struct mlx5_flow_handle * @@ -396,6 +398,8 @@ mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, goto err_esw_get; } + mlx5_eswitch_set_rule_flow_source(esw, spec, attr); + if (mlx5_eswitch_termtbl_required(esw, attr, &flow_act, spec)) rule = mlx5_eswitch_add_termtbl_rule(esw, fdb, spec, attr, &flow_act, dest, i); @@ -462,6 +466,7 @@ mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw, i++; mlx5_eswitch_set_rule_source_port(esw, spec, attr); + mlx5_eswitch_set_rule_flow_source(esw, spec, attr); if (attr->outer_match_level != MLX5_MATCH_NONE) spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; @@ -1132,7 +1137,7 @@ static void esw_set_flow_group_source_port(struct mlx5_eswitch *esw, } } -static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw, int nvports) +static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); struct mlx5_flow_table_attr ft_attr = {}; @@ -1165,7 +1170,7 @@ static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw, int nvports) goto ns_err; } - table_size = nvports * MAX_SQ_NVPORTS + MAX_PF_SQ + + table_size = esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ + MLX5_ESW_MISS_FLOWS + esw->total_vports; /* create the slow path fdb with encap set, so further table instances @@ -1202,7 +1207,7 @@ static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw, int nvports) MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn); MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_port); - ix = nvports * MAX_SQ_NVPORTS + MAX_PF_SQ; + ix = esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ; MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ix - 1); @@ -1270,7 +1275,6 @@ static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw, int nvports) if (err) goto miss_rule_err; - esw->nvports = nvports; kvfree(flow_group_in); return 0; @@ -1311,7 +1315,7 @@ static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw) MLX5_FLOW_STEERING_MODE_DMFS); } -static int esw_create_offloads_table(struct mlx5_eswitch *esw, int nvports) +static int esw_create_offloads_table(struct mlx5_eswitch *esw) { struct mlx5_flow_table_attr ft_attr = {}; struct mlx5_core_dev *dev = esw->dev; @@ -1325,7 +1329,7 @@ static int esw_create_offloads_table(struct mlx5_eswitch *esw, int nvports) return -EOPNOTSUPP; } - ft_attr.max_fte = nvports + MLX5_ESW_MISS_FLOWS; + ft_attr.max_fte = esw->total_vports + MLX5_ESW_MISS_FLOWS; ft_attr.prio = 1; ft_offloads = mlx5_create_flow_table(ns, &ft_attr); @@ -1346,14 +1350,15 @@ static void esw_destroy_offloads_table(struct mlx5_eswitch *esw) mlx5_destroy_flow_table(offloads->ft_offloads); } -static int esw_create_vport_rx_group(struct mlx5_eswitch *esw, int nvports) +static int esw_create_vport_rx_group(struct mlx5_eswitch *esw) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); struct mlx5_flow_group *g; u32 *flow_group_in; + int nvports; int err = 0; - nvports = nvports + MLX5_ESW_MISS_FLOWS; + nvports = esw->total_vports + MLX5_ESW_MISS_FLOWS; flow_group_in = kvzalloc(inlen, GFP_KERNEL); if (!flow_group_in) return -ENOMEM; @@ -1578,13 +1583,6 @@ static int esw_offloads_start(struct mlx5_eswitch *esw, { int err, err1; - if (esw->mode != MLX5_ESWITCH_LEGACY && - !mlx5_core_is_ecpf_esw_manager(esw->dev)) { - NL_SET_ERR_MSG_MOD(extack, - "Can't set offloads mode, SRIOV legacy not enabled"); - return -EINVAL; - } - mlx5_eswitch_disable_locked(esw, false); err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_OFFLOADS, esw->dev->priv.sriov.num_vfs); @@ -1993,15 +1991,8 @@ static void esw_destroy_uplink_offloads_acl_tables(struct mlx5_eswitch *esw) static int esw_offloads_steering_init(struct mlx5_eswitch *esw) { - int num_vfs = esw->esw_funcs.num_vfs; - int total_vports; int err; - if (mlx5_core_is_ecpf_esw_manager(esw->dev)) - total_vports = esw->total_vports; - else - total_vports = num_vfs + MLX5_SPECIAL_VPORTS(esw->dev); - memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb)); mutex_init(&esw->fdb_table.offloads.vports.lock); hash_init(esw->fdb_table.offloads.vports.table); @@ -2010,7 +2001,7 @@ static int esw_offloads_steering_init(struct mlx5_eswitch *esw) if (err) goto create_acl_err; - err = esw_create_offloads_table(esw, total_vports); + err = esw_create_offloads_table(esw); if (err) goto create_offloads_err; @@ -2018,11 +2009,11 @@ static int esw_offloads_steering_init(struct mlx5_eswitch *esw) if (err) goto create_restore_err; - err = esw_create_offloads_fdb_tables(esw, total_vports); + err = esw_create_offloads_fdb_tables(esw); if (err) goto create_fdb_err; - err = esw_create_vport_rx_group(esw, total_vports); + err = esw_create_vport_rx_group(esw); if (err) goto create_fg_err; @@ -2279,17 +2270,6 @@ static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode) return 0; } -static int mlx5_eswitch_check(const struct mlx5_core_dev *dev) -{ - if (MLX5_CAP_GEN(dev, port_type) != MLX5_CAP_PORT_TYPE_ETH) - return -EOPNOTSUPP; - - if(!MLX5_ESWITCH_MANAGER(dev)) - return -EPERM; - - return 0; -} - static int eswitch_devlink_esw_mode_check(const struct mlx5_eswitch *esw) { /* devlink commands in NONE eswitch mode are currently supported only @@ -2302,25 +2282,19 @@ static int eswitch_devlink_esw_mode_check(const struct mlx5_eswitch *esw) int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, struct netlink_ext_ack *extack) { - struct mlx5_core_dev *dev = devlink_priv(devlink); - struct mlx5_eswitch *esw = dev->priv.eswitch; u16 cur_mlx5_mode, mlx5_mode = 0; - int err; + struct mlx5_eswitch *esw; + int err = 0; - err = mlx5_eswitch_check(dev); - if (err) - return err; + esw = mlx5_devlink_eswitch_get(devlink); + if (IS_ERR(esw)) + return PTR_ERR(esw); if (esw_mode_from_devlink(mode, &mlx5_mode)) return -EINVAL; mutex_lock(&esw->mode_lock); - err = eswitch_devlink_esw_mode_check(esw); - if (err) - goto unlock; - cur_mlx5_mode = esw->mode; - if (cur_mlx5_mode == mlx5_mode) goto unlock; @@ -2338,16 +2312,15 @@ unlock: int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode) { - struct mlx5_core_dev *dev = devlink_priv(devlink); - struct mlx5_eswitch *esw = dev->priv.eswitch; + struct mlx5_eswitch *esw; int err; - err = mlx5_eswitch_check(dev); - if (err) - return err; + esw = mlx5_devlink_eswitch_get(devlink); + if (IS_ERR(esw)) + return PTR_ERR(esw); mutex_lock(&esw->mode_lock); - err = eswitch_devlink_esw_mode_check(dev->priv.eswitch); + err = eswitch_devlink_esw_mode_check(esw); if (err) goto unlock; @@ -2361,13 +2334,13 @@ int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode, struct netlink_ext_ack *extack) { struct mlx5_core_dev *dev = devlink_priv(devlink); - struct mlx5_eswitch *esw = dev->priv.eswitch; int err, vport, num_vport; + struct mlx5_eswitch *esw; u8 mlx5_mode; - err = mlx5_eswitch_check(dev); - if (err) - return err; + esw = mlx5_devlink_eswitch_get(devlink); + if (IS_ERR(esw)) + return PTR_ERR(esw); mutex_lock(&esw->mode_lock); err = eswitch_devlink_esw_mode_check(esw); @@ -2378,7 +2351,7 @@ int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode, case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: if (mode == DEVLINK_ESWITCH_INLINE_MODE_NONE) goto out; - /* fall through */ + fallthrough; case MLX5_CAP_INLINE_MODE_L2: NL_SET_ERR_MSG_MOD(extack, "Inline mode can't be set"); err = -EOPNOTSUPP; @@ -2424,13 +2397,12 @@ out: int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode) { - struct mlx5_core_dev *dev = devlink_priv(devlink); - struct mlx5_eswitch *esw = dev->priv.eswitch; + struct mlx5_eswitch *esw; int err; - err = mlx5_eswitch_check(dev); - if (err) - return err; + esw = mlx5_devlink_eswitch_get(devlink); + if (IS_ERR(esw)) + return PTR_ERR(esw); mutex_lock(&esw->mode_lock); err = eswitch_devlink_esw_mode_check(esw); @@ -2448,12 +2420,12 @@ int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink, struct netlink_ext_ack *extack) { struct mlx5_core_dev *dev = devlink_priv(devlink); - struct mlx5_eswitch *esw = dev->priv.eswitch; + struct mlx5_eswitch *esw; int err; - err = mlx5_eswitch_check(dev); - if (err) - return err; + esw = mlx5_devlink_eswitch_get(devlink); + if (IS_ERR(esw)) + return PTR_ERR(esw); mutex_lock(&esw->mode_lock); err = eswitch_devlink_esw_mode_check(esw); @@ -2491,13 +2463,13 @@ int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink, esw->offloads.encap = encap; - err = esw_create_offloads_fdb_tables(esw, esw->nvports); + err = esw_create_offloads_fdb_tables(esw); if (err) { NL_SET_ERR_MSG_MOD(extack, "Failed re-creating fast FDB table"); esw->offloads.encap = !encap; - (void)esw_create_offloads_fdb_tables(esw, esw->nvports); + (void)esw_create_offloads_fdb_tables(esw); } unlock: @@ -2508,13 +2480,13 @@ unlock: int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink, enum devlink_eswitch_encap_mode *encap) { - struct mlx5_core_dev *dev = devlink_priv(devlink); - struct mlx5_eswitch *esw = dev->priv.eswitch; + struct mlx5_eswitch *esw; int err; - err = mlx5_eswitch_check(dev); - if (err) - return err; + esw = mlx5_devlink_eswitch_get(devlink); + if (IS_ERR(esw)) + return PTR_ERR(esw); + mutex_lock(&esw->mode_lock); err = eswitch_devlink_esw_mode_check(esw); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c b/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c index 182d3ac3e73f..831d2c39e153 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c @@ -339,14 +339,14 @@ static void mlx5_fpga_conn_handle_cqe(struct mlx5_fpga_conn *conn, switch (opcode) { case MLX5_CQE_REQ_ERR: status = ((struct mlx5_err_cqe *)cqe)->syndrome; - /* Fall through */ + fallthrough; case MLX5_CQE_REQ: mlx5_fpga_conn_sq_cqe(conn, cqe, status); break; case MLX5_CQE_RESP_ERR: status = ((struct mlx5_err_cqe *)cqe)->syndrome; - /* Fall through */ + fallthrough; case MLX5_CQE_RESP_SEND: mlx5_fpga_conn_rq_cqe(conn, cqe, status); break; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c index b463787d6ca1..cc67366495b0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c @@ -359,7 +359,7 @@ u32 mlx5_fpga_ipsec_device_caps(struct mlx5_core_dev *mdev) return ret; } -unsigned int mlx5_fpga_ipsec_counters_count(struct mlx5_core_dev *mdev) +static unsigned int mlx5_fpga_ipsec_counters_count(struct mlx5_core_dev *mdev) { struct mlx5_fpga_device *fdev = mdev->fpga; @@ -370,8 +370,8 @@ unsigned int mlx5_fpga_ipsec_counters_count(struct mlx5_core_dev *mdev) number_of_ipsec_counters); } -int mlx5_fpga_ipsec_counters_read(struct mlx5_core_dev *mdev, u64 *counters, - unsigned int counters_count) +static int mlx5_fpga_ipsec_counters_read(struct mlx5_core_dev *mdev, u64 *counters, + unsigned int counters_count) { struct mlx5_fpga_device *fdev = mdev->fpga; unsigned int i; @@ -665,12 +665,10 @@ static bool mlx5_is_fpga_egress_ipsec_rule(struct mlx5_core_dev *dev, return true; } -void *mlx5_fpga_ipsec_create_sa_ctx(struct mlx5_core_dev *mdev, - struct mlx5_accel_esp_xfrm *accel_xfrm, - const __be32 saddr[4], - const __be32 daddr[4], - const __be32 spi, bool is_ipv6, - u32 *sa_handle) +static void *mlx5_fpga_ipsec_create_sa_ctx(struct mlx5_core_dev *mdev, + struct mlx5_accel_esp_xfrm *accel_xfrm, + const __be32 saddr[4], const __be32 daddr[4], + const __be32 spi, bool is_ipv6, u32 *sa_handle) { struct mlx5_fpga_ipsec_sa_ctx *sa_ctx; struct mlx5_fpga_esp_xfrm *fpga_xfrm = @@ -862,7 +860,7 @@ mlx5_fpga_ipsec_release_sa_ctx(struct mlx5_fpga_ipsec_sa_ctx *sa_ctx) mutex_unlock(&fipsec->sa_hash_lock); } -void mlx5_fpga_ipsec_delete_sa_ctx(void *context) +static void mlx5_fpga_ipsec_delete_sa_ctx(void *context) { struct mlx5_fpga_esp_xfrm *fpga_xfrm = ((struct mlx5_fpga_ipsec_sa_ctx *)context)->fpga_xfrm; @@ -1264,7 +1262,7 @@ const struct mlx5_flow_cmds *mlx5_fs_cmd_get_default_ipsec_fpga_cmds(enum fs_flo } } -int mlx5_fpga_ipsec_init(struct mlx5_core_dev *mdev) +static int mlx5_fpga_ipsec_init(struct mlx5_core_dev *mdev) { struct mlx5_fpga_conn_attr init_attr = {0}; struct mlx5_fpga_device *fdev = mdev->fpga; @@ -1346,7 +1344,7 @@ static void destroy_rules_rb(struct rb_root *root) } } -void mlx5_fpga_ipsec_cleanup(struct mlx5_core_dev *mdev) +static void mlx5_fpga_ipsec_cleanup(struct mlx5_core_dev *mdev) { struct mlx5_fpga_device *fdev = mdev->fpga; @@ -1451,7 +1449,7 @@ mlx5_fpga_esp_validate_xfrm_attrs(struct mlx5_core_dev *mdev, return 0; } -struct mlx5_accel_esp_xfrm * +static struct mlx5_accel_esp_xfrm * mlx5_fpga_esp_create_xfrm(struct mlx5_core_dev *mdev, const struct mlx5_accel_esp_xfrm_attrs *attrs, u32 flags) @@ -1479,7 +1477,7 @@ mlx5_fpga_esp_create_xfrm(struct mlx5_core_dev *mdev, return &fpga_xfrm->accel_xfrm; } -void mlx5_fpga_esp_destroy_xfrm(struct mlx5_accel_esp_xfrm *xfrm) +static void mlx5_fpga_esp_destroy_xfrm(struct mlx5_accel_esp_xfrm *xfrm) { struct mlx5_fpga_esp_xfrm *fpga_xfrm = container_of(xfrm, struct mlx5_fpga_esp_xfrm, @@ -1488,8 +1486,8 @@ void mlx5_fpga_esp_destroy_xfrm(struct mlx5_accel_esp_xfrm *xfrm) kfree(fpga_xfrm); } -int mlx5_fpga_esp_modify_xfrm(struct mlx5_accel_esp_xfrm *xfrm, - const struct mlx5_accel_esp_xfrm_attrs *attrs) +static int mlx5_fpga_esp_modify_xfrm(struct mlx5_accel_esp_xfrm *xfrm, + const struct mlx5_accel_esp_xfrm_attrs *attrs) { struct mlx5_core_dev *mdev = xfrm->mdev; struct mlx5_fpga_device *fdev = mdev->fpga; @@ -1560,3 +1558,24 @@ change_sw_xfrm_attrs: mutex_unlock(&fpga_xfrm->lock); return err; } + +static const struct mlx5_accel_ipsec_ops fpga_ipsec_ops = { + .device_caps = mlx5_fpga_ipsec_device_caps, + .counters_count = mlx5_fpga_ipsec_counters_count, + .counters_read = mlx5_fpga_ipsec_counters_read, + .create_hw_context = mlx5_fpga_ipsec_create_sa_ctx, + .free_hw_context = mlx5_fpga_ipsec_delete_sa_ctx, + .init = mlx5_fpga_ipsec_init, + .cleanup = mlx5_fpga_ipsec_cleanup, + .esp_create_xfrm = mlx5_fpga_esp_create_xfrm, + .esp_modify_xfrm = mlx5_fpga_esp_modify_xfrm, + .esp_destroy_xfrm = mlx5_fpga_esp_destroy_xfrm, +}; + +const struct mlx5_accel_ipsec_ops *mlx5_fpga_ipsec_ops(struct mlx5_core_dev *mdev) +{ + if (!mlx5_fpga_is_ipsec_device(mdev)) + return NULL; + + return &fpga_ipsec_ops; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.h index 9ba637f0f0f2..db88eb4c49e3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.h @@ -38,44 +38,23 @@ #include "fs_cmd.h" #ifdef CONFIG_MLX5_FPGA_IPSEC +const struct mlx5_accel_ipsec_ops *mlx5_fpga_ipsec_ops(struct mlx5_core_dev *mdev); u32 mlx5_fpga_ipsec_device_caps(struct mlx5_core_dev *mdev); -unsigned int mlx5_fpga_ipsec_counters_count(struct mlx5_core_dev *mdev); -int mlx5_fpga_ipsec_counters_read(struct mlx5_core_dev *mdev, u64 *counters, - unsigned int counters_count); - -void *mlx5_fpga_ipsec_create_sa_ctx(struct mlx5_core_dev *mdev, - struct mlx5_accel_esp_xfrm *accel_xfrm, - const __be32 saddr[4], - const __be32 daddr[4], - const __be32 spi, bool is_ipv6, - u32 *sa_handle); -void mlx5_fpga_ipsec_delete_sa_ctx(void *context); - -int mlx5_fpga_ipsec_init(struct mlx5_core_dev *mdev); -void mlx5_fpga_ipsec_cleanup(struct mlx5_core_dev *mdev); -void mlx5_fpga_ipsec_build_fs_cmds(void); - -struct mlx5_accel_esp_xfrm * -mlx5_fpga_esp_create_xfrm(struct mlx5_core_dev *mdev, - const struct mlx5_accel_esp_xfrm_attrs *attrs, - u32 flags); -void mlx5_fpga_esp_destroy_xfrm(struct mlx5_accel_esp_xfrm *xfrm); -int mlx5_fpga_esp_modify_xfrm(struct mlx5_accel_esp_xfrm *xfrm, - const struct mlx5_accel_esp_xfrm_attrs *attrs); - const struct mlx5_flow_cmds * mlx5_fs_cmd_get_default_ipsec_fpga_cmds(enum fs_flow_table_type type); +void mlx5_fpga_ipsec_build_fs_cmds(void); #else -static inline u32 mlx5_fpga_ipsec_device_caps(struct mlx5_core_dev *mdev) -{ - return 0; -} - +static inline +const struct mlx5_accel_ipsec_ops *mlx5_fpga_ipsec_ops(struct mlx5_core_dev *mdev) +{ return NULL; } +static inline u32 mlx5_fpga_ipsec_device_caps(struct mlx5_core_dev *mdev) { return 0; } static inline const struct mlx5_flow_cmds * mlx5_fs_cmd_get_default_ipsec_fpga_cmds(enum fs_flow_table_type type) { return mlx5_fs_cmd_get_default(type); } +static inline void mlx5_fpga_ipsec_build_fs_cmds(void) {}; + #endif /* CONFIG_MLX5_FPGA_IPSEC */ #endif /* __MLX5_FPGA_IPSEC_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c index 465a1076a477..fee169732de7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c @@ -459,6 +459,8 @@ static int mlx5_cmd_set_fte(struct mlx5_core_dev *dev, MLX5_SET(flow_context, in_flow_context, modify_header_id, fte->action.modify_hdr->id); + MLX5_SET(flow_context, in_flow_context, ipsec_obj_id, fte->action.ipsec_obj_id); + vlan = MLX5_ADDR_OF(flow_context, in_flow_context, push_vlan); MLX5_SET(vlan, vlan, ethtype, fte->action.vlan[0].ethtype); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c index 13e2fb79c21a..9ccec5f8b92a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c @@ -41,7 +41,6 @@ #include "diag/fs_tracepoint.h" #include "accel/ipsec.h" #include "fpga/ipsec.h" -#include "eswitch.h" #define INIT_TREE_NODE_ARRAY_SIZE(...) (sizeof((struct init_tree_node[]){__VA_ARGS__}) /\ sizeof(struct init_tree_node)) @@ -106,8 +105,8 @@ #define ETHTOOL_PRIO_NUM_LEVELS 1 #define ETHTOOL_NUM_PRIOS 11 #define ETHTOOL_MIN_LEVEL (KERNEL_MIN_LEVEL + ETHTOOL_NUM_PRIOS) -/* Vlan, mac, ttc, inner ttc, aRFS */ -#define KERNEL_NIC_PRIO_NUM_LEVELS 5 +/* Vlan, mac, ttc, inner ttc, {aRFS/accel and esp/esp_err} */ +#define KERNEL_NIC_PRIO_NUM_LEVELS 6 #define KERNEL_NIC_NUM_PRIOS 1 /* One more level for tc */ #define KERNEL_MIN_LEVEL (KERNEL_NIC_PRIO_NUM_LEVELS + 1) @@ -797,7 +796,7 @@ static struct mlx5_flow_table *find_closest_ft_recursive(struct fs_node *root, return ft; } -/* If reverse if false then return the first flow table in next priority of +/* If reverse is false then return the first flow table in next priority of * prio in the tree, else return the last flow table in the previous priority * of prio in the tree. */ @@ -829,34 +828,16 @@ static struct mlx5_flow_table *find_prev_chained_ft(struct fs_prio *prio) return find_closest_ft(prio, true); } -static struct fs_prio *find_fwd_ns_prio(struct mlx5_flow_root_namespace *root, - struct mlx5_flow_namespace *ns) -{ - struct mlx5_flow_namespace *root_ns = &root->ns; - struct fs_prio *iter_prio; - struct fs_prio *prio; - - fs_get_obj(prio, ns->node.parent); - list_for_each_entry(iter_prio, &root_ns->node.children, node.list) { - if (iter_prio == prio && - !list_is_last(&prio->node.children, &iter_prio->node.list)) - return list_next_entry(iter_prio, node.list); - } - return NULL; -} - static struct mlx5_flow_table *find_next_fwd_ft(struct mlx5_flow_table *ft, struct mlx5_flow_act *flow_act) { - struct mlx5_flow_root_namespace *root = find_root(&ft->node); struct fs_prio *prio; + bool next_ns; - if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS) - prio = find_fwd_ns_prio(root, ft->ns); - else - fs_get_obj(prio, ft->node.parent); + next_ns = flow_act->action & MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS; + fs_get_obj(prio, next_ns ? ft->ns->node.parent : ft->node.parent); - return (prio) ? find_next_chained_ft(prio) : NULL; + return find_next_chained_ft(prio); } static int connect_fts_in_prio(struct mlx5_core_dev *dev, @@ -865,18 +846,15 @@ static int connect_fts_in_prio(struct mlx5_core_dev *dev, { struct mlx5_flow_root_namespace *root = find_root(&prio->node); struct mlx5_flow_table *iter; - int i = 0; int err; fs_for_each_ft(iter, prio) { - i++; err = root->cmds->modify_flow_table(root, iter, ft); if (err) { - mlx5_core_warn(dev, "Failed to modify flow table %d\n", - iter->id); + mlx5_core_err(dev, + "Failed to modify flow table id %d, type %d, err %d\n", + iter->id, iter->type, err); /* The driver is out of sync with the FW */ - if (i > 1) - WARN_ON(true); return err; } } @@ -1599,6 +1577,7 @@ static struct mlx5_flow_handle *add_rule_fg(struct mlx5_flow_group *fg, static bool counter_is_valid(u32 action) { return (action & (MLX5_FLOW_CONTEXT_ACTION_DROP | + MLX5_FLOW_CONTEXT_ACTION_ALLOW | MLX5_FLOW_CONTEXT_ACTION_FWD_DEST)); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h index 825b662f809b..afe7f0bffb93 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h @@ -39,6 +39,16 @@ #include <linux/llist.h> #include <steering/fs_dr.h> +#define FDB_TC_MAX_CHAIN 3 +#define FDB_FT_CHAIN (FDB_TC_MAX_CHAIN + 1) +#define FDB_TC_SLOW_PATH_CHAIN (FDB_FT_CHAIN + 1) + +/* The index of the last real chain (FT) + 1 as chain zero is valid as well */ +#define FDB_NUM_CHAINS (FDB_FT_CHAIN + 1) + +#define FDB_TC_MAX_PRIO 16 +#define FDB_TC_LEVELS_PER_PRIO 2 + struct mlx5_modify_hdr { enum mlx5_flow_namespace_type ns_type; union { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c index a5fbe7343508..02558ac2ace6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c @@ -35,6 +35,7 @@ #include <linux/module.h> #include "mlx5_core.h" #include "../../mlxfw/mlxfw.h" +#include "accel/tls.h" enum { MCQS_IDENTIFIER_BOOT_IMG = 0x1, @@ -236,7 +237,7 @@ int mlx5_query_hca_caps(struct mlx5_core_dev *dev) return err; } - if (MLX5_CAP_GEN(dev, tls_tx)) { + if (mlx5_accel_is_ktls_tx(dev) || mlx5_accel_is_ktls_rx(dev)) { err = mlx5_core_get_caps(dev, MLX5_CAP_TLS); if (err) return err; @@ -249,6 +250,12 @@ int mlx5_query_hca_caps(struct mlx5_core_dev *dev) return err; } + if (MLX5_CAP_GEN(dev, ipsec_offload)) { + err = mlx5_core_get_caps(dev, MLX5_CAP_IPSEC); + if (err) + return err; + } + return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c index 690b822c6152..97b5fcb1f406 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c @@ -226,13 +226,20 @@ void mlx5i_uninit_underlay_qp(struct mlx5e_priv *priv) int mlx5i_create_underlay_qp(struct mlx5e_priv *priv) { + unsigned char *dev_addr = priv->netdev->dev_addr; u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {}; u32 in[MLX5_ST_SZ_DW(create_qp_in)] = {}; struct mlx5i_priv *ipriv = priv->ppriv; void *addr_path; + int qpn = 0; int ret = 0; void *qpc; + if (MLX5_CAP_GEN(priv->mdev, mkey_by_name)) { + qpn = (dev_addr[1] << 16) + (dev_addr[2] << 8) + dev_addr[3]; + MLX5_SET(create_qp_in, in, input_qpn, qpn); + } + qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); MLX5_SET(qpc, qpc, st, MLX5_QP_ST_UD); MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); @@ -464,8 +471,7 @@ static const struct mlx5e_profile mlx5i_nic_profile = { .update_rx = mlx5i_update_nic_rx, .update_stats = NULL, /* mlx5i_update_stats */ .update_carrier = NULL, /* no HW update in IB link */ - .rx_handlers.handle_rx_cqe = mlx5i_handle_rx_cqe, - .rx_handlers.handle_rx_cqe_mpwqe = NULL, /* Not supported */ + .rx_handlers = &mlx5i_rx_handlers, .max_tc = MLX5I_MAX_NUM_TC, .rq_groups = MLX5E_NUM_RQ_GROUPS(REGULAR), .stats_grps = mlx5i_stats_grps, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h index 79071a15c4ca..b79dc1e28c41 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h @@ -42,6 +42,7 @@ extern const struct ethtool_ops mlx5i_ethtool_ops; extern const struct ethtool_ops mlx5i_pkey_ethtool_ops; +extern const struct mlx5e_rx_handlers mlx5i_rx_handlers; #define MLX5_IB_GRH_BYTES 40 #define MLX5_IPOIB_ENCAP_LEN 4 @@ -117,7 +118,6 @@ struct mlx5i_tx_wqe { void mlx5i_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb, struct mlx5_av *av, u32 dqpn, u32 dqkey, bool xmit_more); -void mlx5i_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe); void mlx5i_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats); #endif /* CONFIG_MLX5_CORE_IPOIB */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c index f70367018862..7163d9f6c4a6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c @@ -349,8 +349,7 @@ static const struct mlx5e_profile mlx5i_pkey_nic_profile = { .disable = NULL, .update_rx = mlx5i_update_nic_rx, .update_stats = NULL, - .rx_handlers.handle_rx_cqe = mlx5i_handle_rx_cqe, - .rx_handlers.handle_rx_cqe_mpwqe = NULL, /* Not supported */ + .rx_handlers = &mlx5i_rx_handlers, .max_tc = MLX5I_MAX_NUM_TC, .rq_groups = MLX5E_NUM_RQ_GROUPS(REGULAR), }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag_mp.c b/drivers/net/ethernet/mellanox/mlx5/core/lag_mp.c index e9089a793632..9e68f5926ab6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag_mp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag_mp.c @@ -198,13 +198,13 @@ static void mlx5_lag_fib_update(struct work_struct *work) /* Protect internal structures from changes */ rtnl_lock(); switch (fib_work->event) { - case FIB_EVENT_ENTRY_REPLACE: /* fall through */ + case FIB_EVENT_ENTRY_REPLACE: case FIB_EVENT_ENTRY_DEL: mlx5_lag_fib_route_event(ldev, fib_work->event, fib_work->fen_info.fi); fib_info_put(fib_work->fen_info.fi); break; - case FIB_EVENT_NH_ADD: /* fall through */ + case FIB_EVENT_NH_ADD: case FIB_EVENT_NH_DEL: fib_nh = fib_work->fnh_info.fib_nh; mlx5_lag_fib_nexthop_event(ldev, @@ -255,7 +255,7 @@ static int mlx5_lag_fib_event(struct notifier_block *nb, return NOTIFY_DONE; switch (event) { - case FIB_EVENT_ENTRY_REPLACE: /* fall through */ + case FIB_EVENT_ENTRY_REPLACE: case FIB_EVENT_ENTRY_DEL: fen_info = container_of(info, struct fib_entry_notifier_info, info); @@ -278,7 +278,7 @@ static int mlx5_lag_fib_event(struct notifier_block *nb, */ fib_info_hold(fib_work->fen_info.fi); break; - case FIB_EVENT_NH_ADD: /* fall through */ + case FIB_EVENT_NH_ADD: case FIB_EVENT_NH_DEL: fnh_info = container_of(info, struct fib_nh_notifier_info, info); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c index ef0706d15a5b..2d55b7c22c03 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c @@ -273,17 +273,17 @@ static int mlx5_extts_configure(struct ptp_clock_info *ptp, if (rq->extts.index >= clock->ptp_info.n_pins) return -EINVAL; + pin = ptp_find_pin(clock->ptp, PTP_PF_EXTTS, rq->extts.index); + if (pin < 0) + return -EBUSY; + if (on) { - pin = ptp_find_pin(clock->ptp, PTP_PF_EXTTS, rq->extts.index); - if (pin < 0) - return -EBUSY; pin_mode = MLX5_PIN_MODE_IN; pattern = !!(rq->extts.flags & PTP_FALLING_EDGE); field_select = MLX5_MTPPS_FS_PIN_MODE | MLX5_MTPPS_FS_PATTERN | MLX5_MTPPS_FS_ENABLE; } else { - pin = rq->extts.index; field_select = MLX5_MTPPS_FS_ENABLE; } @@ -331,12 +331,12 @@ static int mlx5_perout_configure(struct ptp_clock_info *ptp, if (rq->perout.index >= clock->ptp_info.n_pins) return -EINVAL; - if (on) { - pin = ptp_find_pin(clock->ptp, PTP_PF_PEROUT, - rq->perout.index); - if (pin < 0) - return -EBUSY; + pin = ptp_find_pin(clock->ptp, PTP_PF_PEROUT, + rq->perout.index); + if (pin < 0) + return -EBUSY; + if (on) { pin_mode = MLX5_PIN_MODE_OUT; pattern = MLX5_OUT_PATTERN_PERIODIC; ts.tv_sec = rq->perout.period.sec; @@ -362,7 +362,6 @@ static int mlx5_perout_configure(struct ptp_clock_info *ptp, MLX5_MTPPS_FS_ENABLE | MLX5_MTPPS_FS_TIME_STAMP; } else { - pin = rq->perout.index; field_select = MLX5_MTPPS_FS_ENABLE; } @@ -409,10 +408,31 @@ static int mlx5_ptp_enable(struct ptp_clock_info *ptp, return 0; } +enum { + MLX5_MTPPS_REG_CAP_PIN_X_MODE_SUPPORT_PPS_IN = BIT(0), + MLX5_MTPPS_REG_CAP_PIN_X_MODE_SUPPORT_PPS_OUT = BIT(1), +}; + static int mlx5_ptp_verify(struct ptp_clock_info *ptp, unsigned int pin, enum ptp_pin_function func, unsigned int chan) { - return (func == PTP_PF_PHYSYNC) ? -EOPNOTSUPP : 0; + struct mlx5_clock *clock = container_of(ptp, struct mlx5_clock, + ptp_info); + + switch (func) { + case PTP_PF_NONE: + return 0; + case PTP_PF_EXTTS: + return !(clock->pps_info.pin_caps[pin] & + MLX5_MTPPS_REG_CAP_PIN_X_MODE_SUPPORT_PPS_IN); + case PTP_PF_PEROUT: + return !(clock->pps_info.pin_caps[pin] & + MLX5_MTPPS_REG_CAP_PIN_X_MODE_SUPPORT_PPS_OUT); + default: + return -EOPNOTSUPP; + } + + return -EOPNOTSUPP; } static const struct ptp_clock_info mlx5_ptp_clock_info = { @@ -432,6 +452,38 @@ static const struct ptp_clock_info mlx5_ptp_clock_info = { .verify = NULL, }; +static int mlx5_query_mtpps_pin_mode(struct mlx5_core_dev *mdev, u8 pin, + u32 *mtpps, u32 mtpps_size) +{ + u32 in[MLX5_ST_SZ_DW(mtpps_reg)] = {}; + + MLX5_SET(mtpps_reg, in, pin, pin); + + return mlx5_core_access_reg(mdev, in, sizeof(in), mtpps, + mtpps_size, MLX5_REG_MTPPS, 0, 0); +} + +static int mlx5_get_pps_pin_mode(struct mlx5_clock *clock, u8 pin) +{ + struct mlx5_core_dev *mdev = clock->mdev; + u32 out[MLX5_ST_SZ_DW(mtpps_reg)] = {}; + u8 mode; + int err; + + err = mlx5_query_mtpps_pin_mode(mdev, pin, out, sizeof(out)); + if (err || !MLX5_GET(mtpps_reg, out, enable)) + return PTP_PF_NONE; + + mode = MLX5_GET(mtpps_reg, out, pin_mode); + + if (mode == MLX5_PIN_MODE_IN) + return PTP_PF_EXTTS; + else if (mode == MLX5_PIN_MODE_OUT) + return PTP_PF_PEROUT; + + return PTP_PF_NONE; +} + static int mlx5_init_pin_config(struct mlx5_clock *clock) { int i; @@ -451,8 +503,8 @@ static int mlx5_init_pin_config(struct mlx5_clock *clock) sizeof(clock->ptp_info.pin_config[i].name), "mlx5_pps%d", i); clock->ptp_info.pin_config[i].index = i; - clock->ptp_info.pin_config[i].func = PTP_PF_NONE; - clock->ptp_info.pin_config[i].chan = i; + clock->ptp_info.pin_config[i].func = mlx5_get_pps_pin_mode(clock, i); + clock->ptp_info.pin_config[i].chan = 0; } return 0; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c index dcea87ec5977..57eb91bcbca7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c @@ -6,7 +6,7 @@ int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, void *key, u32 sz_bytes, - u32 *p_key_id) + u32 key_type, u32 *p_key_id) { u32 in[MLX5_ST_SZ_DW(create_encryption_key_in)] = {}; u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; @@ -41,8 +41,7 @@ int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, memcpy(key_p, key, sz_bytes); MLX5_SET(encryption_key_obj, obj, key_size, general_obj_key_size); - MLX5_SET(encryption_key_obj, obj, key_type, - MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_TLS); + MLX5_SET(encryption_key_obj, obj, key_type, key_type); MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h index 249539247e2e..d046db7bb047 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h @@ -80,8 +80,14 @@ void mlx5_get_pme_stats(struct mlx5_core_dev *dev, struct mlx5_pme_stats *stats) int mlx5_notifier_call_chain(struct mlx5_events *events, unsigned int event, void *data); /* Crypto */ +enum { + MLX5_ACCEL_OBJ_TLS_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_TLS, + MLX5_ACCEL_OBJ_IPSEC_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_IPSEC, +}; + int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, - void *key, u32 sz_bytes, u32 *p_key_id); + void *key, u32 sz_bytes, + u32 key_type, u32 *p_key_id); void mlx5_destroy_encryption_key(struct mlx5_core_dev *mdev, u32 key_id); static inline struct net *mlx5_core_net(struct mlx5_core_dev *dev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/vxlan.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/vxlan.c index 82c766a95165..3315afe2f8dc 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/vxlan.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/vxlan.c @@ -40,24 +40,16 @@ struct mlx5_vxlan { struct mlx5_core_dev *mdev; - spinlock_t lock; /* protect vxlan table */ /* max_num_ports is usuallly 4, 16 buckets is more than enough */ DECLARE_HASHTABLE(htable, 4); - int num_ports; struct mutex sync_lock; /* sync add/del port HW operations */ }; struct mlx5_vxlan_port { struct hlist_node hlist; - refcount_t refcount; u16 udp_port; }; -static inline u8 mlx5_vxlan_max_udp_ports(struct mlx5_core_dev *mdev) -{ - return MLX5_CAP_ETH(mdev, max_vxlan_udp_ports) ?: 4; -} - static int mlx5_vxlan_core_add_port_cmd(struct mlx5_core_dev *mdev, u16 port) { u32 in[MLX5_ST_SZ_DW(add_vxlan_udp_dport_in)] = {}; @@ -78,113 +70,78 @@ static int mlx5_vxlan_core_del_port_cmd(struct mlx5_core_dev *mdev, u16 port) return mlx5_cmd_exec_in(mdev, delete_vxlan_udp_dport, in); } -static struct mlx5_vxlan_port* -mlx5_vxlan_lookup_port_locked(struct mlx5_vxlan *vxlan, u16 port) +bool mlx5_vxlan_lookup_port(struct mlx5_vxlan *vxlan, u16 port) { struct mlx5_vxlan_port *vxlanp; + bool found = false; - hash_for_each_possible(vxlan->htable, vxlanp, hlist, port) { - if (vxlanp->udp_port == port) - return vxlanp; - } + if (!mlx5_vxlan_allowed(vxlan)) + return NULL; - return NULL; + rcu_read_lock(); + hash_for_each_possible_rcu(vxlan->htable, vxlanp, hlist, port) + if (vxlanp->udp_port == port) { + found = true; + break; + } + rcu_read_unlock(); + + return found; } -struct mlx5_vxlan_port *mlx5_vxlan_lookup_port(struct mlx5_vxlan *vxlan, u16 port) +static struct mlx5_vxlan_port *vxlan_lookup_port(struct mlx5_vxlan *vxlan, u16 port) { struct mlx5_vxlan_port *vxlanp; - if (!mlx5_vxlan_allowed(vxlan)) - return NULL; - - spin_lock_bh(&vxlan->lock); - vxlanp = mlx5_vxlan_lookup_port_locked(vxlan, port); - spin_unlock_bh(&vxlan->lock); - - return vxlanp; + hash_for_each_possible(vxlan->htable, vxlanp, hlist, port) + if (vxlanp->udp_port == port) + return vxlanp; + return NULL; } int mlx5_vxlan_add_port(struct mlx5_vxlan *vxlan, u16 port) { struct mlx5_vxlan_port *vxlanp; - int ret = -ENOSPC; - - vxlanp = mlx5_vxlan_lookup_port(vxlan, port); - if (vxlanp) { - refcount_inc(&vxlanp->refcount); - return 0; - } - - mutex_lock(&vxlan->sync_lock); - if (vxlan->num_ports >= mlx5_vxlan_max_udp_ports(vxlan->mdev)) { - mlx5_core_info(vxlan->mdev, - "UDP port (%d) not offloaded, max number of UDP ports (%d) are already offloaded\n", - port, mlx5_vxlan_max_udp_ports(vxlan->mdev)); - ret = -ENOSPC; - goto unlock; - } - - ret = mlx5_vxlan_core_add_port_cmd(vxlan->mdev, port); - if (ret) - goto unlock; + int ret; vxlanp = kzalloc(sizeof(*vxlanp), GFP_KERNEL); - if (!vxlanp) { - ret = -ENOMEM; - goto err_delete_port; - } - + if (!vxlanp) + return -ENOMEM; vxlanp->udp_port = port; - refcount_set(&vxlanp->refcount, 1); - spin_lock_bh(&vxlan->lock); - hash_add(vxlan->htable, &vxlanp->hlist, port); - spin_unlock_bh(&vxlan->lock); + ret = mlx5_vxlan_core_add_port_cmd(vxlan->mdev, port); + if (ret) { + kfree(vxlanp); + return ret; + } - vxlan->num_ports++; + mutex_lock(&vxlan->sync_lock); + hash_add_rcu(vxlan->htable, &vxlanp->hlist, port); mutex_unlock(&vxlan->sync_lock); - return 0; - -err_delete_port: - mlx5_vxlan_core_del_port_cmd(vxlan->mdev, port); -unlock: - mutex_unlock(&vxlan->sync_lock); - return ret; + return 0; } int mlx5_vxlan_del_port(struct mlx5_vxlan *vxlan, u16 port) { struct mlx5_vxlan_port *vxlanp; - bool remove = false; int ret = 0; mutex_lock(&vxlan->sync_lock); - spin_lock_bh(&vxlan->lock); - vxlanp = mlx5_vxlan_lookup_port_locked(vxlan, port); - if (!vxlanp) { + vxlanp = vxlan_lookup_port(vxlan, port); + if (WARN_ON(!vxlanp)) { ret = -ENOENT; goto out_unlock; } - if (refcount_dec_and_test(&vxlanp->refcount)) { - hash_del(&vxlanp->hlist); - remove = true; - } + hash_del_rcu(&vxlanp->hlist); + synchronize_rcu(); + mlx5_vxlan_core_del_port_cmd(vxlan->mdev, port); + kfree(vxlanp); out_unlock: - spin_unlock_bh(&vxlan->lock); - - if (remove) { - mlx5_vxlan_core_del_port_cmd(vxlan->mdev, port); - kfree(vxlanp); - vxlan->num_ports--; - } - mutex_unlock(&vxlan->sync_lock); - return ret; } @@ -201,7 +158,6 @@ struct mlx5_vxlan *mlx5_vxlan_create(struct mlx5_core_dev *mdev) vxlan->mdev = mdev; mutex_init(&vxlan->sync_lock); - spin_lock_init(&vxlan->lock); hash_init(vxlan->htable); /* Hardware adds 4789 (IANA_VXLAN_UDP_PORT) by default */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/vxlan.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/vxlan.h index 8fb0eb08fa6d..ec766529f49b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/vxlan.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/vxlan.h @@ -37,6 +37,11 @@ struct mlx5_vxlan; struct mlx5_vxlan_port; +static inline u8 mlx5_vxlan_max_udp_ports(struct mlx5_core_dev *mdev) +{ + return MLX5_CAP_ETH(mdev, max_vxlan_udp_ports) ?: 4; +} + static inline bool mlx5_vxlan_allowed(struct mlx5_vxlan *vxlan) { /* not allowed reason is encoded in vxlan pointer as error, @@ -50,15 +55,14 @@ struct mlx5_vxlan *mlx5_vxlan_create(struct mlx5_core_dev *mdev); void mlx5_vxlan_destroy(struct mlx5_vxlan *vxlan); int mlx5_vxlan_add_port(struct mlx5_vxlan *vxlan, u16 port); int mlx5_vxlan_del_port(struct mlx5_vxlan *vxlan, u16 port); -struct mlx5_vxlan_port *mlx5_vxlan_lookup_port(struct mlx5_vxlan *vxlan, u16 port); +bool mlx5_vxlan_lookup_port(struct mlx5_vxlan *vxlan, u16 port); #else static inline struct mlx5_vxlan* mlx5_vxlan_create(struct mlx5_core_dev *mdev) { return ERR_PTR(-EOPNOTSUPP); } static inline void mlx5_vxlan_destroy(struct mlx5_vxlan *vxlan) { return; } static inline int mlx5_vxlan_add_port(struct mlx5_vxlan *vxlan, u16 port) { return -EOPNOTSUPP; } static inline int mlx5_vxlan_del_port(struct mlx5_vxlan *vxlan, u16 port) { return -EOPNOTSUPP; } -static inline struct mx5_vxlan_port* -mlx5_vxlan_lookup_port(struct mlx5_vxlan *vxlan, u16 port) { return NULL; } +static inline bool mlx5_vxlan_lookup_port(struct mlx5_vxlan *vxlan, u16 port) { return false; } #endif #endif /* __MLX5_VXLAN_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 8b658908f044..ce43e3feccd9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -557,6 +557,9 @@ static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx) if (MLX5_CAP_GEN_MAX(dev, release_all_pages)) MLX5_SET(cmd_hca_cap, set_hca_cap, release_all_pages, 1); + if (MLX5_CAP_GEN_MAX(dev, mkey_by_name)) + MLX5_SET(cmd_hca_cap, set_hca_cap, mkey_by_name, 1); + return set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE); } @@ -1089,11 +1092,7 @@ static int mlx5_load(struct mlx5_core_dev *dev) goto err_fpga_start; } - err = mlx5_accel_ipsec_init(dev); - if (err) { - mlx5_core_err(dev, "IPSec device start failed %d\n", err); - goto err_ipsec_start; - } + mlx5_accel_ipsec_init(dev); err = mlx5_accel_tls_init(dev); if (err) { @@ -1135,7 +1134,6 @@ err_fs: mlx5_accel_tls_cleanup(dev); err_tls_start: mlx5_accel_ipsec_cleanup(dev); -err_ipsec_start: mlx5_fpga_device_stop(dev); err_fpga_start: mlx5_rsc_dump_cleanup(dev); @@ -1628,7 +1626,7 @@ static int __init init(void) get_random_bytes(&sw_owner_id, sizeof(sw_owner_id)); mlx5_core_verify_params(); - mlx5_accel_ipsec_build_fs_cmds(); + mlx5_fpga_ipsec_build_fs_cmds(); mlx5_register_debugfs(); err = pci_register_driver(&mlx5_core_driver); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c index 5ddd18639a1e..f9b798af6335 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c @@ -35,6 +35,7 @@ #include <linux/module.h> #include <linux/delay.h> #include <linux/mlx5/driver.h> +#include <linux/xarray.h> #include "mlx5_core.h" #include "lib/eq.h" @@ -73,15 +74,45 @@ enum { MLX5_NUM_4K_IN_PAGE = PAGE_SIZE / MLX5_ADAPTER_PAGE_SIZE, }; +static struct rb_root *page_root_per_func_id(struct mlx5_core_dev *dev, u16 func_id) +{ + struct rb_root *root; + int err; + + root = xa_load(&dev->priv.page_root_xa, func_id); + if (root) + return root; + + root = kzalloc(sizeof(*root), GFP_KERNEL); + if (!root) + return ERR_PTR(-ENOMEM); + + err = xa_insert(&dev->priv.page_root_xa, func_id, root, GFP_KERNEL); + if (err) { + kfree(root); + return ERR_PTR(err); + } + + *root = RB_ROOT; + + return root; +} + static int insert_page(struct mlx5_core_dev *dev, u64 addr, struct page *page, u16 func_id) { - struct rb_root *root = &dev->priv.page_root; - struct rb_node **new = &root->rb_node; struct rb_node *parent = NULL; + struct rb_root *root; + struct rb_node **new; struct fw_page *nfp; struct fw_page *tfp; int i; + root = page_root_per_func_id(dev, func_id); + if (IS_ERR(root)) + return PTR_ERR(root); + + new = &root->rb_node; + while (*new) { parent = *new; tfp = rb_entry(parent, struct fw_page, rb_node); @@ -111,13 +142,20 @@ static int insert_page(struct mlx5_core_dev *dev, u64 addr, struct page *page, u return 0; } -static struct fw_page *find_fw_page(struct mlx5_core_dev *dev, u64 addr) +static struct fw_page *find_fw_page(struct mlx5_core_dev *dev, u64 addr, + u32 func_id) { - struct rb_root *root = &dev->priv.page_root; - struct rb_node *tmp = root->rb_node; struct fw_page *result = NULL; + struct rb_root *root; + struct rb_node *tmp; struct fw_page *tfp; + root = xa_load(&dev->priv.page_root_xa, func_id); + if (WARN_ON_ONCE(!root)) + return NULL; + + tmp = root->rb_node; + while (tmp) { tfp = rb_entry(tmp, struct fw_page, rb_node); if (tfp->addr < addr) { @@ -191,7 +229,13 @@ static int alloc_4k(struct mlx5_core_dev *dev, u64 *addr, u16 func_id) static void free_fwp(struct mlx5_core_dev *dev, struct fw_page *fwp, bool in_free_list) { - rb_erase(&fwp->rb_node, &dev->priv.page_root); + struct rb_root *root; + + root = xa_load(&dev->priv.page_root_xa, fwp->func_id); + if (WARN_ON_ONCE(!root)) + return; + + rb_erase(&fwp->rb_node, root); if (in_free_list) list_del(&fwp->list); dma_unmap_page(dev->device, fwp->addr & MLX5_U64_4K_PAGE_MASK, @@ -200,12 +244,12 @@ static void free_fwp(struct mlx5_core_dev *dev, struct fw_page *fwp, kfree(fwp); } -static void free_4k(struct mlx5_core_dev *dev, u64 addr) +static void free_4k(struct mlx5_core_dev *dev, u64 addr, u32 func_id) { struct fw_page *fwp; int n; - fwp = find_fw_page(dev, addr & MLX5_U64_4K_PAGE_MASK); + fwp = find_fw_page(dev, addr & MLX5_U64_4K_PAGE_MASK, func_id); if (!fwp) { mlx5_core_warn_rl(dev, "page not found\n"); return; @@ -340,7 +384,7 @@ retry: out_4k: for (i--; i >= 0; i--) - free_4k(dev, MLX5_GET64(manage_pages_in, in, pas[i])); + free_4k(dev, MLX5_GET64(manage_pages_in, in, pas[i]), func_id); out_free: kvfree(in); if (notify_fail) @@ -351,16 +395,19 @@ out_free: static void release_all_pages(struct mlx5_core_dev *dev, u32 func_id, bool ec_function) { + struct rb_root *root; struct rb_node *p; int npages = 0; - p = rb_first(&dev->priv.page_root); + root = xa_load(&dev->priv.page_root_xa, func_id); + if (WARN_ON_ONCE(!root)) + return; + + p = rb_first(root); while (p) { struct fw_page *fwp = rb_entry(p, struct fw_page, rb_node); p = rb_next(p); - if (fwp->func_id != func_id) - continue; npages += (MLX5_NUM_4K_IN_PAGE - fwp->free_count); free_fwp(dev, fwp, fwp->free_count); } @@ -378,6 +425,7 @@ static void release_all_pages(struct mlx5_core_dev *dev, u32 func_id, static int reclaim_pages_cmd(struct mlx5_core_dev *dev, u32 *in, int in_size, u32 *out, int out_size) { + struct rb_root *root; struct fw_page *fwp; struct rb_node *p; u32 func_id; @@ -391,12 +439,14 @@ static int reclaim_pages_cmd(struct mlx5_core_dev *dev, npages = MLX5_GET(manage_pages_in, in, input_num_entries); func_id = MLX5_GET(manage_pages_in, in, function_id); - p = rb_first(&dev->priv.page_root); + root = xa_load(&dev->priv.page_root_xa, func_id); + if (WARN_ON_ONCE(!root)) + return -EEXIST; + + p = rb_first(root); while (p && i < npages) { fwp = rb_entry(p, struct fw_page, rb_node); p = rb_next(p); - if (fwp->func_id != func_id) - continue; MLX5_ARRAY_SET64(manage_pages_out, out, pas, i, fwp->addr); i++; @@ -430,7 +480,8 @@ static int reclaim_pages(struct mlx5_core_dev *dev, u32 func_id, int npages, MLX5_SET(manage_pages_in, in, input_num_entries, npages); MLX5_SET(manage_pages_in, in, embedded_cpu_function, ec_function); - mlx5_core_dbg(dev, "npages %d, outlen %d\n", npages, outlen); + mlx5_core_dbg(dev, "func 0x%x, npages %d, outlen %d\n", + func_id, npages, outlen); err = reclaim_pages_cmd(dev, in, sizeof(in), out, outlen); if (err) { mlx5_core_err(dev, "failed reclaiming pages: err %d\n", err); @@ -446,7 +497,7 @@ static int reclaim_pages(struct mlx5_core_dev *dev, u32 func_id, int npages, } for (i = 0; i < num_claimed; i++) - free_4k(dev, MLX5_GET64(manage_pages_out, out, pas[i])); + free_4k(dev, MLX5_GET64(manage_pages_out, out, pas[i]), func_id); if (nclaimed) *nclaimed = num_claimed; @@ -529,8 +580,8 @@ static int req_pages_handler(struct notifier_block *nb, int mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot) { - u16 uninitialized_var(func_id); - s32 uninitialized_var(npages); + u16 func_id; + s32 npages; int err; err = mlx5_cmd_query_pages(dev, &func_id, &npages, boot); @@ -560,35 +611,49 @@ static int optimal_reclaimed_pages(void) return ret; } -int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev) +static int mlx5_reclaim_root_pages(struct mlx5_core_dev *dev, + struct rb_root *root, u16 func_id) { unsigned long end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); - struct fw_page *fwp; - struct rb_node *p; - int nclaimed = 0; - int err = 0; - do { - p = rb_first(&dev->priv.page_root); - if (p) { - fwp = rb_entry(p, struct fw_page, rb_node); - err = reclaim_pages(dev, fwp->func_id, - optimal_reclaimed_pages(), - &nclaimed, mlx5_core_is_ecpf(dev)); - - if (err) { - mlx5_core_warn(dev, "failed reclaiming pages (%d)\n", - err); - return err; - } - if (nclaimed) - end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); + while (!RB_EMPTY_ROOT(root)) { + int nclaimed; + int err; + + err = reclaim_pages(dev, func_id, optimal_reclaimed_pages(), + &nclaimed, mlx5_core_is_ecpf(dev)); + if (err) { + mlx5_core_warn(dev, "failed reclaiming pages (%d) for func id 0x%x\n", + err, func_id); + return err; } + + if (nclaimed) + end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); + if (time_after(jiffies, end)) { mlx5_core_warn(dev, "FW did not return all pages. giving up...\n"); break; } - } while (p); + } + + return 0; +} + +int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev) +{ + struct rb_root *root; + unsigned long id; + void *entry; + + xa_for_each(&dev->priv.page_root_xa, id, entry) { + root = entry; + mlx5_reclaim_root_pages(dev, root, id); + xa_erase(&dev->priv.page_root_xa, id); + kfree(root); + } + + WARN_ON(!xa_empty(&dev->priv.page_root_xa)); WARN(dev->priv.fw_pages, "FW pages counter is %d after reclaiming all pages\n", @@ -605,17 +670,19 @@ int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev) int mlx5_pagealloc_init(struct mlx5_core_dev *dev) { - dev->priv.page_root = RB_ROOT; INIT_LIST_HEAD(&dev->priv.free_list); dev->priv.pg_wq = create_singlethread_workqueue("mlx5_page_allocator"); if (!dev->priv.pg_wq) return -ENOMEM; + xa_init(&dev->priv.page_root_xa); + return 0; } void mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev) { + xa_destroy(&dev->priv.page_root_xa); destroy_workqueue(dev->priv.pg_wq); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c index 31abcbb95ca2..c63f727273d8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c @@ -395,7 +395,7 @@ static int dr_matcher_set_ste_builders(struct mlx5dr_matcher *matcher, /* Check that all mask fields were consumed */ for (i = 0; i < sizeof(struct mlx5dr_match_param); i++) { if (((u8 *)&mask)[i] != 0) { - mlx5dr_err(dmn, "Mask contains unsupported parameters\n"); + mlx5dr_dbg(dmn, "Mask contains unsupported parameters\n"); return -EOPNOTSUPP; } } @@ -474,14 +474,13 @@ static int dr_matcher_add_to_tbl(struct mlx5dr_matcher *matcher) int ret; next_matcher = NULL; - if (!list_empty(&tbl->matcher_list)) - list_for_each_entry(tmp_matcher, &tbl->matcher_list, matcher_list) { - if (tmp_matcher->prio >= matcher->prio) { - next_matcher = tmp_matcher; - break; - } - first = false; + list_for_each_entry(tmp_matcher, &tbl->matcher_list, matcher_list) { + if (tmp_matcher->prio >= matcher->prio) { + next_matcher = tmp_matcher; + break; } + first = false; + } prev_matcher = NULL; if (next_matcher && !first) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_rule.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_rule.c index cd708dcc2e3a..6ec5106bc472 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_rule.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_rule.c @@ -574,9 +574,8 @@ void mlx5dr_rule_update_rule_member(struct mlx5dr_ste *ste, { struct mlx5dr_rule_member *rule_mem; - if (!list_empty(&ste->rule_list)) - list_for_each_entry(rule_mem, &ste->rule_list, use_ste_list) - rule_mem->ste = new_ste; + list_for_each_entry(rule_mem, &ste->rule_list, use_ste_list) + rule_mem->ste = new_ste; } static void dr_rule_clean_rule_members(struct mlx5dr_rule *rule, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c index 8887b2440c7d..9b08eb557a31 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c @@ -279,29 +279,9 @@ static int mlx5_cmd_dr_create_fte(struct mlx5_flow_root_namespace *ns, /* The order of the actions are must to be keep, only the following * order is supported by SW steering: - * TX: push vlan -> modify header -> encap + * TX: modify header -> push vlan -> encap * RX: decap -> pop vlan -> modify header */ - if (fte->action.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) { - tmp_action = create_action_push_vlan(domain, &fte->action.vlan[0]); - if (!tmp_action) { - err = -ENOMEM; - goto free_actions; - } - fs_dr_actions[fs_dr_num_actions++] = tmp_action; - actions[num_actions++] = tmp_action; - } - - if (fte->action.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) { - tmp_action = create_action_push_vlan(domain, &fte->action.vlan[1]); - if (!tmp_action) { - err = -ENOMEM; - goto free_actions; - } - fs_dr_actions[fs_dr_num_actions++] = tmp_action; - actions[num_actions++] = tmp_action; - } - if (fte->action.action & MLX5_FLOW_CONTEXT_ACTION_DECAP) { enum mlx5dr_action_reformat_type decap_type = DR_ACTION_REFORMAT_TYP_TNL_L2_TO_L2; @@ -354,6 +334,26 @@ static int mlx5_cmd_dr_create_fte(struct mlx5_flow_root_namespace *ns, actions[num_actions++] = fte->action.modify_hdr->action.dr_action; + if (fte->action.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) { + tmp_action = create_action_push_vlan(domain, &fte->action.vlan[0]); + if (!tmp_action) { + err = -ENOMEM; + goto free_actions; + } + fs_dr_actions[fs_dr_num_actions++] = tmp_action; + actions[num_actions++] = tmp_action; + } + + if (fte->action.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) { + tmp_action = create_action_push_vlan(domain, &fte->action.vlan[1]); + if (!tmp_action) { + err = -ENOMEM; + goto free_actions; + } + fs_dr_actions[fs_dr_num_actions++] = tmp_action; + actions[num_actions++] = tmp_action; + } + if (delay_encap_set) actions[num_actions++] = fte->action.pkt_reformat->action.dr_action; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/vport.c b/drivers/net/ethernet/mellanox/mlx5/core/vport.c index c107d92dc118..bdafc85fd874 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/vport.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/vport.c @@ -110,7 +110,7 @@ void mlx5_query_min_inline(struct mlx5_core_dev *mdev, case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: if (!mlx5_query_nic_vport_min_inline(mdev, 0, min_inline_mode)) break; - /* fall through */ + fallthrough; case MLX5_CAP_INLINE_MODE_L2: *min_inline_mode = MLX5_INLINE_MODE_L2; break; @@ -173,7 +173,7 @@ int mlx5_query_mac_address(struct mlx5_core_dev *mdev, u8 *addr) EXPORT_SYMBOL_GPL(mlx5_query_mac_address); int mlx5_modify_nic_vport_mac_address(struct mlx5_core_dev *mdev, - u16 vport, u8 *addr) + u16 vport, const u8 *addr) { void *in; int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/wq.h b/drivers/net/ethernet/mellanox/mlx5/core/wq.h index 4cadc336593f..e5c4dcd1425e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/wq.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/wq.h @@ -172,6 +172,11 @@ static inline int mlx5_wq_cyc_cc_bigger(u16 cc1, u16 cc2) return !equal && !smaller; } +static inline u16 mlx5_wq_cyc_get_counter(struct mlx5_wq_cyc *wq) +{ + return wq->wqe_ctr; +} + static inline u32 mlx5_cqwq_get_size(struct mlx5_cqwq *wq) { return wq->fbc.sz_m1 + 1; @@ -290,4 +295,14 @@ static inline void mlx5_wq_ll_update_db_record(struct mlx5_wq_ll *wq) *wq->db = cpu_to_be32(wq->wqe_ctr); } +static inline u16 mlx5_wq_ll_get_head(struct mlx5_wq_ll *wq) +{ + return wq->head; +} + +static inline u16 mlx5_wq_ll_get_counter(struct mlx5_wq_ll *wq) +{ + return wq->wqe_ctr; +} + #endif /* __MLX5_WQ_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlxsw/Makefile b/drivers/net/ethernet/mellanox/mlxsw/Makefile index 4aeabb35c943..892724380ea2 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/Makefile +++ b/drivers/net/ethernet/mellanox/mlxsw/Makefile @@ -30,7 +30,8 @@ mlxsw_spectrum-objs := spectrum.o spectrum_buffers.o \ spectrum_mr_tcam.o spectrum_mr.o \ spectrum_qdisc.o spectrum_span.o \ spectrum_nve.o spectrum_nve_vxlan.o \ - spectrum_dpipe.o spectrum_trap.o + spectrum_dpipe.o spectrum_trap.o \ + spectrum_ethtool.o spectrum_policer.o mlxsw_spectrum-$(CONFIG_MLXSW_SPECTRUM_DCB) += spectrum_dcb.o mlxsw_spectrum-$(CONFIG_PTP_1588_CLOCK) += spectrum_ptp.o obj-$(CONFIG_MLXSW_MINIMAL) += mlxsw_minimal.o diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.c b/drivers/net/ethernet/mellanox/mlxsw/core.c index d6d6fe64887b..08d101138fbe 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core.c @@ -1177,14 +1177,15 @@ static void mlxsw_devlink_trap_fini(struct devlink *devlink, static int mlxsw_devlink_trap_action_set(struct devlink *devlink, const struct devlink_trap *trap, - enum devlink_trap_action action) + enum devlink_trap_action action, + struct netlink_ext_ack *extack) { struct mlxsw_core *mlxsw_core = devlink_priv(devlink); struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver; if (!mlxsw_driver->trap_action_set) return -EOPNOTSUPP; - return mlxsw_driver->trap_action_set(mlxsw_core, trap, action); + return mlxsw_driver->trap_action_set(mlxsw_core, trap, action, extack); } static int @@ -1202,14 +1203,15 @@ mlxsw_devlink_trap_group_init(struct devlink *devlink, static int mlxsw_devlink_trap_group_set(struct devlink *devlink, const struct devlink_trap_group *group, - const struct devlink_trap_policer *policer) + const struct devlink_trap_policer *policer, + struct netlink_ext_ack *extack) { struct mlxsw_core *mlxsw_core = devlink_priv(devlink); struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver; if (!mlxsw_driver->trap_group_set) return -EOPNOTSUPP; - return mlxsw_driver->trap_group_set(mlxsw_core, group, policer); + return mlxsw_driver->trap_group_set(mlxsw_core, group, policer, extack); } static int @@ -1525,7 +1527,8 @@ static bool __is_rx_listener_equal(const struct mlxsw_rx_listener *rxl_a, { return (rxl_a->func == rxl_b->func && rxl_a->local_port == rxl_b->local_port && - rxl_a->trap_id == rxl_b->trap_id); + rxl_a->trap_id == rxl_b->trap_id && + rxl_a->mirror_reason == rxl_b->mirror_reason); } static struct mlxsw_rx_listener_item * @@ -1814,7 +1817,7 @@ static int mlxsw_core_reg_access_emad(struct mlxsw_core *mlxsw_core, err = mlxsw_emad_reg_access(mlxsw_core, reg, payload, type, trans, bulk_list, cb, cb_priv, tid); if (err) { - kfree(trans); + kfree_rcu(trans, rcu); return err; } return 0; @@ -2045,17 +2048,20 @@ void mlxsw_core_skb_receive(struct mlxsw_core *mlxsw_core, struct sk_buff *skb, rxl = &rxl_item->rxl; if ((rxl->local_port == MLXSW_PORT_DONT_CARE || rxl->local_port == local_port) && - rxl->trap_id == rx_info->trap_id) { + rxl->trap_id == rx_info->trap_id && + rxl->mirror_reason == rx_info->mirror_reason) { if (rxl_item->enabled) found = true; break; } } - rcu_read_unlock(); - if (!found) + if (!found) { + rcu_read_unlock(); goto drop; + } rxl->func(skb, local_port, rxl_item->priv); + rcu_read_unlock(); return; drop: @@ -2123,6 +2129,7 @@ static int __mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u8 local_port, enum devlink_port_flavour flavour, u32 port_number, bool split, u32 split_port_subnumber, + bool splittable, u32 lanes, const unsigned char *switch_id, unsigned char switch_id_len) { @@ -2130,12 +2137,19 @@ static int __mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u8 local_port, struct mlxsw_core_port *mlxsw_core_port = &mlxsw_core->ports[local_port]; struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port; + struct devlink_port_attrs attrs = {}; int err; + attrs.split = split; + attrs.lanes = lanes; + attrs.splittable = splittable; + attrs.flavour = flavour; + attrs.phys.port_number = port_number; + attrs.phys.split_subport_number = split_port_subnumber; + memcpy(attrs.switch_id.id, switch_id, switch_id_len); + attrs.switch_id.id_len = switch_id_len; mlxsw_core_port->local_port = local_port; - devlink_port_attrs_set(devlink_port, flavour, port_number, - split, split_port_subnumber, - switch_id, switch_id_len); + devlink_port_attrs_set(devlink_port, &attrs); err = devlink_port_register(devlink, devlink_port, local_port); if (err) memset(mlxsw_core_port, 0, sizeof(*mlxsw_core_port)); @@ -2155,12 +2169,14 @@ static void __mlxsw_core_port_fini(struct mlxsw_core *mlxsw_core, u8 local_port) int mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u8 local_port, u32 port_number, bool split, u32 split_port_subnumber, + bool splittable, u32 lanes, const unsigned char *switch_id, unsigned char switch_id_len) { return __mlxsw_core_port_init(mlxsw_core, local_port, DEVLINK_PORT_FLAVOUR_PHYSICAL, port_number, split, split_port_subnumber, + splittable, lanes, switch_id, switch_id_len); } EXPORT_SYMBOL(mlxsw_core_port_init); @@ -2182,7 +2198,7 @@ int mlxsw_core_cpu_port_init(struct mlxsw_core *mlxsw_core, err = __mlxsw_core_port_init(mlxsw_core, MLXSW_PORT_CPU_PORT, DEVLINK_PORT_FLAVOUR_CPU, - 0, false, 0, + 0, false, 0, false, 0, switch_id, switch_id_len); if (err) return err; diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.h b/drivers/net/ethernet/mellanox/mlxsw/core.h index 22b0dfa7cfae..11af3308f8cc 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.h +++ b/drivers/net/ethernet/mellanox/mlxsw/core.h @@ -61,6 +61,7 @@ void mlxsw_core_ptp_transmitted(struct mlxsw_core *mlxsw_core, struct mlxsw_rx_listener { void (*func)(struct sk_buff *skb, u8 local_port, void *priv); u8 local_port; + u8 mirror_reason; u16 trap_id; }; @@ -88,13 +89,15 @@ struct mlxsw_listener { }; #define __MLXSW_RXL(_func, _trap_id, _en_action, _is_ctrl, _en_trap_group, \ - _dis_action, _enabled_on_register, _dis_trap_group) \ + _dis_action, _enabled_on_register, _dis_trap_group, \ + _mirror_reason) \ { \ .trap_id = MLXSW_TRAP_ID_##_trap_id, \ .rx_listener = \ { \ .func = _func, \ .local_port = MLXSW_PORT_DONT_CARE, \ + .mirror_reason = _mirror_reason, \ .trap_id = MLXSW_TRAP_ID_##_trap_id, \ }, \ .en_action = MLXSW_REG_HPKT_ACTION_##_en_action, \ @@ -108,12 +111,17 @@ struct mlxsw_listener { #define MLXSW_RXL(_func, _trap_id, _en_action, _is_ctrl, _trap_group, \ _dis_action) \ __MLXSW_RXL(_func, _trap_id, _en_action, _is_ctrl, _trap_group, \ - _dis_action, true, _trap_group) + _dis_action, true, _trap_group, 0) #define MLXSW_RXL_DIS(_func, _trap_id, _en_action, _is_ctrl, _en_trap_group, \ _dis_action, _dis_trap_group) \ __MLXSW_RXL(_func, _trap_id, _en_action, _is_ctrl, _en_trap_group, \ - _dis_action, false, _dis_trap_group) + _dis_action, false, _dis_trap_group, 0) + +#define MLXSW_RXL_MIRROR(_func, _session_id, _trap_group, _mirror_reason) \ + __MLXSW_RXL(_func, MIRROR_SESSION##_session_id, TRAP_TO_CPU, false, \ + _trap_group, TRAP_TO_CPU, true, _trap_group, \ + _mirror_reason) #define MLXSW_EVENTL(_func, _trap_id, _trap_group) \ { \ @@ -176,6 +184,7 @@ struct mlxsw_rx_info { u16 lag_id; } u; u8 lag_port_index; + u8 mirror_reason; int trap_id; }; @@ -191,8 +200,8 @@ void mlxsw_core_lag_mapping_clear(struct mlxsw_core *mlxsw_core, void *mlxsw_core_port_driver_priv(struct mlxsw_core_port *mlxsw_core_port); int mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u8 local_port, - u32 port_number, bool split, - u32 split_port_subnumber, + u32 port_number, bool split, u32 split_port_subnumber, + bool splittable, u32 lanes, const unsigned char *switch_id, unsigned char switch_id_len); void mlxsw_core_port_fini(struct mlxsw_core *mlxsw_core, u8 local_port); @@ -324,12 +333,14 @@ struct mlxsw_driver { const struct devlink_trap *trap, void *trap_ctx); int (*trap_action_set)(struct mlxsw_core *mlxsw_core, const struct devlink_trap *trap, - enum devlink_trap_action action); + enum devlink_trap_action action, + struct netlink_ext_ack *extack); int (*trap_group_init)(struct mlxsw_core *mlxsw_core, const struct devlink_trap_group *group); int (*trap_group_set)(struct mlxsw_core *mlxsw_core, const struct devlink_trap_group *group, - const struct devlink_trap_policer *policer); + const struct devlink_trap_policer *policer, + struct netlink_ext_ack *extack); int (*trap_policer_init)(struct mlxsw_core *mlxsw_core, const struct devlink_trap_policer *policer); void (*trap_policer_fini)(struct mlxsw_core *mlxsw_core, diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.c b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.c index c3d04319ff44..4d699fe98cb6 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.c @@ -67,7 +67,9 @@ struct mlxsw_afa { struct rhashtable set_ht; struct rhashtable fwd_entry_ht; struct rhashtable cookie_ht; + struct rhashtable policer_ht; struct idr cookie_idr; + struct list_head policer_list; }; #define MLXSW_AFA_SET_LEN 0xA8 @@ -88,9 +90,11 @@ struct mlxsw_afa_set { struct rhash_head ht_node; struct mlxsw_afa_set_ht_key ht_key; u32 kvdl_index; - bool shared; /* Inserted in hashtable (doesn't mean that + u8 shared:1, /* Inserted in hashtable (doesn't mean that * kvdl_index is valid). */ + has_trap:1, + has_police:1; unsigned int ref_count; struct mlxsw_afa_set *next; /* Pointer to the next set. */ struct mlxsw_afa_set *prev; /* Pointer to the previous set, @@ -175,6 +179,21 @@ static const struct rhashtable_params mlxsw_afa_cookie_ht_params = { .automatic_shrinking = true, }; +struct mlxsw_afa_policer { + struct rhash_head ht_node; + struct list_head list; /* Member of policer_list */ + refcount_t ref_count; + u32 fa_index; + u16 policer_index; +}; + +static const struct rhashtable_params mlxsw_afa_policer_ht_params = { + .key_len = sizeof(u32), + .key_offset = offsetof(struct mlxsw_afa_policer, fa_index), + .head_offset = offsetof(struct mlxsw_afa_policer, ht_node), + .automatic_shrinking = true, +}; + struct mlxsw_afa *mlxsw_afa_create(unsigned int max_acts_per_set, const struct mlxsw_afa_ops *ops, void *ops_priv) @@ -196,12 +215,19 @@ struct mlxsw_afa *mlxsw_afa_create(unsigned int max_acts_per_set, &mlxsw_afa_cookie_ht_params); if (err) goto err_cookie_rhashtable_init; + err = rhashtable_init(&mlxsw_afa->policer_ht, + &mlxsw_afa_policer_ht_params); + if (err) + goto err_policer_rhashtable_init; idr_init(&mlxsw_afa->cookie_idr); + INIT_LIST_HEAD(&mlxsw_afa->policer_list); mlxsw_afa->max_acts_per_set = max_acts_per_set; mlxsw_afa->ops = ops; mlxsw_afa->ops_priv = ops_priv; return mlxsw_afa; +err_policer_rhashtable_init: + rhashtable_destroy(&mlxsw_afa->cookie_ht); err_cookie_rhashtable_init: rhashtable_destroy(&mlxsw_afa->fwd_entry_ht); err_fwd_entry_rhashtable_init: @@ -214,8 +240,10 @@ EXPORT_SYMBOL(mlxsw_afa_create); void mlxsw_afa_destroy(struct mlxsw_afa *mlxsw_afa) { + WARN_ON(!list_empty(&mlxsw_afa->policer_list)); WARN_ON(!idr_is_empty(&mlxsw_afa->cookie_idr)); idr_destroy(&mlxsw_afa->cookie_idr); + rhashtable_destroy(&mlxsw_afa->policer_ht); rhashtable_destroy(&mlxsw_afa->cookie_ht); rhashtable_destroy(&mlxsw_afa->fwd_entry_ht); rhashtable_destroy(&mlxsw_afa->set_ht); @@ -836,19 +864,172 @@ err_cookie_get: return ERR_PTR(err); } +static struct mlxsw_afa_policer * +mlxsw_afa_policer_create(struct mlxsw_afa *mlxsw_afa, u32 fa_index, + u64 rate_bytes_ps, u32 burst, + struct netlink_ext_ack *extack) +{ + struct mlxsw_afa_policer *policer; + int err; + + policer = kzalloc(sizeof(*policer), GFP_KERNEL); + if (!policer) + return ERR_PTR(-ENOMEM); + + err = mlxsw_afa->ops->policer_add(mlxsw_afa->ops_priv, rate_bytes_ps, + burst, &policer->policer_index, + extack); + if (err) + goto err_policer_add; + + refcount_set(&policer->ref_count, 1); + policer->fa_index = fa_index; + + err = rhashtable_insert_fast(&mlxsw_afa->policer_ht, &policer->ht_node, + mlxsw_afa_policer_ht_params); + if (err) + goto err_rhashtable_insert; + + list_add_tail(&policer->list, &mlxsw_afa->policer_list); + + return policer; + +err_rhashtable_insert: + mlxsw_afa->ops->policer_del(mlxsw_afa->ops_priv, + policer->policer_index); +err_policer_add: + kfree(policer); + return ERR_PTR(err); +} + +static void mlxsw_afa_policer_destroy(struct mlxsw_afa *mlxsw_afa, + struct mlxsw_afa_policer *policer) +{ + list_del(&policer->list); + rhashtable_remove_fast(&mlxsw_afa->policer_ht, &policer->ht_node, + mlxsw_afa_policer_ht_params); + mlxsw_afa->ops->policer_del(mlxsw_afa->ops_priv, + policer->policer_index); + kfree(policer); +} + +static struct mlxsw_afa_policer * +mlxsw_afa_policer_get(struct mlxsw_afa *mlxsw_afa, u32 fa_index, + u64 rate_bytes_ps, u32 burst, + struct netlink_ext_ack *extack) +{ + struct mlxsw_afa_policer *policer; + + policer = rhashtable_lookup_fast(&mlxsw_afa->policer_ht, &fa_index, + mlxsw_afa_policer_ht_params); + if (policer) { + refcount_inc(&policer->ref_count); + return policer; + } + + return mlxsw_afa_policer_create(mlxsw_afa, fa_index, rate_bytes_ps, + burst, extack); +} + +static void mlxsw_afa_policer_put(struct mlxsw_afa *mlxsw_afa, + struct mlxsw_afa_policer *policer) +{ + if (!refcount_dec_and_test(&policer->ref_count)) + return; + mlxsw_afa_policer_destroy(mlxsw_afa, policer); +} + +struct mlxsw_afa_policer_ref { + struct mlxsw_afa_resource resource; + struct mlxsw_afa_policer *policer; +}; + +static void +mlxsw_afa_policer_ref_destroy(struct mlxsw_afa_block *block, + struct mlxsw_afa_policer_ref *policer_ref) +{ + mlxsw_afa_resource_del(&policer_ref->resource); + mlxsw_afa_policer_put(block->afa, policer_ref->policer); + kfree(policer_ref); +} + +static void +mlxsw_afa_policer_ref_destructor(struct mlxsw_afa_block *block, + struct mlxsw_afa_resource *resource) +{ + struct mlxsw_afa_policer_ref *policer_ref; + + policer_ref = container_of(resource, struct mlxsw_afa_policer_ref, + resource); + mlxsw_afa_policer_ref_destroy(block, policer_ref); +} + +static struct mlxsw_afa_policer_ref * +mlxsw_afa_policer_ref_create(struct mlxsw_afa_block *block, u32 fa_index, + u64 rate_bytes_ps, u32 burst, + struct netlink_ext_ack *extack) +{ + struct mlxsw_afa_policer_ref *policer_ref; + struct mlxsw_afa_policer *policer; + int err; + + policer_ref = kzalloc(sizeof(*policer_ref), GFP_KERNEL); + if (!policer_ref) + return ERR_PTR(-ENOMEM); + + policer = mlxsw_afa_policer_get(block->afa, fa_index, rate_bytes_ps, + burst, extack); + if (IS_ERR(policer)) { + err = PTR_ERR(policer); + goto err_policer_get; + } + + policer_ref->policer = policer; + policer_ref->resource.destructor = mlxsw_afa_policer_ref_destructor; + mlxsw_afa_resource_add(block, &policer_ref->resource); + + return policer_ref; + +err_policer_get: + kfree(policer_ref); + return ERR_PTR(err); +} + #define MLXSW_AFA_ONE_ACTION_LEN 32 #define MLXSW_AFA_PAYLOAD_OFFSET 4 -static char *mlxsw_afa_block_append_action(struct mlxsw_afa_block *block, - u8 action_code, u8 action_size) +enum mlxsw_afa_action_type { + MLXSW_AFA_ACTION_TYPE_TRAP, + MLXSW_AFA_ACTION_TYPE_POLICE, + MLXSW_AFA_ACTION_TYPE_OTHER, +}; + +static bool +mlxsw_afa_block_need_split(const struct mlxsw_afa_block *block, + enum mlxsw_afa_action_type type) +{ + struct mlxsw_afa_set *cur_set = block->cur_set; + + /* Due to a hardware limitation, police action cannot be in the same + * action set with MLXSW_AFA_TRAP_CODE or MLXSW_AFA_TRAPWU_CODE + * actions. Work around this limitation by creating a new action set + * and place the new action there. + */ + return (cur_set->has_trap && type == MLXSW_AFA_ACTION_TYPE_POLICE) || + (cur_set->has_police && type == MLXSW_AFA_ACTION_TYPE_TRAP); +} + +static char *mlxsw_afa_block_append_action_ext(struct mlxsw_afa_block *block, + u8 action_code, u8 action_size, + enum mlxsw_afa_action_type type) { char *oneact; char *actions; if (block->finished) return ERR_PTR(-EINVAL); - if (block->cur_act_index + action_size > - block->afa->max_acts_per_set) { + if (block->cur_act_index + action_size > block->afa->max_acts_per_set || + mlxsw_afa_block_need_split(block, type)) { struct mlxsw_afa_set *set; /* The appended action won't fit into the current action set, @@ -863,6 +1044,17 @@ static char *mlxsw_afa_block_append_action(struct mlxsw_afa_block *block, block->cur_set = set; } + switch (type) { + case MLXSW_AFA_ACTION_TYPE_TRAP: + block->cur_set->has_trap = true; + break; + case MLXSW_AFA_ACTION_TYPE_POLICE: + block->cur_set->has_police = true; + break; + default: + break; + } + actions = block->cur_set->ht_key.enc_actions; oneact = actions + block->cur_act_index * MLXSW_AFA_ONE_ACTION_LEN; block->cur_act_index += action_size; @@ -870,6 +1062,14 @@ static char *mlxsw_afa_block_append_action(struct mlxsw_afa_block *block, return oneact + MLXSW_AFA_PAYLOAD_OFFSET; } +static char *mlxsw_afa_block_append_action(struct mlxsw_afa_block *block, + u8 action_code, u8 action_size) +{ + return mlxsw_afa_block_append_action_ext(block, action_code, + action_size, + MLXSW_AFA_ACTION_TYPE_OTHER); +} + /* VLAN Action * ----------- * VLAN action is used for manipulating VLANs. It can be used to implement QinQ, @@ -1048,11 +1248,20 @@ mlxsw_afa_trap_mirror_pack(char *payload, bool mirror_enable, mlxsw_afa_trap_mirror_agent_set(payload, mirror_agent); } +static char *mlxsw_afa_block_append_action_trap(struct mlxsw_afa_block *block, + u8 action_code, u8 action_size) +{ + return mlxsw_afa_block_append_action_ext(block, action_code, + action_size, + MLXSW_AFA_ACTION_TYPE_TRAP); +} + static int mlxsw_afa_block_append_drop_plain(struct mlxsw_afa_block *block, bool ingress) { - char *act = mlxsw_afa_block_append_action(block, MLXSW_AFA_TRAP_CODE, - MLXSW_AFA_TRAP_SIZE); + char *act = mlxsw_afa_block_append_action_trap(block, + MLXSW_AFA_TRAP_CODE, + MLXSW_AFA_TRAP_SIZE); if (IS_ERR(act)) return PTR_ERR(act); @@ -1081,8 +1290,8 @@ mlxsw_afa_block_append_drop_with_cookie(struct mlxsw_afa_block *block, } cookie_index = cookie_ref->cookie->cookie_index; - act = mlxsw_afa_block_append_action(block, MLXSW_AFA_TRAPWU_CODE, - MLXSW_AFA_TRAPWU_SIZE); + act = mlxsw_afa_block_append_action_trap(block, MLXSW_AFA_TRAPWU_CODE, + MLXSW_AFA_TRAPWU_SIZE); if (IS_ERR(act)) { NL_SET_ERR_MSG_MOD(extack, "Cannot append drop with cookie action"); err = PTR_ERR(act); @@ -1113,8 +1322,9 @@ EXPORT_SYMBOL(mlxsw_afa_block_append_drop); int mlxsw_afa_block_append_trap(struct mlxsw_afa_block *block, u16 trap_id) { - char *act = mlxsw_afa_block_append_action(block, MLXSW_AFA_TRAP_CODE, - MLXSW_AFA_TRAP_SIZE); + char *act = mlxsw_afa_block_append_action_trap(block, + MLXSW_AFA_TRAP_CODE, + MLXSW_AFA_TRAP_SIZE); if (IS_ERR(act)) return PTR_ERR(act); @@ -1127,8 +1337,9 @@ EXPORT_SYMBOL(mlxsw_afa_block_append_trap); int mlxsw_afa_block_append_trap_and_forward(struct mlxsw_afa_block *block, u16 trap_id) { - char *act = mlxsw_afa_block_append_action(block, MLXSW_AFA_TRAP_CODE, - MLXSW_AFA_TRAP_SIZE); + char *act = mlxsw_afa_block_append_action_trap(block, + MLXSW_AFA_TRAP_CODE, + MLXSW_AFA_TRAP_SIZE); if (IS_ERR(act)) return PTR_ERR(act); @@ -1199,9 +1410,10 @@ static int mlxsw_afa_block_append_allocated_mirror(struct mlxsw_afa_block *block, u8 mirror_agent) { - char *act = mlxsw_afa_block_append_action(block, - MLXSW_AFA_TRAP_CODE, - MLXSW_AFA_TRAP_SIZE); + char *act = mlxsw_afa_block_append_action_trap(block, + MLXSW_AFA_TRAP_CODE, + MLXSW_AFA_TRAP_SIZE); + if (IS_ERR(act)) return PTR_ERR(act); mlxsw_afa_trap_pack(act, MLXSW_AFA_TRAP_TRAP_ACTION_NOP, @@ -1496,6 +1708,19 @@ EXPORT_SYMBOL(mlxsw_afa_block_append_fwd); #define MLXSW_AFA_POLCNT_CODE 0x08 #define MLXSW_AFA_POLCNT_SIZE 1 +enum { + MLXSW_AFA_POLCNT_COUNTER, + MLXSW_AFA_POLCNT_POLICER, +}; + +/* afa_polcnt_c_p + * Counter or policer. + * Indicates whether the action binds a policer or a counter to the flow. + * 0: Counter + * 1: Policer + */ +MLXSW_ITEM32(afa, polcnt, c_p, 0x00, 31, 1); + enum mlxsw_afa_polcnt_counter_set_type { /* No count */ MLXSW_AFA_POLCNT_COUNTER_SET_TYPE_NO_COUNT = 0x00, @@ -1515,15 +1740,28 @@ MLXSW_ITEM32(afa, polcnt, counter_set_type, 0x04, 24, 8); */ MLXSW_ITEM32(afa, polcnt, counter_index, 0x04, 0, 24); +/* afa_polcnt_pid + * Policer ID. + * Reserved when c_p = 0 + */ +MLXSW_ITEM32(afa, polcnt, pid, 0x08, 0, 14); + static inline void mlxsw_afa_polcnt_pack(char *payload, enum mlxsw_afa_polcnt_counter_set_type set_type, u32 counter_index) { + mlxsw_afa_polcnt_c_p_set(payload, MLXSW_AFA_POLCNT_COUNTER); mlxsw_afa_polcnt_counter_set_type_set(payload, set_type); mlxsw_afa_polcnt_counter_index_set(payload, counter_index); } +static void mlxsw_afa_polcnt_policer_pack(char *payload, u16 policer_index) +{ + mlxsw_afa_polcnt_c_p_set(payload, MLXSW_AFA_POLCNT_POLICER); + mlxsw_afa_polcnt_pid_set(payload, policer_index); +} + int mlxsw_afa_block_append_allocated_counter(struct mlxsw_afa_block *block, u32 counter_index) { @@ -1567,6 +1805,40 @@ err_append_allocated_counter: } EXPORT_SYMBOL(mlxsw_afa_block_append_counter); +int mlxsw_afa_block_append_police(struct mlxsw_afa_block *block, + u32 fa_index, u64 rate_bytes_ps, u32 burst, + u16 *p_policer_index, + struct netlink_ext_ack *extack) +{ + struct mlxsw_afa_policer_ref *policer_ref; + char *act; + int err; + + policer_ref = mlxsw_afa_policer_ref_create(block, fa_index, + rate_bytes_ps, + burst, extack); + if (IS_ERR(policer_ref)) + return PTR_ERR(policer_ref); + *p_policer_index = policer_ref->policer->policer_index; + + act = mlxsw_afa_block_append_action_ext(block, MLXSW_AFA_POLCNT_CODE, + MLXSW_AFA_POLCNT_SIZE, + MLXSW_AFA_ACTION_TYPE_POLICE); + if (IS_ERR(act)) { + NL_SET_ERR_MSG_MOD(extack, "Cannot append police action"); + err = PTR_ERR(act); + goto err_append_action; + } + mlxsw_afa_polcnt_policer_pack(act, *p_policer_index); + + return 0; + +err_append_action: + mlxsw_afa_policer_ref_destroy(block, policer_ref); + return err; +} +EXPORT_SYMBOL(mlxsw_afa_block_append_police); + /* Virtual Router and Forwarding Domain Action * ------------------------------------------- * Virtual Switch action is used for manipulate the Virtual Router (VR), @@ -1684,3 +1956,54 @@ int mlxsw_afa_block_append_mcrouter(struct mlxsw_afa_block *block, return 0; } EXPORT_SYMBOL(mlxsw_afa_block_append_mcrouter); + +/* L4 Port Action + * -------------- + * The L4_PORT_ACTION is used for modifying the sport and dport fields of the packet, e.g. for NAT. + * If (the L4 is TCP) or if (the L4 is UDP and checksum field!=0) then the L4 checksum is updated. + */ + +#define MLXSW_AFA_L4PORT_CODE 0x12 +#define MLXSW_AFA_L4PORT_SIZE 1 + +enum mlxsw_afa_l4port_s_d { + /* configure src_l4_port */ + MLXSW_AFA_L4PORT_S_D_SRC, + /* configure dst_l4_port */ + MLXSW_AFA_L4PORT_S_D_DST, +}; + +/* afa_l4port_s_d + * Source or destination. + */ +MLXSW_ITEM32(afa, l4port, s_d, 0x00, 31, 1); + +/* afa_l4port_l4_port + * Number of port to change to. + */ +MLXSW_ITEM32(afa, l4port, l4_port, 0x08, 0, 16); + +static void mlxsw_afa_l4port_pack(char *payload, enum mlxsw_afa_l4port_s_d s_d, u16 l4_port) +{ + mlxsw_afa_l4port_s_d_set(payload, s_d); + mlxsw_afa_l4port_l4_port_set(payload, l4_port); +} + +int mlxsw_afa_block_append_l4port(struct mlxsw_afa_block *block, bool is_dport, u16 l4_port, + struct netlink_ext_ack *extack) +{ + enum mlxsw_afa_l4port_s_d s_d = is_dport ? MLXSW_AFA_L4PORT_S_D_DST : + MLXSW_AFA_L4PORT_S_D_SRC; + char *act = mlxsw_afa_block_append_action(block, + MLXSW_AFA_L4PORT_CODE, + MLXSW_AFA_L4PORT_SIZE); + + if (IS_ERR(act)) { + NL_SET_ERR_MSG_MOD(extack, "Cannot append L4_PORT action"); + return PTR_ERR(act); + } + + mlxsw_afa_l4port_pack(act, s_d, l4_port); + return 0; +} +EXPORT_SYMBOL(mlxsw_afa_block_append_l4port); diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.h b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.h index 8c2705e16ef7..b652497b1002 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.h +++ b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.h @@ -26,6 +26,10 @@ struct mlxsw_afa_ops { bool ingress, int *p_span_id); void (*mirror_del)(void *priv, u8 local_in_port, int span_id, bool ingress); + int (*policer_add)(void *priv, u64 rate_bytes_ps, u32 burst, + u16 *p_policer_index, + struct netlink_ext_ack *extack); + void (*policer_del)(void *priv, u16 policer_index); bool dummy_first_set; }; @@ -82,5 +86,11 @@ int mlxsw_afa_block_append_fid_set(struct mlxsw_afa_block *block, u16 fid, int mlxsw_afa_block_append_mcrouter(struct mlxsw_afa_block *block, u16 expected_irif, u16 min_mtu, bool rmid_valid, u32 kvdl_index); +int mlxsw_afa_block_append_l4port(struct mlxsw_afa_block *block, bool is_dport, u16 l4_port, + struct netlink_ext_ack *extack); +int mlxsw_afa_block_append_police(struct mlxsw_afa_block *block, + u32 fa_index, u64 rate_bytes_ps, u32 burst, + u16 *p_policer_index, + struct netlink_ext_ack *extack); #endif diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_env.c b/drivers/net/ethernet/mellanox/mlxsw/core_env.c index a7d86df7123f..44fa02cbb683 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_env.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_env.c @@ -11,7 +11,7 @@ #include "reg.h" static int mlxsw_env_validate_cable_ident(struct mlxsw_core *core, int id, - bool *qsfp) + bool *qsfp, bool *cmis) { char eeprom_tmp[MLXSW_REG_MCIA_EEPROM_SIZE]; char mcia_pl[MLXSW_REG_MCIA_LEN]; @@ -25,15 +25,19 @@ static int mlxsw_env_validate_cable_ident(struct mlxsw_core *core, int id, return err; mlxsw_reg_mcia_eeprom_memcpy_from(mcia_pl, eeprom_tmp); ident = eeprom_tmp[0]; + *cmis = false; switch (ident) { case MLXSW_REG_MCIA_EEPROM_MODULE_INFO_ID_SFP: *qsfp = false; break; case MLXSW_REG_MCIA_EEPROM_MODULE_INFO_ID_QSFP: /* fall-through */ case MLXSW_REG_MCIA_EEPROM_MODULE_INFO_ID_QSFP_PLUS: /* fall-through */ - case MLXSW_REG_MCIA_EEPROM_MODULE_INFO_ID_QSFP28: /* fall-through */ + case MLXSW_REG_MCIA_EEPROM_MODULE_INFO_ID_QSFP28: + *qsfp = true; + break; case MLXSW_REG_MCIA_EEPROM_MODULE_INFO_ID_QSFP_DD: *qsfp = true; + *cmis = true; break; default: return -EINVAL; @@ -70,8 +74,9 @@ mlxsw_env_query_module_eeprom(struct mlxsw_core *mlxsw_core, int module, if (qsfp) { /* When reading upper pages 1, 2 and 3 the offset * starts at 128. Please refer to "QSFP+ Memory Map" - * figure in SFF-8436 specification for graphical - * depiction. + * figure in SFF-8436 specification and to "CMIS Module + * Memory Map" figure in CMIS specification for + * graphical depiction. */ page = MLXSW_REG_MCIA_PAGE_GET(offset); offset -= MLXSW_REG_MCIA_EEPROM_UP_PAGE_LENGTH * page; @@ -116,7 +121,8 @@ int mlxsw_env_module_temp_thresholds_get(struct mlxsw_core *core, int module, char mcia_pl[MLXSW_REG_MCIA_LEN] = {0}; char mtmp_pl[MLXSW_REG_MTMP_LEN]; unsigned int module_temp; - bool qsfp; + bool qsfp, cmis; + int page; int err; mlxsw_reg_mtmp_pack(mtmp_pl, MLXSW_REG_MTMP_MODULE_INDEX_MIN + module, @@ -140,21 +146,28 @@ int mlxsw_env_module_temp_thresholds_get(struct mlxsw_core *core, int module, */ /* Validate module identifier value. */ - err = mlxsw_env_validate_cable_ident(core, module, &qsfp); + err = mlxsw_env_validate_cable_ident(core, module, &qsfp, &cmis); if (err) return err; - if (qsfp) - mlxsw_reg_mcia_pack(mcia_pl, module, 0, - MLXSW_REG_MCIA_TH_PAGE_NUM, + if (qsfp) { + /* For QSFP/CMIS module-defined thresholds are located in page + * 02h, otherwise in page 03h. + */ + if (cmis) + page = MLXSW_REG_MCIA_TH_PAGE_CMIS_NUM; + else + page = MLXSW_REG_MCIA_TH_PAGE_NUM; + mlxsw_reg_mcia_pack(mcia_pl, module, 0, page, MLXSW_REG_MCIA_TH_PAGE_OFF + off, MLXSW_REG_MCIA_TH_ITEM_SIZE, MLXSW_REG_MCIA_I2C_ADDR_LOW); - else + } else { mlxsw_reg_mcia_pack(mcia_pl, module, 0, MLXSW_REG_MCIA_PAGE0_LO, off, MLXSW_REG_MCIA_TH_ITEM_SIZE, MLXSW_REG_MCIA_I2C_ADDR_HIGH); + } err = mlxsw_reg_query(core, MLXSW_REG(mcia), mcia_pl); if (err) @@ -221,6 +234,22 @@ int mlxsw_env_get_module_info(struct mlxsw_core *mlxsw_core, int module, else modinfo->eeprom_len = ETH_MODULE_SFF_8472_LEN / 2; break; + case MLXSW_REG_MCIA_EEPROM_MODULE_INFO_ID_QSFP_DD: + /* Use SFF_8636 as base type. ethtool should recognize specific + * type through the identifier value. + */ + modinfo->type = ETH_MODULE_SFF_8636; + /* Verify if module EEPROM is a flat memory. In case of flat + * memory only page 00h (0-255 bytes) can be read. Otherwise + * upper pages 01h and 02h can also be read. Upper pages 10h + * and 11h are currently not supported by the driver. + */ + if (module_info[MLXSW_REG_MCIA_EEPROM_MODULE_INFO_TYPE_ID] & + MLXSW_REG_MCIA_EEPROM_CMIS_FLAT_MEMORY) + modinfo->eeprom_len = ETH_MODULE_SFF_8636_LEN; + else + modinfo->eeprom_len = ETH_MODULE_SFF_8472_LEN; + break; default: return -EINVAL; } @@ -235,8 +264,8 @@ int mlxsw_env_get_module_eeprom(struct net_device *netdev, { int offset = ee->offset; unsigned int read_size; + bool qsfp, cmis; int i = 0; - bool qsfp; int err; if (!ee->len) @@ -244,7 +273,7 @@ int mlxsw_env_get_module_eeprom(struct net_device *netdev, memset(data, 0, ee->len); /* Validate module identifier value. */ - err = mlxsw_env_validate_cable_ident(mlxsw_core, module, &qsfp); + err = mlxsw_env_validate_cable_ident(mlxsw_core, module, &qsfp, &cmis); if (err) return err; diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c index 05f8d5a92862..8fa286ccdd6b 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c @@ -98,7 +98,6 @@ struct mlxsw_thermal_module { struct mlxsw_thermal *parent; struct thermal_zone_device *tzdev; struct mlxsw_thermal_trip trips[MLXSW_THERMAL_NUM_TRIPS]; - enum thermal_device_mode mode; int module; /* Module or gearbox number */ }; @@ -110,7 +109,6 @@ struct mlxsw_thermal { struct thermal_cooling_device *cdevs[MLXSW_MFCR_PWMS_MAX]; u8 cooling_levels[MLXSW_THERMAL_MAX_STATE + 1]; struct mlxsw_thermal_trip trips[MLXSW_THERMAL_NUM_TRIPS]; - enum thermal_device_mode mode; struct mlxsw_thermal_module *tz_module_arr; u8 tz_module_num; struct mlxsw_thermal_module *tz_gearbox_arr; @@ -277,36 +275,6 @@ static int mlxsw_thermal_unbind(struct thermal_zone_device *tzdev, return 0; } -static int mlxsw_thermal_get_mode(struct thermal_zone_device *tzdev, - enum thermal_device_mode *mode) -{ - struct mlxsw_thermal *thermal = tzdev->devdata; - - *mode = thermal->mode; - - return 0; -} - -static int mlxsw_thermal_set_mode(struct thermal_zone_device *tzdev, - enum thermal_device_mode mode) -{ - struct mlxsw_thermal *thermal = tzdev->devdata; - - mutex_lock(&tzdev->lock); - - if (mode == THERMAL_DEVICE_ENABLED) - tzdev->polling_delay = thermal->polling_delay; - else - tzdev->polling_delay = 0; - - mutex_unlock(&tzdev->lock); - - thermal->mode = mode; - thermal_zone_device_update(tzdev, THERMAL_EVENT_UNSPECIFIED); - - return 0; -} - static int mlxsw_thermal_get_temp(struct thermal_zone_device *tzdev, int *p_temp) { @@ -406,8 +374,6 @@ static int mlxsw_thermal_trend_get(struct thermal_zone_device *tzdev, static struct thermal_zone_device_ops mlxsw_thermal_ops = { .bind = mlxsw_thermal_bind, .unbind = mlxsw_thermal_unbind, - .get_mode = mlxsw_thermal_get_mode, - .set_mode = mlxsw_thermal_set_mode, .get_temp = mlxsw_thermal_get_temp, .get_trip_type = mlxsw_thermal_get_trip_type, .get_trip_temp = mlxsw_thermal_get_trip_temp, @@ -465,37 +431,6 @@ static int mlxsw_thermal_module_unbind(struct thermal_zone_device *tzdev, return err; } -static int mlxsw_thermal_module_mode_get(struct thermal_zone_device *tzdev, - enum thermal_device_mode *mode) -{ - struct mlxsw_thermal_module *tz = tzdev->devdata; - - *mode = tz->mode; - - return 0; -} - -static int mlxsw_thermal_module_mode_set(struct thermal_zone_device *tzdev, - enum thermal_device_mode mode) -{ - struct mlxsw_thermal_module *tz = tzdev->devdata; - struct mlxsw_thermal *thermal = tz->parent; - - mutex_lock(&tzdev->lock); - - if (mode == THERMAL_DEVICE_ENABLED) - tzdev->polling_delay = thermal->polling_delay; - else - tzdev->polling_delay = 0; - - mutex_unlock(&tzdev->lock); - - tz->mode = mode; - thermal_zone_device_update(tzdev, THERMAL_EVENT_UNSPECIFIED); - - return 0; -} - static int mlxsw_thermal_module_temp_get(struct thermal_zone_device *tzdev, int *p_temp) { @@ -611,8 +546,6 @@ static int mlxsw_thermal_module_trend_get(struct thermal_zone_device *tzdev, static struct thermal_zone_device_ops mlxsw_thermal_module_ops = { .bind = mlxsw_thermal_module_bind, .unbind = mlxsw_thermal_module_unbind, - .get_mode = mlxsw_thermal_module_mode_get, - .set_mode = mlxsw_thermal_module_mode_set, .get_temp = mlxsw_thermal_module_temp_get, .get_trip_type = mlxsw_thermal_module_trip_type_get, .get_trip_temp = mlxsw_thermal_module_trip_temp_get, @@ -650,8 +583,6 @@ static int mlxsw_thermal_gearbox_temp_get(struct thermal_zone_device *tzdev, static struct thermal_zone_device_ops mlxsw_thermal_gearbox_ops = { .bind = mlxsw_thermal_module_bind, .unbind = mlxsw_thermal_module_unbind, - .get_mode = mlxsw_thermal_module_mode_get, - .set_mode = mlxsw_thermal_module_mode_set, .get_temp = mlxsw_thermal_gearbox_temp_get, .get_trip_type = mlxsw_thermal_module_trip_type_get, .get_trip_temp = mlxsw_thermal_module_trip_temp_get, @@ -780,8 +711,11 @@ mlxsw_thermal_module_tz_init(struct mlxsw_thermal_module *module_tz) return err; } - module_tz->mode = THERMAL_DEVICE_ENABLED; - return 0; + err = thermal_zone_device_enable(module_tz->tzdev); + if (err) + thermal_zone_device_unregister(module_tz->tzdev); + + return err; } static void mlxsw_thermal_module_tz_fini(struct thermal_zone_device *tzdev) @@ -884,6 +818,7 @@ static int mlxsw_thermal_gearbox_tz_init(struct mlxsw_thermal_module *gearbox_tz) { char tz_name[MLXSW_THERMAL_ZONE_MAX_NAME]; + int ret; snprintf(tz_name, sizeof(tz_name), "mlxsw-gearbox%d", gearbox_tz->module + 1); @@ -896,8 +831,11 @@ mlxsw_thermal_gearbox_tz_init(struct mlxsw_thermal_module *gearbox_tz) if (IS_ERR(gearbox_tz->tzdev)) return PTR_ERR(gearbox_tz->tzdev); - gearbox_tz->mode = THERMAL_DEVICE_ENABLED; - return 0; + ret = thermal_zone_device_enable(gearbox_tz->tzdev); + if (ret) + thermal_zone_device_unregister(gearbox_tz->tzdev); + + return ret; } static void @@ -1065,10 +1003,15 @@ int mlxsw_thermal_init(struct mlxsw_core *core, if (err) goto err_unreg_modules_tzdev; - thermal->mode = THERMAL_DEVICE_ENABLED; + err = thermal_zone_device_enable(thermal->tzdev); + if (err) + goto err_unreg_gearboxes; + *p_thermal = thermal; return 0; +err_unreg_gearboxes: + mlxsw_thermal_gearboxes_fini(thermal); err_unreg_modules_tzdev: mlxsw_thermal_modules_fini(thermal); err_unreg_tzdev: diff --git a/drivers/net/ethernet/mellanox/mlxsw/minimal.c b/drivers/net/ethernet/mellanox/mlxsw/minimal.c index c4caeeadcba9..c010db2c9dba 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/minimal.c +++ b/drivers/net/ethernet/mellanox/mlxsw/minimal.c @@ -164,8 +164,8 @@ mlxsw_m_port_create(struct mlxsw_m *mlxsw_m, u8 local_port, u8 module) int err; err = mlxsw_core_port_init(mlxsw_m->core, local_port, - module + 1, false, 0, - mlxsw_m->base_mac, + module + 1, false, 0, false, + 0, mlxsw_m->base_mac, sizeof(mlxsw_m->base_mac)); if (err) { dev_err(mlxsw_m->bus_info->dev, "Port %d: Failed to init core port\n", diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index c04ec1a92826..1c64b03ff48e 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -547,9 +547,9 @@ static void mlxsw_pci_cqe_rdq_handle(struct mlxsw_pci *mlxsw_pci, { struct pci_dev *pdev = mlxsw_pci->pdev; struct mlxsw_pci_queue_elem_info *elem_info; + struct mlxsw_rx_info rx_info = {}; char *wqe; struct sk_buff *skb; - struct mlxsw_rx_info rx_info; u16 byte_count; int err; @@ -582,6 +582,10 @@ static void mlxsw_pci_cqe_rdq_handle(struct mlxsw_pci *mlxsw_pci, if (mlxsw_pci->max_cqe_ver >= MLXSW_PCI_CQE_V2) cookie_index = mlxsw_pci_cqe2_user_def_val_orig_pkt_len_get(cqe); mlxsw_skb_cb(skb)->cookie_index = cookie_index; + } else if (rx_info.trap_id >= MLXSW_TRAP_ID_MIRROR_SESSION0 && + rx_info.trap_id <= MLXSW_TRAP_ID_MIRROR_SESSION7 && + mlxsw_pci->max_cqe_ver >= MLXSW_PCI_CQE_V2) { + rx_info.mirror_reason = mlxsw_pci_cqe2_mirror_reason_get(cqe); } byte_count = mlxsw_pci_cqe_byte_count_get(cqe); diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h b/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h index 32c7cabfb261..a2c1fbd3e0d1 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h +++ b/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h @@ -176,7 +176,7 @@ MLXSW_ITEM32(pci, cqe, byte_count, 0x04, 0, 14); /* pci_cqe_trap_id * Trap ID that captured the packet. */ -MLXSW_ITEM32(pci, cqe, trap_id, 0x08, 0, 9); +MLXSW_ITEM32(pci, cqe, trap_id, 0x08, 0, 10); /* pci_cqe_crc * Length include CRC. Indicates the length field includes @@ -213,6 +213,11 @@ mlxsw_pci_cqe_item_helpers(dqn, 0, 12, 12); */ MLXSW_ITEM32(pci, cqe2, user_def_val_orig_pkt_len, 0x14, 0, 20); +/* pci_cqe_mirror_reason + * Mirror reason. + */ +MLXSW_ITEM32(pci, cqe2, mirror_reason, 0x18, 24, 8); + /* pci_cqe_owner * Ownership bit. */ diff --git a/drivers/net/ethernet/mellanox/mlxsw/reg.h b/drivers/net/ethernet/mellanox/mlxsw/reg.h index fcb88d4271bf..079b080de7f7 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/reg.h +++ b/drivers/net/ethernet/mellanox/mlxsw/reg.h @@ -3405,11 +3405,20 @@ MLXSW_ITEM32(reg, qpcr, violate_action, 0x18, 0, 4); */ MLXSW_ITEM64(reg, qpcr, violate_count, 0x20, 0, 64); +/* Packets */ #define MLXSW_REG_QPCR_LOWEST_CIR 1 #define MLXSW_REG_QPCR_HIGHEST_CIR (2 * 1000 * 1000 * 1000) /* 2Gpps */ #define MLXSW_REG_QPCR_LOWEST_CBS 4 #define MLXSW_REG_QPCR_HIGHEST_CBS 24 +/* Bandwidth */ +#define MLXSW_REG_QPCR_LOWEST_CIR_BITS 1024 /* bps */ +#define MLXSW_REG_QPCR_HIGHEST_CIR_BITS 2000000000000ULL /* 2Tbps */ +#define MLXSW_REG_QPCR_LOWEST_CBS_BITS_SP1 4 +#define MLXSW_REG_QPCR_LOWEST_CBS_BITS_SP2 4 +#define MLXSW_REG_QPCR_HIGHEST_CBS_BITS_SP1 25 +#define MLXSW_REG_QPCR_HIGHEST_CBS_BITS_SP2 31 + static inline void mlxsw_reg_qpcr_pack(char *payload, u16 pid, enum mlxsw_reg_qpcr_ir_units ir_units, bool bytes, u32 cir, u16 cbs) @@ -5438,6 +5447,56 @@ static inline void mlxsw_reg_pplr_pack(char *payload, u8 local_port, MLXSW_REG_PPLR_LB_TYPE_BIT_PHY_LOCAL : 0); } +/* PDDR - Port Diagnostics Database Register + * ----------------------------------------- + * The PDDR enables to read the Phy debug database + */ +#define MLXSW_REG_PDDR_ID 0x5031 +#define MLXSW_REG_PDDR_LEN 0x100 + +MLXSW_REG_DEFINE(pddr, MLXSW_REG_PDDR_ID, MLXSW_REG_PDDR_LEN); + +/* reg_pddr_local_port + * Local port number. + * Access: Index + */ +MLXSW_ITEM32(reg, pddr, local_port, 0x00, 16, 8); + +enum mlxsw_reg_pddr_page_select { + MLXSW_REG_PDDR_PAGE_SELECT_TROUBLESHOOTING_INFO = 1, +}; + +/* reg_pddr_page_select + * Page select index. + * Access: Index + */ +MLXSW_ITEM32(reg, pddr, page_select, 0x04, 0, 8); + +enum mlxsw_reg_pddr_trblsh_group_opcode { + /* Monitor opcodes */ + MLXSW_REG_PDDR_TRBLSH_GROUP_OPCODE_MONITOR, +}; + +/* reg_pddr_group_opcode + * Group selector. + * Access: Index + */ +MLXSW_ITEM32(reg, pddr, trblsh_group_opcode, 0x08, 0, 16); + +/* reg_pddr_status_opcode + * Group selector. + * Access: RO + */ +MLXSW_ITEM32(reg, pddr, trblsh_status_opcode, 0x0C, 0, 16); + +static inline void mlxsw_reg_pddr_pack(char *payload, u8 local_port, + u8 page_select) +{ + MLXSW_REG_ZERO(pddr, payload); + mlxsw_reg_pddr_local_port_set(payload, local_port); + mlxsw_reg_pddr_page_select_set(payload, page_select); +} + /* PMTM - Port Module Type Mapping Register * ---------------------------------------- * The PMTM allows query or configuration of module types. @@ -5536,6 +5595,7 @@ enum mlxsw_reg_htgt_trap_group { MLXSW_REG_HTGT_TRAP_GROUP_SP_MULTICAST, MLXSW_REG_HTGT_TRAP_GROUP_SP_NEIGH_DISCOVERY, MLXSW_REG_HTGT_TRAP_GROUP_SP_ROUTER_EXP, + MLXSW_REG_HTGT_TRAP_GROUP_SP_EXTERNAL_ROUTE, MLXSW_REG_HTGT_TRAP_GROUP_SP_IP2ME, MLXSW_REG_HTGT_TRAP_GROUP_SP_DHCP, MLXSW_REG_HTGT_TRAP_GROUP_SP_EVENT, @@ -5554,6 +5614,7 @@ enum mlxsw_reg_htgt_trap_group { MLXSW_REG_HTGT_TRAP_GROUP_SP_L3_EXCEPTIONS, MLXSW_REG_HTGT_TRAP_GROUP_SP_TUNNEL_DISCARDS, MLXSW_REG_HTGT_TRAP_GROUP_SP_ACL_DISCARDS, + MLXSW_REG_HTGT_TRAP_GROUP_SP_BUFFER_DISCARDS, __MLXSW_REG_HTGT_TRAP_GROUP_MAX, MLXSW_REG_HTGT_TRAP_GROUP_MAX = __MLXSW_REG_HTGT_TRAP_GROUP_MAX - 1 @@ -5728,7 +5789,7 @@ MLXSW_ITEM32(reg, hpkt, trap_group, 0x00, 12, 6); * Note: A trap ID can only be associated with a single trap group. The device * will associate the trap ID with the last trap group configured. */ -MLXSW_ITEM32(reg, hpkt, trap_id, 0x00, 0, 9); +MLXSW_ITEM32(reg, hpkt, trap_id, 0x00, 0, 10); enum { MLXSW_REG_HPKT_CTRL_PACKET_DEFAULT, @@ -8546,8 +8607,10 @@ MLXSW_ITEM32(reg, mcia, size, 0x08, 0, 16); #define MLXSW_REG_MCIA_PAGE0_LO_OFF 0xa0 #define MLXSW_REG_MCIA_TH_ITEM_SIZE 2 #define MLXSW_REG_MCIA_TH_PAGE_NUM 3 +#define MLXSW_REG_MCIA_TH_PAGE_CMIS_NUM 2 #define MLXSW_REG_MCIA_PAGE0_LO 0 #define MLXSW_REG_MCIA_TH_PAGE_OFF 0x80 +#define MLXSW_REG_MCIA_EEPROM_CMIS_FLAT_MEMORY BIT(7) enum mlxsw_reg_mcia_eeprom_module_info_rev_id { MLXSW_REG_MCIA_EEPROM_MODULE_INFO_REV_ID_UNSPC = 0x00, @@ -8566,6 +8629,7 @@ enum mlxsw_reg_mcia_eeprom_module_info_id { enum mlxsw_reg_mcia_eeprom_module_info { MLXSW_REG_MCIA_EEPROM_MODULE_INFO_ID, MLXSW_REG_MCIA_EEPROM_MODULE_INFO_REV_ID, + MLXSW_REG_MCIA_EEPROM_MODULE_INFO_TYPE_ID, MLXSW_REG_MCIA_EEPROM_MODULE_INFO_SIZE, }; @@ -8612,6 +8676,13 @@ MLXSW_REG_DEFINE(mpat, MLXSW_REG_MPAT_ID, MLXSW_REG_MPAT_LEN); */ MLXSW_ITEM32(reg, mpat, pa_id, 0x00, 28, 4); +/* reg_mpat_session_id + * Mirror Session ID. + * Used for MIRROR_SESSION<i> trap. + * Access: RW + */ +MLXSW_ITEM32(reg, mpat, session_id, 0x00, 24, 4); + /* reg_mpat_system_port * A unique port identifier for the final destination of the packet. * Access: RW @@ -8669,6 +8740,18 @@ enum mlxsw_reg_mpat_span_type { */ MLXSW_ITEM32(reg, mpat, span_type, 0x04, 0, 4); +/* reg_mpat_pide + * Policer enable. + * Access: RW + */ +MLXSW_ITEM32(reg, mpat, pide, 0x0C, 15, 1); + +/* reg_mpat_pid + * Policer ID. + * Access: RW + */ +MLXSW_ITEM32(reg, mpat, pid, 0x0C, 0, 14); + /* Remote SPAN - Ethernet VLAN * - - - - - - - - - - - - - - */ @@ -9452,6 +9535,114 @@ MLXSW_ITEM32(reg, mogcr, ptp_iftc, 0x00, 1, 1); */ MLXSW_ITEM32(reg, mogcr, ptp_eftc, 0x00, 0, 1); +/* reg_mogcr_mirroring_pid_base + * Base policer id for mirroring policers. + * Must have an even value (e.g. 1000, not 1001). + * Reserved when SwitchX/-2, Switch-IB/2, Spectrum-1 and Quantum. + * Access: RW + */ +MLXSW_ITEM32(reg, mogcr, mirroring_pid_base, 0x0C, 0, 14); + +/* MPAGR - Monitoring Port Analyzer Global Register + * ------------------------------------------------ + * This register is used for global port analyzer configurations. + * Note: This register is not supported by current FW versions for Spectrum-1. + */ +#define MLXSW_REG_MPAGR_ID 0x9089 +#define MLXSW_REG_MPAGR_LEN 0x0C + +MLXSW_REG_DEFINE(mpagr, MLXSW_REG_MPAGR_ID, MLXSW_REG_MPAGR_LEN); + +enum mlxsw_reg_mpagr_trigger { + MLXSW_REG_MPAGR_TRIGGER_EGRESS, + MLXSW_REG_MPAGR_TRIGGER_INGRESS, + MLXSW_REG_MPAGR_TRIGGER_INGRESS_WRED, + MLXSW_REG_MPAGR_TRIGGER_INGRESS_SHARED_BUFFER, + MLXSW_REG_MPAGR_TRIGGER_INGRESS_ING_CONG, + MLXSW_REG_MPAGR_TRIGGER_INGRESS_EGR_CONG, + MLXSW_REG_MPAGR_TRIGGER_EGRESS_ECN, + MLXSW_REG_MPAGR_TRIGGER_EGRESS_HIGH_LATENCY, +}; + +/* reg_mpagr_trigger + * Mirror trigger. + * Access: Index + */ +MLXSW_ITEM32(reg, mpagr, trigger, 0x00, 0, 4); + +/* reg_mpagr_pa_id + * Port analyzer ID. + * Access: RW + */ +MLXSW_ITEM32(reg, mpagr, pa_id, 0x04, 0, 4); + +/* reg_mpagr_probability_rate + * Sampling rate. + * Valid values are: 1 to 3.5*10^9 + * Value of 1 means "sample all". Default is 1. + * Access: RW + */ +MLXSW_ITEM32(reg, mpagr, probability_rate, 0x08, 0, 32); + +static inline void mlxsw_reg_mpagr_pack(char *payload, + enum mlxsw_reg_mpagr_trigger trigger, + u8 pa_id, u32 probability_rate) +{ + MLXSW_REG_ZERO(mpagr, payload); + mlxsw_reg_mpagr_trigger_set(payload, trigger); + mlxsw_reg_mpagr_pa_id_set(payload, pa_id); + mlxsw_reg_mpagr_probability_rate_set(payload, probability_rate); +} + +/* MOMTE - Monitoring Mirror Trigger Enable Register + * ------------------------------------------------- + * This register is used to configure the mirror enable for different mirror + * reasons. + */ +#define MLXSW_REG_MOMTE_ID 0x908D +#define MLXSW_REG_MOMTE_LEN 0x10 + +MLXSW_REG_DEFINE(momte, MLXSW_REG_MOMTE_ID, MLXSW_REG_MOMTE_LEN); + +/* reg_momte_local_port + * Local port number. + * Access: Index + */ +MLXSW_ITEM32(reg, momte, local_port, 0x00, 16, 8); + +enum mlxsw_reg_momte_type { + MLXSW_REG_MOMTE_TYPE_WRED = 0x20, + MLXSW_REG_MOMTE_TYPE_SHARED_BUFFER_TCLASS = 0x31, + MLXSW_REG_MOMTE_TYPE_SHARED_BUFFER_TCLASS_DESCRIPTORS = 0x32, + MLXSW_REG_MOMTE_TYPE_SHARED_BUFFER_EGRESS_PORT = 0x33, + MLXSW_REG_MOMTE_TYPE_ING_CONG = 0x40, + MLXSW_REG_MOMTE_TYPE_EGR_CONG = 0x50, + MLXSW_REG_MOMTE_TYPE_ECN = 0x60, + MLXSW_REG_MOMTE_TYPE_HIGH_LATENCY = 0x70, +}; + +/* reg_momte_type + * Type of mirroring. + * Access: Index + */ +MLXSW_ITEM32(reg, momte, type, 0x04, 0, 8); + +/* reg_momte_tclass_en + * TClass/PG mirror enable. Each bit represents corresponding tclass. + * 0: disable (default) + * 1: enable + * Access: RW + */ +MLXSW_ITEM_BIT_ARRAY(reg, momte, tclass_en, 0x08, 0x08, 1); + +static inline void mlxsw_reg_momte_pack(char *payload, u8 local_port, + enum mlxsw_reg_momte_type type) +{ + MLXSW_REG_ZERO(momte, payload); + mlxsw_reg_momte_local_port_set(payload, local_port); + mlxsw_reg_momte_type_set(payload, type); +} + /* MTPPPC - Time Precision Packet Port Configuration * ------------------------------------------------- * This register serves for configuration of which PTP messages should be @@ -10758,6 +10949,7 @@ static const struct mlxsw_reg_info *mlxsw_reg_infos[] = { MLXSW_REG(pbmc), MLXSW_REG(pspa), MLXSW_REG(pplr), + MLXSW_REG(pddr), MLXSW_REG(pmtm), MLXSW_REG(htgt), MLXSW_REG(hpkt), @@ -10802,6 +10994,8 @@ static const struct mlxsw_reg_info *mlxsw_reg_infos[] = { MLXSW_REG(mgpc), MLXSW_REG(mprs), MLXSW_REG(mogcr), + MLXSW_REG(mpagr), + MLXSW_REG(momte), MLXSW_REG(mtpppc), MLXSW_REG(mtpptr), MLXSW_REG(mtptpt), diff --git a/drivers/net/ethernet/mellanox/mlxsw/resources.h b/drivers/net/ethernet/mellanox/mlxsw/resources.h index d62496ef299c..a56c9e19a390 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/resources.h +++ b/drivers/net/ethernet/mellanox/mlxsw/resources.h @@ -47,6 +47,7 @@ enum mlxsw_res_id { MLXSW_RES_ID_ACL_ERPT_ENTRIES_8KB, MLXSW_RES_ID_ACL_ERPT_ENTRIES_12KB, MLXSW_RES_ID_ACL_MAX_BF_LOG, + MLXSW_RES_ID_MAX_GLOBAL_POLICERS, MLXSW_RES_ID_MAX_CPU_POLICERS, MLXSW_RES_ID_MAX_VRS, MLXSW_RES_ID_MAX_RIFS, @@ -105,6 +106,7 @@ static u16 mlxsw_res_ids[] = { [MLXSW_RES_ID_ACL_ERPT_ENTRIES_8KB] = 0x2952, [MLXSW_RES_ID_ACL_ERPT_ENTRIES_12KB] = 0x2953, [MLXSW_RES_ID_ACL_MAX_BF_LOG] = 0x2960, + [MLXSW_RES_ID_MAX_GLOBAL_POLICERS] = 0x2A10, [MLXSW_RES_ID_MAX_CPU_POLICERS] = 0x2A13, [MLXSW_RES_ID_MAX_VRS] = 0x2C01, [MLXSW_RES_ID_MAX_RIFS] = 0x2C02, diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c index 029ea344ad65..fdf9aa8314b2 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c @@ -45,8 +45,8 @@ #include "../mlxfw/mlxfw.h" #define MLXSW_SP1_FWREV_MAJOR 13 -#define MLXSW_SP1_FWREV_MINOR 2000 -#define MLXSW_SP1_FWREV_SUBMINOR 2714 +#define MLXSW_SP1_FWREV_MINOR 2007 +#define MLXSW_SP1_FWREV_SUBMINOR 1168 #define MLXSW_SP1_FWREV_CAN_RESET_MINOR 1702 static const struct mlxsw_fw_rev mlxsw_sp1_fw_rev = { @@ -62,8 +62,8 @@ static const struct mlxsw_fw_rev mlxsw_sp1_fw_rev = { "." __stringify(MLXSW_SP1_FWREV_SUBMINOR) ".mfa2" #define MLXSW_SP2_FWREV_MAJOR 29 -#define MLXSW_SP2_FWREV_MINOR 2000 -#define MLXSW_SP2_FWREV_SUBMINOR 2714 +#define MLXSW_SP2_FWREV_MINOR 2007 +#define MLXSW_SP2_FWREV_SUBMINOR 1168 static const struct mlxsw_fw_rev mlxsw_sp2_fw_rev = { .major = MLXSW_SP2_FWREV_MAJOR, @@ -76,10 +76,24 @@ static const struct mlxsw_fw_rev mlxsw_sp2_fw_rev = { "." __stringify(MLXSW_SP2_FWREV_MINOR) \ "." __stringify(MLXSW_SP2_FWREV_SUBMINOR) ".mfa2" +#define MLXSW_SP3_FWREV_MAJOR 30 +#define MLXSW_SP3_FWREV_MINOR 2007 +#define MLXSW_SP3_FWREV_SUBMINOR 1168 + +static const struct mlxsw_fw_rev mlxsw_sp3_fw_rev = { + .major = MLXSW_SP3_FWREV_MAJOR, + .minor = MLXSW_SP3_FWREV_MINOR, + .subminor = MLXSW_SP3_FWREV_SUBMINOR, +}; + +#define MLXSW_SP3_FW_FILENAME \ + "mellanox/mlxsw_spectrum3-" __stringify(MLXSW_SP3_FWREV_MAJOR) \ + "." __stringify(MLXSW_SP3_FWREV_MINOR) \ + "." __stringify(MLXSW_SP3_FWREV_SUBMINOR) ".mfa2" + static const char mlxsw_sp1_driver_name[] = "mlxsw_spectrum"; static const char mlxsw_sp2_driver_name[] = "mlxsw_spectrum2"; static const char mlxsw_sp3_driver_name[] = "mlxsw_spectrum3"; -static const char mlxsw_sp_driver_version[] = "1.0"; static const unsigned char mlxsw_sp1_mac_mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xfc, 0x00 @@ -161,43 +175,6 @@ struct mlxsw_sp_mlxfw_dev { struct mlxsw_sp *mlxsw_sp; }; -struct mlxsw_sp_ptp_ops { - struct mlxsw_sp_ptp_clock * - (*clock_init)(struct mlxsw_sp *mlxsw_sp, struct device *dev); - void (*clock_fini)(struct mlxsw_sp_ptp_clock *clock); - - struct mlxsw_sp_ptp_state *(*init)(struct mlxsw_sp *mlxsw_sp); - void (*fini)(struct mlxsw_sp_ptp_state *ptp_state); - - /* Notify a driver that a packet that might be PTP was received. Driver - * is responsible for freeing the passed-in SKB. - */ - void (*receive)(struct mlxsw_sp *mlxsw_sp, struct sk_buff *skb, - u8 local_port); - - /* Notify a driver that a timestamped packet was transmitted. Driver - * is responsible for freeing the passed-in SKB. - */ - void (*transmitted)(struct mlxsw_sp *mlxsw_sp, struct sk_buff *skb, - u8 local_port); - - int (*hwtstamp_get)(struct mlxsw_sp_port *mlxsw_sp_port, - struct hwtstamp_config *config); - int (*hwtstamp_set)(struct mlxsw_sp_port *mlxsw_sp_port, - struct hwtstamp_config *config); - void (*shaper_work)(struct work_struct *work); - int (*get_ts_info)(struct mlxsw_sp *mlxsw_sp, - struct ethtool_ts_info *info); - int (*get_stats_count)(void); - void (*get_stats_strings)(u8 **p); - void (*get_stats)(struct mlxsw_sp_port *mlxsw_sp_port, - u64 *data, int data_index); -}; - -struct mlxsw_sp_span_ops { - u32 (*buffsize_get)(int mtu, u32 speed); -}; - static int mlxsw_sp_component_query(struct mlxfw_dev *mlxfw_dev, u16 component_index, u32 *p_max_size, u8 *p_align_bits, u16 *p_max_write_size) @@ -580,8 +557,8 @@ static int mlxsw_sp_base_mac_get(struct mlxsw_sp *mlxsw_sp) return 0; } -static int mlxsw_sp_port_admin_status_set(struct mlxsw_sp_port *mlxsw_sp_port, - bool is_up) +int mlxsw_sp_port_admin_status_set(struct mlxsw_sp_port *mlxsw_sp_port, + bool is_up) { struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; char paos_pl[MLXSW_REG_PAOS_LEN]; @@ -995,8 +972,8 @@ int __mlxsw_sp_port_headroom_set(struct mlxsw_sp_port *mlxsw_sp_port, int mtu, return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pbmc), pbmc_pl); } -static int mlxsw_sp_port_headroom_set(struct mlxsw_sp_port *mlxsw_sp_port, - int mtu, bool pause_en) +int mlxsw_sp_port_headroom_set(struct mlxsw_sp_port *mlxsw_sp_port, + int mtu, bool pause_en) { u8 def_prio_tc[IEEE_8021QAZ_MAX_TCS] = {0}; bool dcb_en = !!mlxsw_sp_port->dcb.ets; @@ -1088,8 +1065,8 @@ static int mlxsw_sp_port_get_offload_stats(int attr_id, const struct net_device return -EINVAL; } -static int mlxsw_sp_port_get_stats_raw(struct net_device *dev, int grp, - int prio, char *ppcnt_pl) +int mlxsw_sp_port_get_stats_raw(struct net_device *dev, int grp, + int prio, char *ppcnt_pl) { struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; @@ -1352,6 +1329,21 @@ static int mlxsw_sp_port_kill_vid(struct net_device *dev, return 0; } +static int mlxsw_sp_setup_tc_block(struct mlxsw_sp_port *mlxsw_sp_port, + struct flow_block_offload *f) +{ + switch (f->binder_type) { + case FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS: + return mlxsw_sp_setup_tc_block_clsact(mlxsw_sp_port, f, true); + case FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS: + return mlxsw_sp_setup_tc_block_clsact(mlxsw_sp_port, f, false); + case FLOW_BLOCK_BINDER_TYPE_RED_EARLY_DROP: + return mlxsw_sp_setup_tc_block_qevent_early_drop(mlxsw_sp_port, f); + default: + return -EOPNOTSUPP; + } +} + static int mlxsw_sp_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data) { @@ -1547,1500 +1539,6 @@ static const struct net_device_ops mlxsw_sp_port_netdev_ops = { .ndo_do_ioctl = mlxsw_sp_port_ioctl, }; -static void mlxsw_sp_port_get_drvinfo(struct net_device *dev, - struct ethtool_drvinfo *drvinfo) -{ - struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); - struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; - - strlcpy(drvinfo->driver, mlxsw_sp->bus_info->device_kind, - sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, mlxsw_sp_driver_version, - sizeof(drvinfo->version)); - snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), - "%d.%d.%d", - mlxsw_sp->bus_info->fw_rev.major, - mlxsw_sp->bus_info->fw_rev.minor, - mlxsw_sp->bus_info->fw_rev.subminor); - strlcpy(drvinfo->bus_info, mlxsw_sp->bus_info->device_name, - sizeof(drvinfo->bus_info)); -} - -static void mlxsw_sp_port_get_pauseparam(struct net_device *dev, - struct ethtool_pauseparam *pause) -{ - struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); - - pause->rx_pause = mlxsw_sp_port->link.rx_pause; - pause->tx_pause = mlxsw_sp_port->link.tx_pause; -} - -static int mlxsw_sp_port_pause_set(struct mlxsw_sp_port *mlxsw_sp_port, - struct ethtool_pauseparam *pause) -{ - char pfcc_pl[MLXSW_REG_PFCC_LEN]; - - mlxsw_reg_pfcc_pack(pfcc_pl, mlxsw_sp_port->local_port); - mlxsw_reg_pfcc_pprx_set(pfcc_pl, pause->rx_pause); - mlxsw_reg_pfcc_pptx_set(pfcc_pl, pause->tx_pause); - - return mlxsw_reg_write(mlxsw_sp_port->mlxsw_sp->core, MLXSW_REG(pfcc), - pfcc_pl); -} - -static int mlxsw_sp_port_set_pauseparam(struct net_device *dev, - struct ethtool_pauseparam *pause) -{ - struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); - bool pause_en = pause->tx_pause || pause->rx_pause; - int err; - - if (mlxsw_sp_port->dcb.pfc && mlxsw_sp_port->dcb.pfc->pfc_en) { - netdev_err(dev, "PFC already enabled on port\n"); - return -EINVAL; - } - - if (pause->autoneg) { - netdev_err(dev, "PAUSE frames autonegotiation isn't supported\n"); - return -EINVAL; - } - - err = mlxsw_sp_port_headroom_set(mlxsw_sp_port, dev->mtu, pause_en); - if (err) { - netdev_err(dev, "Failed to configure port's headroom\n"); - return err; - } - - err = mlxsw_sp_port_pause_set(mlxsw_sp_port, pause); - if (err) { - netdev_err(dev, "Failed to set PAUSE parameters\n"); - goto err_port_pause_configure; - } - - mlxsw_sp_port->link.rx_pause = pause->rx_pause; - mlxsw_sp_port->link.tx_pause = pause->tx_pause; - - return 0; - -err_port_pause_configure: - pause_en = mlxsw_sp_port_is_pause_en(mlxsw_sp_port); - mlxsw_sp_port_headroom_set(mlxsw_sp_port, dev->mtu, pause_en); - return err; -} - -struct mlxsw_sp_port_hw_stats { - char str[ETH_GSTRING_LEN]; - u64 (*getter)(const char *payload); - bool cells_bytes; -}; - -static struct mlxsw_sp_port_hw_stats mlxsw_sp_port_hw_stats[] = { - { - .str = "a_frames_transmitted_ok", - .getter = mlxsw_reg_ppcnt_a_frames_transmitted_ok_get, - }, - { - .str = "a_frames_received_ok", - .getter = mlxsw_reg_ppcnt_a_frames_received_ok_get, - }, - { - .str = "a_frame_check_sequence_errors", - .getter = mlxsw_reg_ppcnt_a_frame_check_sequence_errors_get, - }, - { - .str = "a_alignment_errors", - .getter = mlxsw_reg_ppcnt_a_alignment_errors_get, - }, - { - .str = "a_octets_transmitted_ok", - .getter = mlxsw_reg_ppcnt_a_octets_transmitted_ok_get, - }, - { - .str = "a_octets_received_ok", - .getter = mlxsw_reg_ppcnt_a_octets_received_ok_get, - }, - { - .str = "a_multicast_frames_xmitted_ok", - .getter = mlxsw_reg_ppcnt_a_multicast_frames_xmitted_ok_get, - }, - { - .str = "a_broadcast_frames_xmitted_ok", - .getter = mlxsw_reg_ppcnt_a_broadcast_frames_xmitted_ok_get, - }, - { - .str = "a_multicast_frames_received_ok", - .getter = mlxsw_reg_ppcnt_a_multicast_frames_received_ok_get, - }, - { - .str = "a_broadcast_frames_received_ok", - .getter = mlxsw_reg_ppcnt_a_broadcast_frames_received_ok_get, - }, - { - .str = "a_in_range_length_errors", - .getter = mlxsw_reg_ppcnt_a_in_range_length_errors_get, - }, - { - .str = "a_out_of_range_length_field", - .getter = mlxsw_reg_ppcnt_a_out_of_range_length_field_get, - }, - { - .str = "a_frame_too_long_errors", - .getter = mlxsw_reg_ppcnt_a_frame_too_long_errors_get, - }, - { - .str = "a_symbol_error_during_carrier", - .getter = mlxsw_reg_ppcnt_a_symbol_error_during_carrier_get, - }, - { - .str = "a_mac_control_frames_transmitted", - .getter = mlxsw_reg_ppcnt_a_mac_control_frames_transmitted_get, - }, - { - .str = "a_mac_control_frames_received", - .getter = mlxsw_reg_ppcnt_a_mac_control_frames_received_get, - }, - { - .str = "a_unsupported_opcodes_received", - .getter = mlxsw_reg_ppcnt_a_unsupported_opcodes_received_get, - }, - { - .str = "a_pause_mac_ctrl_frames_received", - .getter = mlxsw_reg_ppcnt_a_pause_mac_ctrl_frames_received_get, - }, - { - .str = "a_pause_mac_ctrl_frames_xmitted", - .getter = mlxsw_reg_ppcnt_a_pause_mac_ctrl_frames_transmitted_get, - }, -}; - -#define MLXSW_SP_PORT_HW_STATS_LEN ARRAY_SIZE(mlxsw_sp_port_hw_stats) - -static struct mlxsw_sp_port_hw_stats mlxsw_sp_port_hw_rfc_2863_stats[] = { - { - .str = "if_in_discards", - .getter = mlxsw_reg_ppcnt_if_in_discards_get, - }, - { - .str = "if_out_discards", - .getter = mlxsw_reg_ppcnt_if_out_discards_get, - }, - { - .str = "if_out_errors", - .getter = mlxsw_reg_ppcnt_if_out_errors_get, - }, -}; - -#define MLXSW_SP_PORT_HW_RFC_2863_STATS_LEN \ - ARRAY_SIZE(mlxsw_sp_port_hw_rfc_2863_stats) - -static struct mlxsw_sp_port_hw_stats mlxsw_sp_port_hw_rfc_2819_stats[] = { - { - .str = "ether_stats_undersize_pkts", - .getter = mlxsw_reg_ppcnt_ether_stats_undersize_pkts_get, - }, - { - .str = "ether_stats_oversize_pkts", - .getter = mlxsw_reg_ppcnt_ether_stats_oversize_pkts_get, - }, - { - .str = "ether_stats_fragments", - .getter = mlxsw_reg_ppcnt_ether_stats_fragments_get, - }, - { - .str = "ether_pkts64octets", - .getter = mlxsw_reg_ppcnt_ether_stats_pkts64octets_get, - }, - { - .str = "ether_pkts65to127octets", - .getter = mlxsw_reg_ppcnt_ether_stats_pkts65to127octets_get, - }, - { - .str = "ether_pkts128to255octets", - .getter = mlxsw_reg_ppcnt_ether_stats_pkts128to255octets_get, - }, - { - .str = "ether_pkts256to511octets", - .getter = mlxsw_reg_ppcnt_ether_stats_pkts256to511octets_get, - }, - { - .str = "ether_pkts512to1023octets", - .getter = mlxsw_reg_ppcnt_ether_stats_pkts512to1023octets_get, - }, - { - .str = "ether_pkts1024to1518octets", - .getter = mlxsw_reg_ppcnt_ether_stats_pkts1024to1518octets_get, - }, - { - .str = "ether_pkts1519to2047octets", - .getter = mlxsw_reg_ppcnt_ether_stats_pkts1519to2047octets_get, - }, - { - .str = "ether_pkts2048to4095octets", - .getter = mlxsw_reg_ppcnt_ether_stats_pkts2048to4095octets_get, - }, - { - .str = "ether_pkts4096to8191octets", - .getter = mlxsw_reg_ppcnt_ether_stats_pkts4096to8191octets_get, - }, - { - .str = "ether_pkts8192to10239octets", - .getter = mlxsw_reg_ppcnt_ether_stats_pkts8192to10239octets_get, - }, -}; - -#define MLXSW_SP_PORT_HW_RFC_2819_STATS_LEN \ - ARRAY_SIZE(mlxsw_sp_port_hw_rfc_2819_stats) - -static struct mlxsw_sp_port_hw_stats mlxsw_sp_port_hw_rfc_3635_stats[] = { - { - .str = "dot3stats_fcs_errors", - .getter = mlxsw_reg_ppcnt_dot3stats_fcs_errors_get, - }, - { - .str = "dot3stats_symbol_errors", - .getter = mlxsw_reg_ppcnt_dot3stats_symbol_errors_get, - }, - { - .str = "dot3control_in_unknown_opcodes", - .getter = mlxsw_reg_ppcnt_dot3control_in_unknown_opcodes_get, - }, - { - .str = "dot3in_pause_frames", - .getter = mlxsw_reg_ppcnt_dot3in_pause_frames_get, - }, -}; - -#define MLXSW_SP_PORT_HW_RFC_3635_STATS_LEN \ - ARRAY_SIZE(mlxsw_sp_port_hw_rfc_3635_stats) - -static struct mlxsw_sp_port_hw_stats mlxsw_sp_port_hw_ext_stats[] = { - { - .str = "ecn_marked", - .getter = mlxsw_reg_ppcnt_ecn_marked_get, - }, -}; - -#define MLXSW_SP_PORT_HW_EXT_STATS_LEN ARRAY_SIZE(mlxsw_sp_port_hw_ext_stats) - -static struct mlxsw_sp_port_hw_stats mlxsw_sp_port_hw_discard_stats[] = { - { - .str = "discard_ingress_general", - .getter = mlxsw_reg_ppcnt_ingress_general_get, - }, - { - .str = "discard_ingress_policy_engine", - .getter = mlxsw_reg_ppcnt_ingress_policy_engine_get, - }, - { - .str = "discard_ingress_vlan_membership", - .getter = mlxsw_reg_ppcnt_ingress_vlan_membership_get, - }, - { - .str = "discard_ingress_tag_frame_type", - .getter = mlxsw_reg_ppcnt_ingress_tag_frame_type_get, - }, - { - .str = "discard_egress_vlan_membership", - .getter = mlxsw_reg_ppcnt_egress_vlan_membership_get, - }, - { - .str = "discard_loopback_filter", - .getter = mlxsw_reg_ppcnt_loopback_filter_get, - }, - { - .str = "discard_egress_general", - .getter = mlxsw_reg_ppcnt_egress_general_get, - }, - { - .str = "discard_egress_hoq", - .getter = mlxsw_reg_ppcnt_egress_hoq_get, - }, - { - .str = "discard_egress_policy_engine", - .getter = mlxsw_reg_ppcnt_egress_policy_engine_get, - }, - { - .str = "discard_ingress_tx_link_down", - .getter = mlxsw_reg_ppcnt_ingress_tx_link_down_get, - }, - { - .str = "discard_egress_stp_filter", - .getter = mlxsw_reg_ppcnt_egress_stp_filter_get, - }, - { - .str = "discard_egress_sll", - .getter = mlxsw_reg_ppcnt_egress_sll_get, - }, -}; - -#define MLXSW_SP_PORT_HW_DISCARD_STATS_LEN \ - ARRAY_SIZE(mlxsw_sp_port_hw_discard_stats) - -static struct mlxsw_sp_port_hw_stats mlxsw_sp_port_hw_prio_stats[] = { - { - .str = "rx_octets_prio", - .getter = mlxsw_reg_ppcnt_rx_octets_get, - }, - { - .str = "rx_frames_prio", - .getter = mlxsw_reg_ppcnt_rx_frames_get, - }, - { - .str = "tx_octets_prio", - .getter = mlxsw_reg_ppcnt_tx_octets_get, - }, - { - .str = "tx_frames_prio", - .getter = mlxsw_reg_ppcnt_tx_frames_get, - }, - { - .str = "rx_pause_prio", - .getter = mlxsw_reg_ppcnt_rx_pause_get, - }, - { - .str = "rx_pause_duration_prio", - .getter = mlxsw_reg_ppcnt_rx_pause_duration_get, - }, - { - .str = "tx_pause_prio", - .getter = mlxsw_reg_ppcnt_tx_pause_get, - }, - { - .str = "tx_pause_duration_prio", - .getter = mlxsw_reg_ppcnt_tx_pause_duration_get, - }, -}; - -#define MLXSW_SP_PORT_HW_PRIO_STATS_LEN ARRAY_SIZE(mlxsw_sp_port_hw_prio_stats) - -static struct mlxsw_sp_port_hw_stats mlxsw_sp_port_hw_tc_stats[] = { - { - .str = "tc_transmit_queue_tc", - .getter = mlxsw_reg_ppcnt_tc_transmit_queue_get, - .cells_bytes = true, - }, - { - .str = "tc_no_buffer_discard_uc_tc", - .getter = mlxsw_reg_ppcnt_tc_no_buffer_discard_uc_get, - }, -}; - -#define MLXSW_SP_PORT_HW_TC_STATS_LEN ARRAY_SIZE(mlxsw_sp_port_hw_tc_stats) - -#define MLXSW_SP_PORT_ETHTOOL_STATS_LEN (MLXSW_SP_PORT_HW_STATS_LEN + \ - MLXSW_SP_PORT_HW_RFC_2863_STATS_LEN + \ - MLXSW_SP_PORT_HW_RFC_2819_STATS_LEN + \ - MLXSW_SP_PORT_HW_RFC_3635_STATS_LEN + \ - MLXSW_SP_PORT_HW_EXT_STATS_LEN + \ - MLXSW_SP_PORT_HW_DISCARD_STATS_LEN + \ - (MLXSW_SP_PORT_HW_PRIO_STATS_LEN * \ - IEEE_8021QAZ_MAX_TCS) + \ - (MLXSW_SP_PORT_HW_TC_STATS_LEN * \ - TC_MAX_QUEUE)) - -static void mlxsw_sp_port_get_prio_strings(u8 **p, int prio) -{ - int i; - - for (i = 0; i < MLXSW_SP_PORT_HW_PRIO_STATS_LEN; i++) { - snprintf(*p, ETH_GSTRING_LEN, "%.29s_%.1d", - mlxsw_sp_port_hw_prio_stats[i].str, prio); - *p += ETH_GSTRING_LEN; - } -} - -static void mlxsw_sp_port_get_tc_strings(u8 **p, int tc) -{ - int i; - - for (i = 0; i < MLXSW_SP_PORT_HW_TC_STATS_LEN; i++) { - snprintf(*p, ETH_GSTRING_LEN, "%.29s_%.1d", - mlxsw_sp_port_hw_tc_stats[i].str, tc); - *p += ETH_GSTRING_LEN; - } -} - -static void mlxsw_sp_port_get_strings(struct net_device *dev, - u32 stringset, u8 *data) -{ - struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); - u8 *p = data; - int i; - - switch (stringset) { - case ETH_SS_STATS: - for (i = 0; i < MLXSW_SP_PORT_HW_STATS_LEN; i++) { - memcpy(p, mlxsw_sp_port_hw_stats[i].str, - ETH_GSTRING_LEN); - p += ETH_GSTRING_LEN; - } - - for (i = 0; i < MLXSW_SP_PORT_HW_RFC_2863_STATS_LEN; i++) { - memcpy(p, mlxsw_sp_port_hw_rfc_2863_stats[i].str, - ETH_GSTRING_LEN); - p += ETH_GSTRING_LEN; - } - - for (i = 0; i < MLXSW_SP_PORT_HW_RFC_2819_STATS_LEN; i++) { - memcpy(p, mlxsw_sp_port_hw_rfc_2819_stats[i].str, - ETH_GSTRING_LEN); - p += ETH_GSTRING_LEN; - } - - for (i = 0; i < MLXSW_SP_PORT_HW_RFC_3635_STATS_LEN; i++) { - memcpy(p, mlxsw_sp_port_hw_rfc_3635_stats[i].str, - ETH_GSTRING_LEN); - p += ETH_GSTRING_LEN; - } - - for (i = 0; i < MLXSW_SP_PORT_HW_EXT_STATS_LEN; i++) { - memcpy(p, mlxsw_sp_port_hw_ext_stats[i].str, - ETH_GSTRING_LEN); - p += ETH_GSTRING_LEN; - } - - for (i = 0; i < MLXSW_SP_PORT_HW_DISCARD_STATS_LEN; i++) { - memcpy(p, mlxsw_sp_port_hw_discard_stats[i].str, - ETH_GSTRING_LEN); - p += ETH_GSTRING_LEN; - } - - for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) - mlxsw_sp_port_get_prio_strings(&p, i); - - for (i = 0; i < TC_MAX_QUEUE; i++) - mlxsw_sp_port_get_tc_strings(&p, i); - - mlxsw_sp_port->mlxsw_sp->ptp_ops->get_stats_strings(&p); - break; - } -} - -static int mlxsw_sp_port_set_phys_id(struct net_device *dev, - enum ethtool_phys_id_state state) -{ - struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); - struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; - char mlcr_pl[MLXSW_REG_MLCR_LEN]; - bool active; - - switch (state) { - case ETHTOOL_ID_ACTIVE: - active = true; - break; - case ETHTOOL_ID_INACTIVE: - active = false; - break; - default: - return -EOPNOTSUPP; - } - - mlxsw_reg_mlcr_pack(mlcr_pl, mlxsw_sp_port->local_port, active); - return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(mlcr), mlcr_pl); -} - -static int -mlxsw_sp_get_hw_stats_by_group(struct mlxsw_sp_port_hw_stats **p_hw_stats, - int *p_len, enum mlxsw_reg_ppcnt_grp grp) -{ - switch (grp) { - case MLXSW_REG_PPCNT_IEEE_8023_CNT: - *p_hw_stats = mlxsw_sp_port_hw_stats; - *p_len = MLXSW_SP_PORT_HW_STATS_LEN; - break; - case MLXSW_REG_PPCNT_RFC_2863_CNT: - *p_hw_stats = mlxsw_sp_port_hw_rfc_2863_stats; - *p_len = MLXSW_SP_PORT_HW_RFC_2863_STATS_LEN; - break; - case MLXSW_REG_PPCNT_RFC_2819_CNT: - *p_hw_stats = mlxsw_sp_port_hw_rfc_2819_stats; - *p_len = MLXSW_SP_PORT_HW_RFC_2819_STATS_LEN; - break; - case MLXSW_REG_PPCNT_RFC_3635_CNT: - *p_hw_stats = mlxsw_sp_port_hw_rfc_3635_stats; - *p_len = MLXSW_SP_PORT_HW_RFC_3635_STATS_LEN; - break; - case MLXSW_REG_PPCNT_EXT_CNT: - *p_hw_stats = mlxsw_sp_port_hw_ext_stats; - *p_len = MLXSW_SP_PORT_HW_EXT_STATS_LEN; - break; - case MLXSW_REG_PPCNT_DISCARD_CNT: - *p_hw_stats = mlxsw_sp_port_hw_discard_stats; - *p_len = MLXSW_SP_PORT_HW_DISCARD_STATS_LEN; - break; - case MLXSW_REG_PPCNT_PRIO_CNT: - *p_hw_stats = mlxsw_sp_port_hw_prio_stats; - *p_len = MLXSW_SP_PORT_HW_PRIO_STATS_LEN; - break; - case MLXSW_REG_PPCNT_TC_CNT: - *p_hw_stats = mlxsw_sp_port_hw_tc_stats; - *p_len = MLXSW_SP_PORT_HW_TC_STATS_LEN; - break; - default: - WARN_ON(1); - return -EOPNOTSUPP; - } - return 0; -} - -static void __mlxsw_sp_port_get_stats(struct net_device *dev, - enum mlxsw_reg_ppcnt_grp grp, int prio, - u64 *data, int data_index) -{ - struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); - struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; - struct mlxsw_sp_port_hw_stats *hw_stats; - char ppcnt_pl[MLXSW_REG_PPCNT_LEN]; - int i, len; - int err; - - err = mlxsw_sp_get_hw_stats_by_group(&hw_stats, &len, grp); - if (err) - return; - mlxsw_sp_port_get_stats_raw(dev, grp, prio, ppcnt_pl); - for (i = 0; i < len; i++) { - data[data_index + i] = hw_stats[i].getter(ppcnt_pl); - if (!hw_stats[i].cells_bytes) - continue; - data[data_index + i] = mlxsw_sp_cells_bytes(mlxsw_sp, - data[data_index + i]); - } -} - -static void mlxsw_sp_port_get_stats(struct net_device *dev, - struct ethtool_stats *stats, u64 *data) -{ - struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); - int i, data_index = 0; - - /* IEEE 802.3 Counters */ - __mlxsw_sp_port_get_stats(dev, MLXSW_REG_PPCNT_IEEE_8023_CNT, 0, - data, data_index); - data_index = MLXSW_SP_PORT_HW_STATS_LEN; - - /* RFC 2863 Counters */ - __mlxsw_sp_port_get_stats(dev, MLXSW_REG_PPCNT_RFC_2863_CNT, 0, - data, data_index); - data_index += MLXSW_SP_PORT_HW_RFC_2863_STATS_LEN; - - /* RFC 2819 Counters */ - __mlxsw_sp_port_get_stats(dev, MLXSW_REG_PPCNT_RFC_2819_CNT, 0, - data, data_index); - data_index += MLXSW_SP_PORT_HW_RFC_2819_STATS_LEN; - - /* RFC 3635 Counters */ - __mlxsw_sp_port_get_stats(dev, MLXSW_REG_PPCNT_RFC_3635_CNT, 0, - data, data_index); - data_index += MLXSW_SP_PORT_HW_RFC_3635_STATS_LEN; - - /* Extended Counters */ - __mlxsw_sp_port_get_stats(dev, MLXSW_REG_PPCNT_EXT_CNT, 0, - data, data_index); - data_index += MLXSW_SP_PORT_HW_EXT_STATS_LEN; - - /* Discard Counters */ - __mlxsw_sp_port_get_stats(dev, MLXSW_REG_PPCNT_DISCARD_CNT, 0, - data, data_index); - data_index += MLXSW_SP_PORT_HW_DISCARD_STATS_LEN; - - /* Per-Priority Counters */ - for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) { - __mlxsw_sp_port_get_stats(dev, MLXSW_REG_PPCNT_PRIO_CNT, i, - data, data_index); - data_index += MLXSW_SP_PORT_HW_PRIO_STATS_LEN; - } - - /* Per-TC Counters */ - for (i = 0; i < TC_MAX_QUEUE; i++) { - __mlxsw_sp_port_get_stats(dev, MLXSW_REG_PPCNT_TC_CNT, i, - data, data_index); - data_index += MLXSW_SP_PORT_HW_TC_STATS_LEN; - } - - /* PTP counters */ - mlxsw_sp_port->mlxsw_sp->ptp_ops->get_stats(mlxsw_sp_port, - data, data_index); - data_index += mlxsw_sp_port->mlxsw_sp->ptp_ops->get_stats_count(); -} - -static int mlxsw_sp_port_get_sset_count(struct net_device *dev, int sset) -{ - struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); - - switch (sset) { - case ETH_SS_STATS: - return MLXSW_SP_PORT_ETHTOOL_STATS_LEN + - mlxsw_sp_port->mlxsw_sp->ptp_ops->get_stats_count(); - default: - return -EOPNOTSUPP; - } -} - -struct mlxsw_sp1_port_link_mode { - enum ethtool_link_mode_bit_indices mask_ethtool; - u32 mask; - u32 speed; -}; - -static const struct mlxsw_sp1_port_link_mode mlxsw_sp1_port_link_mode[] = { - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_100BASE_T, - .mask_ethtool = ETHTOOL_LINK_MODE_100baseT_Full_BIT, - .speed = SPEED_100, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_SGMII | - MLXSW_REG_PTYS_ETH_SPEED_1000BASE_KX, - .mask_ethtool = ETHTOOL_LINK_MODE_1000baseKX_Full_BIT, - .speed = SPEED_1000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_10GBASE_T, - .mask_ethtool = ETHTOOL_LINK_MODE_10000baseT_Full_BIT, - .speed = SPEED_10000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_10GBASE_CX4 | - MLXSW_REG_PTYS_ETH_SPEED_10GBASE_KX4, - .mask_ethtool = ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT, - .speed = SPEED_10000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_10GBASE_KR | - MLXSW_REG_PTYS_ETH_SPEED_10GBASE_CR | - MLXSW_REG_PTYS_ETH_SPEED_10GBASE_SR | - MLXSW_REG_PTYS_ETH_SPEED_10GBASE_ER_LR, - .mask_ethtool = ETHTOOL_LINK_MODE_10000baseKR_Full_BIT, - .speed = SPEED_10000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_20GBASE_KR2, - .mask_ethtool = ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT, - .speed = SPEED_20000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_40GBASE_CR4, - .mask_ethtool = ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT, - .speed = SPEED_40000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_40GBASE_KR4, - .mask_ethtool = ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT, - .speed = SPEED_40000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_40GBASE_SR4, - .mask_ethtool = ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT, - .speed = SPEED_40000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_40GBASE_LR4_ER4, - .mask_ethtool = ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT, - .speed = SPEED_40000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_25GBASE_CR, - .mask_ethtool = ETHTOOL_LINK_MODE_25000baseCR_Full_BIT, - .speed = SPEED_25000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_25GBASE_KR, - .mask_ethtool = ETHTOOL_LINK_MODE_25000baseKR_Full_BIT, - .speed = SPEED_25000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_25GBASE_SR, - .mask_ethtool = ETHTOOL_LINK_MODE_25000baseSR_Full_BIT, - .speed = SPEED_25000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_50GBASE_CR2, - .mask_ethtool = ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT, - .speed = SPEED_50000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_50GBASE_KR2, - .mask_ethtool = ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT, - .speed = SPEED_50000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_50GBASE_SR2, - .mask_ethtool = ETHTOOL_LINK_MODE_50000baseSR2_Full_BIT, - .speed = SPEED_50000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_100GBASE_CR4, - .mask_ethtool = ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT, - .speed = SPEED_100000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_100GBASE_SR4, - .mask_ethtool = ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT, - .speed = SPEED_100000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_100GBASE_KR4, - .mask_ethtool = ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT, - .speed = SPEED_100000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_100GBASE_LR4_ER4, - .mask_ethtool = ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT, - .speed = SPEED_100000, - }, -}; - -#define MLXSW_SP1_PORT_LINK_MODE_LEN ARRAY_SIZE(mlxsw_sp1_port_link_mode) - -static void -mlxsw_sp1_from_ptys_supported_port(struct mlxsw_sp *mlxsw_sp, - u32 ptys_eth_proto, - struct ethtool_link_ksettings *cmd) -{ - if (ptys_eth_proto & (MLXSW_REG_PTYS_ETH_SPEED_10GBASE_CR | - MLXSW_REG_PTYS_ETH_SPEED_10GBASE_SR | - MLXSW_REG_PTYS_ETH_SPEED_40GBASE_CR4 | - MLXSW_REG_PTYS_ETH_SPEED_40GBASE_SR4 | - MLXSW_REG_PTYS_ETH_SPEED_100GBASE_SR4 | - MLXSW_REG_PTYS_ETH_SPEED_SGMII)) - ethtool_link_ksettings_add_link_mode(cmd, supported, FIBRE); - - if (ptys_eth_proto & (MLXSW_REG_PTYS_ETH_SPEED_10GBASE_KR | - MLXSW_REG_PTYS_ETH_SPEED_10GBASE_KX4 | - MLXSW_REG_PTYS_ETH_SPEED_40GBASE_KR4 | - MLXSW_REG_PTYS_ETH_SPEED_100GBASE_KR4 | - MLXSW_REG_PTYS_ETH_SPEED_1000BASE_KX)) - ethtool_link_ksettings_add_link_mode(cmd, supported, Backplane); -} - -static void -mlxsw_sp1_from_ptys_link(struct mlxsw_sp *mlxsw_sp, u32 ptys_eth_proto, - u8 width, unsigned long *mode) -{ - int i; - - for (i = 0; i < MLXSW_SP1_PORT_LINK_MODE_LEN; i++) { - if (ptys_eth_proto & mlxsw_sp1_port_link_mode[i].mask) - __set_bit(mlxsw_sp1_port_link_mode[i].mask_ethtool, - mode); - } -} - -static u32 -mlxsw_sp1_from_ptys_speed(struct mlxsw_sp *mlxsw_sp, u32 ptys_eth_proto) -{ - int i; - - for (i = 0; i < MLXSW_SP1_PORT_LINK_MODE_LEN; i++) { - if (ptys_eth_proto & mlxsw_sp1_port_link_mode[i].mask) - return mlxsw_sp1_port_link_mode[i].speed; - } - - return SPEED_UNKNOWN; -} - -static void -mlxsw_sp1_from_ptys_speed_duplex(struct mlxsw_sp *mlxsw_sp, bool carrier_ok, - u32 ptys_eth_proto, - struct ethtool_link_ksettings *cmd) -{ - cmd->base.speed = SPEED_UNKNOWN; - cmd->base.duplex = DUPLEX_UNKNOWN; - - if (!carrier_ok) - return; - - cmd->base.speed = mlxsw_sp1_from_ptys_speed(mlxsw_sp, ptys_eth_proto); - if (cmd->base.speed != SPEED_UNKNOWN) - cmd->base.duplex = DUPLEX_FULL; -} - -static u32 -mlxsw_sp1_to_ptys_advert_link(struct mlxsw_sp *mlxsw_sp, u8 width, - const struct ethtool_link_ksettings *cmd) -{ - u32 ptys_proto = 0; - int i; - - for (i = 0; i < MLXSW_SP1_PORT_LINK_MODE_LEN; i++) { - if (test_bit(mlxsw_sp1_port_link_mode[i].mask_ethtool, - cmd->link_modes.advertising)) - ptys_proto |= mlxsw_sp1_port_link_mode[i].mask; - } - return ptys_proto; -} - -static u32 mlxsw_sp1_to_ptys_speed(struct mlxsw_sp *mlxsw_sp, u8 width, - u32 speed) -{ - u32 ptys_proto = 0; - int i; - - for (i = 0; i < MLXSW_SP1_PORT_LINK_MODE_LEN; i++) { - if (speed == mlxsw_sp1_port_link_mode[i].speed) - ptys_proto |= mlxsw_sp1_port_link_mode[i].mask; - } - return ptys_proto; -} - -static void -mlxsw_sp1_reg_ptys_eth_pack(struct mlxsw_sp *mlxsw_sp, char *payload, - u8 local_port, u32 proto_admin, bool autoneg) -{ - mlxsw_reg_ptys_eth_pack(payload, local_port, proto_admin, autoneg); -} - -static void -mlxsw_sp1_reg_ptys_eth_unpack(struct mlxsw_sp *mlxsw_sp, char *payload, - u32 *p_eth_proto_cap, u32 *p_eth_proto_admin, - u32 *p_eth_proto_oper) -{ - mlxsw_reg_ptys_eth_unpack(payload, p_eth_proto_cap, p_eth_proto_admin, - p_eth_proto_oper); -} - -static const struct mlxsw_sp_port_type_speed_ops -mlxsw_sp1_port_type_speed_ops = { - .from_ptys_supported_port = mlxsw_sp1_from_ptys_supported_port, - .from_ptys_link = mlxsw_sp1_from_ptys_link, - .from_ptys_speed = mlxsw_sp1_from_ptys_speed, - .from_ptys_speed_duplex = mlxsw_sp1_from_ptys_speed_duplex, - .to_ptys_advert_link = mlxsw_sp1_to_ptys_advert_link, - .to_ptys_speed = mlxsw_sp1_to_ptys_speed, - .reg_ptys_eth_pack = mlxsw_sp1_reg_ptys_eth_pack, - .reg_ptys_eth_unpack = mlxsw_sp1_reg_ptys_eth_unpack, -}; - -static const enum ethtool_link_mode_bit_indices -mlxsw_sp2_mask_ethtool_sgmii_100m[] = { - ETHTOOL_LINK_MODE_100baseT_Full_BIT, -}; - -#define MLXSW_SP2_MASK_ETHTOOL_SGMII_100M_LEN \ - ARRAY_SIZE(mlxsw_sp2_mask_ethtool_sgmii_100m) - -static const enum ethtool_link_mode_bit_indices -mlxsw_sp2_mask_ethtool_1000base_x_sgmii[] = { - ETHTOOL_LINK_MODE_1000baseT_Full_BIT, - ETHTOOL_LINK_MODE_1000baseKX_Full_BIT, -}; - -#define MLXSW_SP2_MASK_ETHTOOL_1000BASE_X_SGMII_LEN \ - ARRAY_SIZE(mlxsw_sp2_mask_ethtool_1000base_x_sgmii) - -static const enum ethtool_link_mode_bit_indices -mlxsw_sp2_mask_ethtool_2_5gbase_x_2_5gmii[] = { - ETHTOOL_LINK_MODE_2500baseX_Full_BIT, -}; - -#define MLXSW_SP2_MASK_ETHTOOL_2_5GBASE_X_2_5GMII_LEN \ - ARRAY_SIZE(mlxsw_sp2_mask_ethtool_2_5gbase_x_2_5gmii) - -static const enum ethtool_link_mode_bit_indices -mlxsw_sp2_mask_ethtool_5gbase_r[] = { - ETHTOOL_LINK_MODE_5000baseT_Full_BIT, -}; - -#define MLXSW_SP2_MASK_ETHTOOL_5GBASE_R_LEN \ - ARRAY_SIZE(mlxsw_sp2_mask_ethtool_5gbase_r) - -static const enum ethtool_link_mode_bit_indices -mlxsw_sp2_mask_ethtool_xfi_xaui_1_10g[] = { - ETHTOOL_LINK_MODE_10000baseT_Full_BIT, - ETHTOOL_LINK_MODE_10000baseKR_Full_BIT, - ETHTOOL_LINK_MODE_10000baseR_FEC_BIT, - ETHTOOL_LINK_MODE_10000baseCR_Full_BIT, - ETHTOOL_LINK_MODE_10000baseSR_Full_BIT, - ETHTOOL_LINK_MODE_10000baseLR_Full_BIT, - ETHTOOL_LINK_MODE_10000baseER_Full_BIT, -}; - -#define MLXSW_SP2_MASK_ETHTOOL_XFI_XAUI_1_10G_LEN \ - ARRAY_SIZE(mlxsw_sp2_mask_ethtool_xfi_xaui_1_10g) - -static const enum ethtool_link_mode_bit_indices -mlxsw_sp2_mask_ethtool_xlaui_4_xlppi_4_40g[] = { - ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT, - ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT, - ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT, - ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT, -}; - -#define MLXSW_SP2_MASK_ETHTOOL_XLAUI_4_XLPPI_4_40G_LEN \ - ARRAY_SIZE(mlxsw_sp2_mask_ethtool_xlaui_4_xlppi_4_40g) - -static const enum ethtool_link_mode_bit_indices -mlxsw_sp2_mask_ethtool_25gaui_1_25gbase_cr_kr[] = { - ETHTOOL_LINK_MODE_25000baseCR_Full_BIT, - ETHTOOL_LINK_MODE_25000baseKR_Full_BIT, - ETHTOOL_LINK_MODE_25000baseSR_Full_BIT, -}; - -#define MLXSW_SP2_MASK_ETHTOOL_25GAUI_1_25GBASE_CR_KR_LEN \ - ARRAY_SIZE(mlxsw_sp2_mask_ethtool_25gaui_1_25gbase_cr_kr) - -static const enum ethtool_link_mode_bit_indices -mlxsw_sp2_mask_ethtool_50gaui_2_laui_2_50gbase_cr2_kr2[] = { - ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT, - ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT, - ETHTOOL_LINK_MODE_50000baseSR2_Full_BIT, -}; - -#define MLXSW_SP2_MASK_ETHTOOL_50GAUI_2_LAUI_2_50GBASE_CR2_KR2_LEN \ - ARRAY_SIZE(mlxsw_sp2_mask_ethtool_50gaui_2_laui_2_50gbase_cr2_kr2) - -static const enum ethtool_link_mode_bit_indices -mlxsw_sp2_mask_ethtool_50gaui_1_laui_1_50gbase_cr_kr[] = { - ETHTOOL_LINK_MODE_50000baseKR_Full_BIT, - ETHTOOL_LINK_MODE_50000baseSR_Full_BIT, - ETHTOOL_LINK_MODE_50000baseCR_Full_BIT, - ETHTOOL_LINK_MODE_50000baseLR_ER_FR_Full_BIT, - ETHTOOL_LINK_MODE_50000baseDR_Full_BIT, -}; - -#define MLXSW_SP2_MASK_ETHTOOL_50GAUI_1_LAUI_1_50GBASE_CR_KR_LEN \ - ARRAY_SIZE(mlxsw_sp2_mask_ethtool_50gaui_1_laui_1_50gbase_cr_kr) - -static const enum ethtool_link_mode_bit_indices -mlxsw_sp2_mask_ethtool_caui_4_100gbase_cr4_kr4[] = { - ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT, - ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT, - ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT, - ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT, -}; - -#define MLXSW_SP2_MASK_ETHTOOL_CAUI_4_100GBASE_CR4_KR4_LEN \ - ARRAY_SIZE(mlxsw_sp2_mask_ethtool_caui_4_100gbase_cr4_kr4) - -static const enum ethtool_link_mode_bit_indices -mlxsw_sp2_mask_ethtool_100gaui_2_100gbase_cr2_kr2[] = { - ETHTOOL_LINK_MODE_100000baseKR2_Full_BIT, - ETHTOOL_LINK_MODE_100000baseSR2_Full_BIT, - ETHTOOL_LINK_MODE_100000baseCR2_Full_BIT, - ETHTOOL_LINK_MODE_100000baseLR2_ER2_FR2_Full_BIT, - ETHTOOL_LINK_MODE_100000baseDR2_Full_BIT, -}; - -#define MLXSW_SP2_MASK_ETHTOOL_100GAUI_2_100GBASE_CR2_KR2_LEN \ - ARRAY_SIZE(mlxsw_sp2_mask_ethtool_100gaui_2_100gbase_cr2_kr2) - -static const enum ethtool_link_mode_bit_indices -mlxsw_sp2_mask_ethtool_200gaui_4_200gbase_cr4_kr4[] = { - ETHTOOL_LINK_MODE_200000baseKR4_Full_BIT, - ETHTOOL_LINK_MODE_200000baseSR4_Full_BIT, - ETHTOOL_LINK_MODE_200000baseLR4_ER4_FR4_Full_BIT, - ETHTOOL_LINK_MODE_200000baseDR4_Full_BIT, - ETHTOOL_LINK_MODE_200000baseCR4_Full_BIT, -}; - -#define MLXSW_SP2_MASK_ETHTOOL_200GAUI_4_200GBASE_CR4_KR4_LEN \ - ARRAY_SIZE(mlxsw_sp2_mask_ethtool_200gaui_4_200gbase_cr4_kr4) - -static const enum ethtool_link_mode_bit_indices -mlxsw_sp2_mask_ethtool_400gaui_8[] = { - ETHTOOL_LINK_MODE_400000baseKR8_Full_BIT, - ETHTOOL_LINK_MODE_400000baseSR8_Full_BIT, - ETHTOOL_LINK_MODE_400000baseLR8_ER8_FR8_Full_BIT, - ETHTOOL_LINK_MODE_400000baseDR8_Full_BIT, - ETHTOOL_LINK_MODE_400000baseCR8_Full_BIT, -}; - -#define MLXSW_SP2_MASK_ETHTOOL_400GAUI_8_LEN \ - ARRAY_SIZE(mlxsw_sp2_mask_ethtool_400gaui_8) - -#define MLXSW_SP_PORT_MASK_WIDTH_1X BIT(0) -#define MLXSW_SP_PORT_MASK_WIDTH_2X BIT(1) -#define MLXSW_SP_PORT_MASK_WIDTH_4X BIT(2) -#define MLXSW_SP_PORT_MASK_WIDTH_8X BIT(3) - -static u8 mlxsw_sp_port_mask_width_get(u8 width) -{ - switch (width) { - case 1: - return MLXSW_SP_PORT_MASK_WIDTH_1X; - case 2: - return MLXSW_SP_PORT_MASK_WIDTH_2X; - case 4: - return MLXSW_SP_PORT_MASK_WIDTH_4X; - case 8: - return MLXSW_SP_PORT_MASK_WIDTH_8X; - default: - WARN_ON_ONCE(1); - return 0; - } -} - -struct mlxsw_sp2_port_link_mode { - const enum ethtool_link_mode_bit_indices *mask_ethtool; - int m_ethtool_len; - u32 mask; - u32 speed; - u8 mask_width; -}; - -static const struct mlxsw_sp2_port_link_mode mlxsw_sp2_port_link_mode[] = { - { - .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_SGMII_100M, - .mask_ethtool = mlxsw_sp2_mask_ethtool_sgmii_100m, - .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_SGMII_100M_LEN, - .mask_width = MLXSW_SP_PORT_MASK_WIDTH_1X | - MLXSW_SP_PORT_MASK_WIDTH_2X | - MLXSW_SP_PORT_MASK_WIDTH_4X | - MLXSW_SP_PORT_MASK_WIDTH_8X, - .speed = SPEED_100, - }, - { - .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_1000BASE_X_SGMII, - .mask_ethtool = mlxsw_sp2_mask_ethtool_1000base_x_sgmii, - .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_1000BASE_X_SGMII_LEN, - .mask_width = MLXSW_SP_PORT_MASK_WIDTH_1X | - MLXSW_SP_PORT_MASK_WIDTH_2X | - MLXSW_SP_PORT_MASK_WIDTH_4X | - MLXSW_SP_PORT_MASK_WIDTH_8X, - .speed = SPEED_1000, - }, - { - .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_2_5GBASE_X_2_5GMII, - .mask_ethtool = mlxsw_sp2_mask_ethtool_2_5gbase_x_2_5gmii, - .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_2_5GBASE_X_2_5GMII_LEN, - .mask_width = MLXSW_SP_PORT_MASK_WIDTH_1X | - MLXSW_SP_PORT_MASK_WIDTH_2X | - MLXSW_SP_PORT_MASK_WIDTH_4X | - MLXSW_SP_PORT_MASK_WIDTH_8X, - .speed = SPEED_2500, - }, - { - .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_5GBASE_R, - .mask_ethtool = mlxsw_sp2_mask_ethtool_5gbase_r, - .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_5GBASE_R_LEN, - .mask_width = MLXSW_SP_PORT_MASK_WIDTH_1X | - MLXSW_SP_PORT_MASK_WIDTH_2X | - MLXSW_SP_PORT_MASK_WIDTH_4X | - MLXSW_SP_PORT_MASK_WIDTH_8X, - .speed = SPEED_5000, - }, - { - .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_XFI_XAUI_1_10G, - .mask_ethtool = mlxsw_sp2_mask_ethtool_xfi_xaui_1_10g, - .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_XFI_XAUI_1_10G_LEN, - .mask_width = MLXSW_SP_PORT_MASK_WIDTH_1X | - MLXSW_SP_PORT_MASK_WIDTH_2X | - MLXSW_SP_PORT_MASK_WIDTH_4X | - MLXSW_SP_PORT_MASK_WIDTH_8X, - .speed = SPEED_10000, - }, - { - .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_XLAUI_4_XLPPI_4_40G, - .mask_ethtool = mlxsw_sp2_mask_ethtool_xlaui_4_xlppi_4_40g, - .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_XLAUI_4_XLPPI_4_40G_LEN, - .mask_width = MLXSW_SP_PORT_MASK_WIDTH_4X | - MLXSW_SP_PORT_MASK_WIDTH_8X, - .speed = SPEED_40000, - }, - { - .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_25GAUI_1_25GBASE_CR_KR, - .mask_ethtool = mlxsw_sp2_mask_ethtool_25gaui_1_25gbase_cr_kr, - .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_25GAUI_1_25GBASE_CR_KR_LEN, - .mask_width = MLXSW_SP_PORT_MASK_WIDTH_1X | - MLXSW_SP_PORT_MASK_WIDTH_2X | - MLXSW_SP_PORT_MASK_WIDTH_4X | - MLXSW_SP_PORT_MASK_WIDTH_8X, - .speed = SPEED_25000, - }, - { - .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_50GAUI_2_LAUI_2_50GBASE_CR2_KR2, - .mask_ethtool = mlxsw_sp2_mask_ethtool_50gaui_2_laui_2_50gbase_cr2_kr2, - .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_50GAUI_2_LAUI_2_50GBASE_CR2_KR2_LEN, - .mask_width = MLXSW_SP_PORT_MASK_WIDTH_2X | - MLXSW_SP_PORT_MASK_WIDTH_4X | - MLXSW_SP_PORT_MASK_WIDTH_8X, - .speed = SPEED_50000, - }, - { - .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_50GAUI_1_LAUI_1_50GBASE_CR_KR, - .mask_ethtool = mlxsw_sp2_mask_ethtool_50gaui_1_laui_1_50gbase_cr_kr, - .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_50GAUI_1_LAUI_1_50GBASE_CR_KR_LEN, - .mask_width = MLXSW_SP_PORT_MASK_WIDTH_1X, - .speed = SPEED_50000, - }, - { - .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_CAUI_4_100GBASE_CR4_KR4, - .mask_ethtool = mlxsw_sp2_mask_ethtool_caui_4_100gbase_cr4_kr4, - .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_CAUI_4_100GBASE_CR4_KR4_LEN, - .mask_width = MLXSW_SP_PORT_MASK_WIDTH_4X | - MLXSW_SP_PORT_MASK_WIDTH_8X, - .speed = SPEED_100000, - }, - { - .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_100GAUI_2_100GBASE_CR2_KR2, - .mask_ethtool = mlxsw_sp2_mask_ethtool_100gaui_2_100gbase_cr2_kr2, - .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_100GAUI_2_100GBASE_CR2_KR2_LEN, - .mask_width = MLXSW_SP_PORT_MASK_WIDTH_2X, - .speed = SPEED_100000, - }, - { - .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_200GAUI_4_200GBASE_CR4_KR4, - .mask_ethtool = mlxsw_sp2_mask_ethtool_200gaui_4_200gbase_cr4_kr4, - .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_200GAUI_4_200GBASE_CR4_KR4_LEN, - .mask_width = MLXSW_SP_PORT_MASK_WIDTH_4X | - MLXSW_SP_PORT_MASK_WIDTH_8X, - .speed = SPEED_200000, - }, - { - .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_400GAUI_8, - .mask_ethtool = mlxsw_sp2_mask_ethtool_400gaui_8, - .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_400GAUI_8_LEN, - .mask_width = MLXSW_SP_PORT_MASK_WIDTH_8X, - .speed = SPEED_400000, - }, -}; - -#define MLXSW_SP2_PORT_LINK_MODE_LEN ARRAY_SIZE(mlxsw_sp2_port_link_mode) - -static void -mlxsw_sp2_from_ptys_supported_port(struct mlxsw_sp *mlxsw_sp, - u32 ptys_eth_proto, - struct ethtool_link_ksettings *cmd) -{ - ethtool_link_ksettings_add_link_mode(cmd, supported, FIBRE); - ethtool_link_ksettings_add_link_mode(cmd, supported, Backplane); -} - -static void -mlxsw_sp2_set_bit_ethtool(const struct mlxsw_sp2_port_link_mode *link_mode, - unsigned long *mode) -{ - int i; - - for (i = 0; i < link_mode->m_ethtool_len; i++) - __set_bit(link_mode->mask_ethtool[i], mode); -} - -static void -mlxsw_sp2_from_ptys_link(struct mlxsw_sp *mlxsw_sp, u32 ptys_eth_proto, - u8 width, unsigned long *mode) -{ - u8 mask_width = mlxsw_sp_port_mask_width_get(width); - int i; - - for (i = 0; i < MLXSW_SP2_PORT_LINK_MODE_LEN; i++) { - if ((ptys_eth_proto & mlxsw_sp2_port_link_mode[i].mask) && - (mask_width & mlxsw_sp2_port_link_mode[i].mask_width)) - mlxsw_sp2_set_bit_ethtool(&mlxsw_sp2_port_link_mode[i], - mode); - } -} - -static u32 -mlxsw_sp2_from_ptys_speed(struct mlxsw_sp *mlxsw_sp, u32 ptys_eth_proto) -{ - int i; - - for (i = 0; i < MLXSW_SP2_PORT_LINK_MODE_LEN; i++) { - if (ptys_eth_proto & mlxsw_sp2_port_link_mode[i].mask) - return mlxsw_sp2_port_link_mode[i].speed; - } - - return SPEED_UNKNOWN; -} - -static void -mlxsw_sp2_from_ptys_speed_duplex(struct mlxsw_sp *mlxsw_sp, bool carrier_ok, - u32 ptys_eth_proto, - struct ethtool_link_ksettings *cmd) -{ - cmd->base.speed = SPEED_UNKNOWN; - cmd->base.duplex = DUPLEX_UNKNOWN; - - if (!carrier_ok) - return; - - cmd->base.speed = mlxsw_sp2_from_ptys_speed(mlxsw_sp, ptys_eth_proto); - if (cmd->base.speed != SPEED_UNKNOWN) - cmd->base.duplex = DUPLEX_FULL; -} - -static bool -mlxsw_sp2_test_bit_ethtool(const struct mlxsw_sp2_port_link_mode *link_mode, - const unsigned long *mode) -{ - int cnt = 0; - int i; - - for (i = 0; i < link_mode->m_ethtool_len; i++) { - if (test_bit(link_mode->mask_ethtool[i], mode)) - cnt++; - } - - return cnt == link_mode->m_ethtool_len; -} - -static u32 -mlxsw_sp2_to_ptys_advert_link(struct mlxsw_sp *mlxsw_sp, u8 width, - const struct ethtool_link_ksettings *cmd) -{ - u8 mask_width = mlxsw_sp_port_mask_width_get(width); - u32 ptys_proto = 0; - int i; - - for (i = 0; i < MLXSW_SP2_PORT_LINK_MODE_LEN; i++) { - if ((mask_width & mlxsw_sp2_port_link_mode[i].mask_width) && - mlxsw_sp2_test_bit_ethtool(&mlxsw_sp2_port_link_mode[i], - cmd->link_modes.advertising)) - ptys_proto |= mlxsw_sp2_port_link_mode[i].mask; - } - return ptys_proto; -} - -static u32 mlxsw_sp2_to_ptys_speed(struct mlxsw_sp *mlxsw_sp, - u8 width, u32 speed) -{ - u8 mask_width = mlxsw_sp_port_mask_width_get(width); - u32 ptys_proto = 0; - int i; - - for (i = 0; i < MLXSW_SP2_PORT_LINK_MODE_LEN; i++) { - if ((speed == mlxsw_sp2_port_link_mode[i].speed) && - (mask_width & mlxsw_sp2_port_link_mode[i].mask_width)) - ptys_proto |= mlxsw_sp2_port_link_mode[i].mask; - } - return ptys_proto; -} - -static void -mlxsw_sp2_reg_ptys_eth_pack(struct mlxsw_sp *mlxsw_sp, char *payload, - u8 local_port, u32 proto_admin, - bool autoneg) -{ - mlxsw_reg_ptys_ext_eth_pack(payload, local_port, proto_admin, autoneg); -} - -static void -mlxsw_sp2_reg_ptys_eth_unpack(struct mlxsw_sp *mlxsw_sp, char *payload, - u32 *p_eth_proto_cap, u32 *p_eth_proto_admin, - u32 *p_eth_proto_oper) -{ - mlxsw_reg_ptys_ext_eth_unpack(payload, p_eth_proto_cap, - p_eth_proto_admin, p_eth_proto_oper); -} - -static const struct mlxsw_sp_port_type_speed_ops -mlxsw_sp2_port_type_speed_ops = { - .from_ptys_supported_port = mlxsw_sp2_from_ptys_supported_port, - .from_ptys_link = mlxsw_sp2_from_ptys_link, - .from_ptys_speed = mlxsw_sp2_from_ptys_speed, - .from_ptys_speed_duplex = mlxsw_sp2_from_ptys_speed_duplex, - .to_ptys_advert_link = mlxsw_sp2_to_ptys_advert_link, - .to_ptys_speed = mlxsw_sp2_to_ptys_speed, - .reg_ptys_eth_pack = mlxsw_sp2_reg_ptys_eth_pack, - .reg_ptys_eth_unpack = mlxsw_sp2_reg_ptys_eth_unpack, -}; - -static void -mlxsw_sp_port_get_link_supported(struct mlxsw_sp *mlxsw_sp, u32 eth_proto_cap, - u8 width, struct ethtool_link_ksettings *cmd) -{ - const struct mlxsw_sp_port_type_speed_ops *ops; - - ops = mlxsw_sp->port_type_speed_ops; - - ethtool_link_ksettings_add_link_mode(cmd, supported, Asym_Pause); - ethtool_link_ksettings_add_link_mode(cmd, supported, Autoneg); - ethtool_link_ksettings_add_link_mode(cmd, supported, Pause); - - ops->from_ptys_supported_port(mlxsw_sp, eth_proto_cap, cmd); - ops->from_ptys_link(mlxsw_sp, eth_proto_cap, width, - cmd->link_modes.supported); -} - -static void -mlxsw_sp_port_get_link_advertise(struct mlxsw_sp *mlxsw_sp, - u32 eth_proto_admin, bool autoneg, u8 width, - struct ethtool_link_ksettings *cmd) -{ - const struct mlxsw_sp_port_type_speed_ops *ops; - - ops = mlxsw_sp->port_type_speed_ops; - - if (!autoneg) - return; - - ethtool_link_ksettings_add_link_mode(cmd, advertising, Autoneg); - ops->from_ptys_link(mlxsw_sp, eth_proto_admin, width, - cmd->link_modes.advertising); -} - -static u8 -mlxsw_sp_port_connector_port(enum mlxsw_reg_ptys_connector_type connector_type) -{ - switch (connector_type) { - case MLXSW_REG_PTYS_CONNECTOR_TYPE_UNKNOWN_OR_NO_CONNECTOR: - return PORT_OTHER; - case MLXSW_REG_PTYS_CONNECTOR_TYPE_PORT_NONE: - return PORT_NONE; - case MLXSW_REG_PTYS_CONNECTOR_TYPE_PORT_TP: - return PORT_TP; - case MLXSW_REG_PTYS_CONNECTOR_TYPE_PORT_AUI: - return PORT_AUI; - case MLXSW_REG_PTYS_CONNECTOR_TYPE_PORT_BNC: - return PORT_BNC; - case MLXSW_REG_PTYS_CONNECTOR_TYPE_PORT_MII: - return PORT_MII; - case MLXSW_REG_PTYS_CONNECTOR_TYPE_PORT_FIBRE: - return PORT_FIBRE; - case MLXSW_REG_PTYS_CONNECTOR_TYPE_PORT_DA: - return PORT_DA; - case MLXSW_REG_PTYS_CONNECTOR_TYPE_PORT_OTHER: - return PORT_OTHER; - default: - WARN_ON_ONCE(1); - return PORT_OTHER; - } -} - -static int mlxsw_sp_port_get_link_ksettings(struct net_device *dev, - struct ethtool_link_ksettings *cmd) -{ - u32 eth_proto_cap, eth_proto_admin, eth_proto_oper; - struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); - struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; - const struct mlxsw_sp_port_type_speed_ops *ops; - char ptys_pl[MLXSW_REG_PTYS_LEN]; - u8 connector_type; - bool autoneg; - int err; - - ops = mlxsw_sp->port_type_speed_ops; - - autoneg = mlxsw_sp_port->link.autoneg; - ops->reg_ptys_eth_pack(mlxsw_sp, ptys_pl, mlxsw_sp_port->local_port, - 0, false); - err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ptys), ptys_pl); - if (err) - return err; - ops->reg_ptys_eth_unpack(mlxsw_sp, ptys_pl, ð_proto_cap, - ð_proto_admin, ð_proto_oper); - - mlxsw_sp_port_get_link_supported(mlxsw_sp, eth_proto_cap, - mlxsw_sp_port->mapping.width, cmd); - - mlxsw_sp_port_get_link_advertise(mlxsw_sp, eth_proto_admin, autoneg, - mlxsw_sp_port->mapping.width, cmd); - - cmd->base.autoneg = autoneg ? AUTONEG_ENABLE : AUTONEG_DISABLE; - connector_type = mlxsw_reg_ptys_connector_type_get(ptys_pl); - cmd->base.port = mlxsw_sp_port_connector_port(connector_type); - ops->from_ptys_speed_duplex(mlxsw_sp, netif_carrier_ok(dev), - eth_proto_oper, cmd); - - return 0; -} - -static int -mlxsw_sp_port_set_link_ksettings(struct net_device *dev, - const struct ethtool_link_ksettings *cmd) -{ - struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); - struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; - const struct mlxsw_sp_port_type_speed_ops *ops; - char ptys_pl[MLXSW_REG_PTYS_LEN]; - u32 eth_proto_cap, eth_proto_new; - bool autoneg; - int err; - - ops = mlxsw_sp->port_type_speed_ops; - - ops->reg_ptys_eth_pack(mlxsw_sp, ptys_pl, mlxsw_sp_port->local_port, - 0, false); - err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ptys), ptys_pl); - if (err) - return err; - ops->reg_ptys_eth_unpack(mlxsw_sp, ptys_pl, ð_proto_cap, NULL, NULL); - - autoneg = cmd->base.autoneg == AUTONEG_ENABLE; - eth_proto_new = autoneg ? - ops->to_ptys_advert_link(mlxsw_sp, mlxsw_sp_port->mapping.width, - cmd) : - ops->to_ptys_speed(mlxsw_sp, mlxsw_sp_port->mapping.width, - cmd->base.speed); - - eth_proto_new = eth_proto_new & eth_proto_cap; - if (!eth_proto_new) { - netdev_err(dev, "No supported speed requested\n"); - return -EINVAL; - } - - ops->reg_ptys_eth_pack(mlxsw_sp, ptys_pl, mlxsw_sp_port->local_port, - eth_proto_new, autoneg); - err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptys), ptys_pl); - if (err) - return err; - - mlxsw_sp_port->link.autoneg = autoneg; - - if (!netif_running(dev)) - return 0; - - mlxsw_sp_port_admin_status_set(mlxsw_sp_port, false); - mlxsw_sp_port_admin_status_set(mlxsw_sp_port, true); - - return 0; -} - -static int mlxsw_sp_get_module_info(struct net_device *netdev, - struct ethtool_modinfo *modinfo) -{ - struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(netdev); - struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; - int err; - - err = mlxsw_env_get_module_info(mlxsw_sp->core, - mlxsw_sp_port->mapping.module, - modinfo); - - return err; -} - -static int mlxsw_sp_get_module_eeprom(struct net_device *netdev, - struct ethtool_eeprom *ee, - u8 *data) -{ - struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(netdev); - struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; - int err; - - err = mlxsw_env_get_module_eeprom(netdev, mlxsw_sp->core, - mlxsw_sp_port->mapping.module, ee, - data); - - return err; -} - -static int -mlxsw_sp_get_ts_info(struct net_device *netdev, struct ethtool_ts_info *info) -{ - struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(netdev); - struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; - - return mlxsw_sp->ptp_ops->get_ts_info(mlxsw_sp, info); -} - -static const struct ethtool_ops mlxsw_sp_port_ethtool_ops = { - .get_drvinfo = mlxsw_sp_port_get_drvinfo, - .get_link = ethtool_op_get_link, - .get_pauseparam = mlxsw_sp_port_get_pauseparam, - .set_pauseparam = mlxsw_sp_port_set_pauseparam, - .get_strings = mlxsw_sp_port_get_strings, - .set_phys_id = mlxsw_sp_port_set_phys_id, - .get_ethtool_stats = mlxsw_sp_port_get_stats, - .get_sset_count = mlxsw_sp_port_get_sset_count, - .get_link_ksettings = mlxsw_sp_port_get_link_ksettings, - .set_link_ksettings = mlxsw_sp_port_set_link_ksettings, - .get_module_info = mlxsw_sp_get_module_info, - .get_module_eeprom = mlxsw_sp_get_module_eeprom, - .get_ts_info = mlxsw_sp_get_ts_info, -}; - static int mlxsw_sp_port_speed_by_width_set(struct mlxsw_sp_port *mlxsw_sp_port) { @@ -3246,12 +1744,16 @@ static int mlxsw_sp_port_create(struct mlxsw_sp *mlxsw_sp, u8 local_port, struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; bool split = !!split_base_local_port; struct mlxsw_sp_port *mlxsw_sp_port; + u32 lanes = port_mapping->width; struct net_device *dev; + bool splittable; int err; + splittable = lanes > 1 && !split; err = mlxsw_core_port_init(mlxsw_sp->core, local_port, port_mapping->module + 1, split, - port_mapping->lane / port_mapping->width, + port_mapping->lane / lanes, + splittable, lanes, mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac)); if (err) { @@ -3745,13 +2247,6 @@ static int mlxsw_sp_port_split(struct mlxsw_core *mlxsw_core, u8 local_port, return -EINVAL; } - /* Split ports cannot be split. */ - if (mlxsw_sp_port->split) { - netdev_err(mlxsw_sp_port->dev, "Port cannot be split further\n"); - NL_SET_ERR_MSG_MOD(extack, "Port cannot be split further"); - return -EINVAL; - } - max_width = mlxsw_core_module_max_width(mlxsw_core, mlxsw_sp_port->mapping.module); if (max_width < 0) { @@ -3760,19 +2255,13 @@ static int mlxsw_sp_port_split(struct mlxsw_core *mlxsw_core, u8 local_port, return max_width; } - /* Split port with non-max and 1 module width cannot be split. */ - if (mlxsw_sp_port->mapping.width != max_width || max_width == 1) { + /* Split port with non-max cannot be split. */ + if (mlxsw_sp_port->mapping.width != max_width) { netdev_err(mlxsw_sp_port->dev, "Port cannot be split\n"); NL_SET_ERR_MSG_MOD(extack, "Port cannot be split"); return -EINVAL; } - if (count == 1 || !is_power_of_2(count) || count > max_width) { - netdev_err(mlxsw_sp_port->dev, "Invalid split count\n"); - NL_SET_ERR_MSG_MOD(extack, "Invalid split count"); - return -EINVAL; - } - offset = mlxsw_sp_local_ports_offset(mlxsw_core, count, max_width); if (offset < 0) { netdev_err(mlxsw_sp_port->dev, "Cannot obtain local port offset\n"); @@ -4334,52 +2823,6 @@ static const struct mlxsw_sp_ptp_ops mlxsw_sp2_ptp_ops = { .get_stats = mlxsw_sp2_get_stats, }; -static u32 mlxsw_sp1_span_buffsize_get(int mtu, u32 speed) -{ - return mtu * 5 / 2; -} - -static const struct mlxsw_sp_span_ops mlxsw_sp1_span_ops = { - .buffsize_get = mlxsw_sp1_span_buffsize_get, -}; - -#define MLXSW_SP2_SPAN_EG_MIRROR_BUFFER_FACTOR 38 -#define MLXSW_SP3_SPAN_EG_MIRROR_BUFFER_FACTOR 50 - -static u32 __mlxsw_sp_span_buffsize_get(int mtu, u32 speed, u32 buffer_factor) -{ - return 3 * mtu + buffer_factor * speed / 1000; -} - -static u32 mlxsw_sp2_span_buffsize_get(int mtu, u32 speed) -{ - int factor = MLXSW_SP2_SPAN_EG_MIRROR_BUFFER_FACTOR; - - return __mlxsw_sp_span_buffsize_get(mtu, speed, factor); -} - -static const struct mlxsw_sp_span_ops mlxsw_sp2_span_ops = { - .buffsize_get = mlxsw_sp2_span_buffsize_get, -}; - -static u32 mlxsw_sp3_span_buffsize_get(int mtu, u32 speed) -{ - int factor = MLXSW_SP3_SPAN_EG_MIRROR_BUFFER_FACTOR; - - return __mlxsw_sp_span_buffsize_get(mtu, speed, factor); -} - -static const struct mlxsw_sp_span_ops mlxsw_sp3_span_ops = { - .buffsize_get = mlxsw_sp3_span_buffsize_get, -}; - -u32 mlxsw_sp_span_buffsize_get(struct mlxsw_sp *mlxsw_sp, int mtu, u32 speed) -{ - u32 buffsize = mlxsw_sp->span_ops->buffsize_get(speed, mtu); - - return mlxsw_sp_bytes_cells(mlxsw_sp, buffsize) + 1; -} - static int mlxsw_sp_netdevice_event(struct notifier_block *unused, unsigned long event, void *ptr); @@ -4417,6 +2860,12 @@ static int mlxsw_sp_init(struct mlxsw_core *mlxsw_core, goto err_fids_init; } + err = mlxsw_sp_policers_init(mlxsw_sp); + if (err) { + dev_err(mlxsw_sp->bus_info->dev, "Failed to initialize policers\n"); + goto err_policers_init; + } + err = mlxsw_sp_traps_init(mlxsw_sp); if (err) { dev_err(mlxsw_sp->bus_info->dev, "Failed to set traps\n"); @@ -4576,6 +3025,8 @@ err_buffers_init: err_devlink_traps_init: mlxsw_sp_traps_fini(mlxsw_sp); err_traps_init: + mlxsw_sp_policers_fini(mlxsw_sp); +err_policers_init: mlxsw_sp_fids_fini(mlxsw_sp); err_fids_init: mlxsw_sp_kvdl_fini(mlxsw_sp); @@ -4594,6 +3045,7 @@ static int mlxsw_sp1_init(struct mlxsw_core *mlxsw_core, mlxsw_sp->afa_ops = &mlxsw_sp1_act_afa_ops; mlxsw_sp->afk_ops = &mlxsw_sp1_afk_ops; mlxsw_sp->mr_tcam_ops = &mlxsw_sp1_mr_tcam_ops; + mlxsw_sp->acl_rulei_ops = &mlxsw_sp1_acl_rulei_ops; mlxsw_sp->acl_tcam_ops = &mlxsw_sp1_acl_tcam_ops; mlxsw_sp->nve_ops_arr = mlxsw_sp1_nve_ops_arr; mlxsw_sp->mac_mask = mlxsw_sp1_mac_mask; @@ -4602,6 +3054,8 @@ static int mlxsw_sp1_init(struct mlxsw_core *mlxsw_core, mlxsw_sp->port_type_speed_ops = &mlxsw_sp1_port_type_speed_ops; mlxsw_sp->ptp_ops = &mlxsw_sp1_ptp_ops; mlxsw_sp->span_ops = &mlxsw_sp1_span_ops; + mlxsw_sp->policer_core_ops = &mlxsw_sp1_policer_core_ops; + mlxsw_sp->trap_ops = &mlxsw_sp1_trap_ops; mlxsw_sp->listeners = mlxsw_sp1_listener; mlxsw_sp->listeners_count = ARRAY_SIZE(mlxsw_sp1_listener); mlxsw_sp->lowest_shaper_bs = MLXSW_REG_QEEC_LOWEST_SHAPER_BS_SP1; @@ -4621,6 +3075,7 @@ static int mlxsw_sp2_init(struct mlxsw_core *mlxsw_core, mlxsw_sp->afa_ops = &mlxsw_sp2_act_afa_ops; mlxsw_sp->afk_ops = &mlxsw_sp2_afk_ops; mlxsw_sp->mr_tcam_ops = &mlxsw_sp2_mr_tcam_ops; + mlxsw_sp->acl_rulei_ops = &mlxsw_sp2_acl_rulei_ops; mlxsw_sp->acl_tcam_ops = &mlxsw_sp2_acl_tcam_ops; mlxsw_sp->nve_ops_arr = mlxsw_sp2_nve_ops_arr; mlxsw_sp->mac_mask = mlxsw_sp2_mac_mask; @@ -4629,6 +3084,8 @@ static int mlxsw_sp2_init(struct mlxsw_core *mlxsw_core, mlxsw_sp->port_type_speed_ops = &mlxsw_sp2_port_type_speed_ops; mlxsw_sp->ptp_ops = &mlxsw_sp2_ptp_ops; mlxsw_sp->span_ops = &mlxsw_sp2_span_ops; + mlxsw_sp->policer_core_ops = &mlxsw_sp2_policer_core_ops; + mlxsw_sp->trap_ops = &mlxsw_sp2_trap_ops; mlxsw_sp->lowest_shaper_bs = MLXSW_REG_QEEC_LOWEST_SHAPER_BS_SP2; return mlxsw_sp_init(mlxsw_core, mlxsw_bus_info, extack); @@ -4640,10 +3097,13 @@ static int mlxsw_sp3_init(struct mlxsw_core *mlxsw_core, { struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core); + mlxsw_sp->req_rev = &mlxsw_sp3_fw_rev; + mlxsw_sp->fw_filename = MLXSW_SP3_FW_FILENAME; mlxsw_sp->kvdl_ops = &mlxsw_sp2_kvdl_ops; mlxsw_sp->afa_ops = &mlxsw_sp2_act_afa_ops; mlxsw_sp->afk_ops = &mlxsw_sp2_afk_ops; mlxsw_sp->mr_tcam_ops = &mlxsw_sp2_mr_tcam_ops; + mlxsw_sp->acl_rulei_ops = &mlxsw_sp2_acl_rulei_ops; mlxsw_sp->acl_tcam_ops = &mlxsw_sp2_acl_tcam_ops; mlxsw_sp->nve_ops_arr = mlxsw_sp2_nve_ops_arr; mlxsw_sp->mac_mask = mlxsw_sp2_mac_mask; @@ -4652,6 +3112,8 @@ static int mlxsw_sp3_init(struct mlxsw_core *mlxsw_core, mlxsw_sp->port_type_speed_ops = &mlxsw_sp2_port_type_speed_ops; mlxsw_sp->ptp_ops = &mlxsw_sp2_ptp_ops; mlxsw_sp->span_ops = &mlxsw_sp3_span_ops; + mlxsw_sp->policer_core_ops = &mlxsw_sp2_policer_core_ops; + mlxsw_sp->trap_ops = &mlxsw_sp2_trap_ops; mlxsw_sp->lowest_shaper_bs = MLXSW_REG_QEEC_LOWEST_SHAPER_BS_SP3; return mlxsw_sp_init(mlxsw_core, mlxsw_bus_info, extack); @@ -4681,6 +3143,7 @@ static void mlxsw_sp_fini(struct mlxsw_core *mlxsw_core) mlxsw_sp_buffers_fini(mlxsw_sp); mlxsw_sp_devlink_traps_fini(mlxsw_sp); mlxsw_sp_traps_fini(mlxsw_sp); + mlxsw_sp_policers_fini(mlxsw_sp); mlxsw_sp_fids_fini(mlxsw_sp); mlxsw_sp_kvdl_fini(mlxsw_sp); } @@ -4892,6 +3355,10 @@ static int mlxsw_sp1_resources_register(struct mlxsw_core *mlxsw_core) if (err) goto err_resources_counter_register; + err = mlxsw_sp_policer_resources_register(mlxsw_core); + if (err) + goto err_resources_counter_register; + return 0; err_resources_counter_register: @@ -4916,6 +3383,10 @@ static int mlxsw_sp2_resources_register(struct mlxsw_core *mlxsw_core) if (err) goto err_resources_counter_register; + err = mlxsw_sp_policer_resources_register(mlxsw_core); + if (err) + goto err_resources_counter_register; + return 0; err_resources_counter_register: @@ -6330,3 +4801,4 @@ MODULE_DEVICE_TABLE(pci, mlxsw_sp2_pci_id_table); MODULE_DEVICE_TABLE(pci, mlxsw_sp3_pci_id_table); MODULE_FIRMWARE(MLXSW_SP1_FW_FILENAME); MODULE_FIRMWARE(MLXSW_SP2_FW_FILENAME); +MODULE_FIRMWARE(MLXSW_SP3_FW_FILENAME); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h index 3abe3e7d89bc..f9ba59641b4d 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h @@ -62,6 +62,8 @@ enum mlxsw_sp_resource_id { MLXSW_SP_RESOURCE_COUNTERS, MLXSW_SP_RESOURCE_COUNTERS_FLOW, MLXSW_SP_RESOURCE_COUNTERS_RIF, + MLXSW_SP_RESOURCE_GLOBAL_POLICERS, + MLXSW_SP_RESOURCE_SINGLE_RATE_POLICERS, }; struct mlxsw_sp_port; @@ -120,6 +122,7 @@ struct mlxsw_sp_kvdl; struct mlxsw_sp_nve; struct mlxsw_sp_kvdl_ops; struct mlxsw_sp_mr_tcam_ops; +struct mlxsw_sp_acl_rulei_ops; struct mlxsw_sp_acl_tcam_ops; struct mlxsw_sp_nve_ops; struct mlxsw_sp_sb_vals; @@ -150,6 +153,7 @@ struct mlxsw_sp { struct mlxsw_afa *afa; struct mlxsw_sp_acl *acl; struct mlxsw_sp_fid_core *fid_core; + struct mlxsw_sp_policer_core *policer_core; struct mlxsw_sp_kvdl *kvdl; struct mlxsw_sp_nve *nve; struct notifier_block netdevice_nb; @@ -164,6 +168,7 @@ struct mlxsw_sp { const struct mlxsw_afa_ops *afa_ops; const struct mlxsw_afk_ops *afk_ops; const struct mlxsw_sp_mr_tcam_ops *mr_tcam_ops; + const struct mlxsw_sp_acl_rulei_ops *acl_rulei_ops; const struct mlxsw_sp_acl_tcam_ops *acl_tcam_ops; const struct mlxsw_sp_nve_ops **nve_ops_arr; const struct mlxsw_sp_rif_ops **rif_ops_arr; @@ -171,11 +176,46 @@ struct mlxsw_sp { const struct mlxsw_sp_port_type_speed_ops *port_type_speed_ops; const struct mlxsw_sp_ptp_ops *ptp_ops; const struct mlxsw_sp_span_ops *span_ops; + const struct mlxsw_sp_policer_core_ops *policer_core_ops; + const struct mlxsw_sp_trap_ops *trap_ops; const struct mlxsw_listener *listeners; size_t listeners_count; u32 lowest_shaper_bs; }; +struct mlxsw_sp_ptp_ops { + struct mlxsw_sp_ptp_clock * + (*clock_init)(struct mlxsw_sp *mlxsw_sp, struct device *dev); + void (*clock_fini)(struct mlxsw_sp_ptp_clock *clock); + + struct mlxsw_sp_ptp_state *(*init)(struct mlxsw_sp *mlxsw_sp); + void (*fini)(struct mlxsw_sp_ptp_state *ptp_state); + + /* Notify a driver that a packet that might be PTP was received. Driver + * is responsible for freeing the passed-in SKB. + */ + void (*receive)(struct mlxsw_sp *mlxsw_sp, struct sk_buff *skb, + u8 local_port); + + /* Notify a driver that a timestamped packet was transmitted. Driver + * is responsible for freeing the passed-in SKB. + */ + void (*transmitted)(struct mlxsw_sp *mlxsw_sp, struct sk_buff *skb, + u8 local_port); + + int (*hwtstamp_get)(struct mlxsw_sp_port *mlxsw_sp_port, + struct hwtstamp_config *config); + int (*hwtstamp_set)(struct mlxsw_sp_port *mlxsw_sp_port, + struct hwtstamp_config *config); + void (*shaper_work)(struct work_struct *work); + int (*get_ts_info)(struct mlxsw_sp *mlxsw_sp, + struct ethtool_ts_info *info); + int (*get_stats_count)(void); + void (*get_stats_strings)(u8 **p); + void (*get_stats)(struct mlxsw_sp_port *mlxsw_sp_port, + u64 *data, int data_index); +}; + static inline struct mlxsw_sp_upper * mlxsw_sp_lag_get(struct mlxsw_sp *mlxsw_sp, u16 lag_id) { @@ -391,6 +431,13 @@ enum mlxsw_sp_flood_type { MLXSW_SP_FLOOD_TYPE_MC, }; +int mlxsw_sp_port_headroom_set(struct mlxsw_sp_port *mlxsw_sp_port, + int mtu, bool pause_en); +int mlxsw_sp_port_get_stats_raw(struct net_device *dev, int grp, + int prio, char *ppcnt_pl); +int mlxsw_sp_port_admin_status_set(struct mlxsw_sp_port *mlxsw_sp_port, + bool is_up); + /* spectrum_buffers.c */ int mlxsw_sp_buffers_init(struct mlxsw_sp *mlxsw_sp); void mlxsw_sp_buffers_fini(struct mlxsw_sp *mlxsw_sp); @@ -497,7 +544,6 @@ int mlxsw_sp_flow_counter_alloc(struct mlxsw_sp *mlxsw_sp, unsigned int *p_counter_index); void mlxsw_sp_flow_counter_free(struct mlxsw_sp *mlxsw_sp, unsigned int counter_index); -u32 mlxsw_sp_span_buffsize_get(struct mlxsw_sp *mlxsw_sp, int mtu, u32 speed); bool mlxsw_sp_port_dev_check(const struct net_device *dev); struct mlxsw_sp *mlxsw_sp_lower_get(struct net_device *dev); struct mlxsw_sp_port *mlxsw_sp_port_dev_lower_find(struct net_device *dev); @@ -644,8 +690,10 @@ struct mlxsw_sp_acl_rule_info { u8 action_created:1, ingress_bind_blocker:1, egress_bind_blocker:1, - counter_valid:1; + counter_valid:1, + policer_index_valid:1; unsigned int counter_index; + u16 policer_index; }; /* spectrum_flow.c */ @@ -669,7 +717,6 @@ struct mlxsw_sp_flow_block { struct mlxsw_sp_flow_block_binding { struct list_head list; - struct net_device *dev; struct mlxsw_sp_port *mlxsw_sp_port; bool ingress; }; @@ -727,8 +774,9 @@ mlxsw_sp_flow_block_is_mixed_bound(const struct mlxsw_sp_flow_block *block) struct mlxsw_sp_flow_block *mlxsw_sp_flow_block_create(struct mlxsw_sp *mlxsw_sp, struct net *net); void mlxsw_sp_flow_block_destroy(struct mlxsw_sp_flow_block *block); -int mlxsw_sp_setup_tc_block(struct mlxsw_sp_port *mlxsw_sp_port, - struct flow_block_offload *f); +int mlxsw_sp_setup_tc_block_clsact(struct mlxsw_sp_port *mlxsw_sp_port, + struct flow_block_offload *f, + bool ingress); /* spectrum_acl.c */ struct mlxsw_sp_acl_ruleset; @@ -806,6 +854,10 @@ int mlxsw_sp_acl_rulei_act_mangle(struct mlxsw_sp *mlxsw_sp, enum flow_action_mangle_base htype, u32 offset, u32 mask, u32 val, struct netlink_ext_ack *extack); +int mlxsw_sp_acl_rulei_act_police(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_acl_rule_info *rulei, + u32 index, u64 rate_bytes_ps, + u32 burst, struct netlink_ext_ack *extack); int mlxsw_sp_acl_rulei_act_count(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_acl_rule_info *rulei, struct netlink_ext_ack *extack); @@ -838,7 +890,8 @@ struct mlxsw_sp_acl_rule_info * mlxsw_sp_acl_rule_rulei(struct mlxsw_sp_acl_rule *rule); int mlxsw_sp_acl_rule_get_stats(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_acl_rule *rule, - u64 *packets, u64 *bytes, u64 *last_use, + u64 *packets, u64 *bytes, u64 *drops, + u64 *last_use, enum flow_action_hw_stats *used_hw_stats); struct mlxsw_sp_fid *mlxsw_sp_acl_dummy_fid(struct mlxsw_sp *mlxsw_sp); @@ -854,6 +907,17 @@ void mlxsw_sp_acl_fini(struct mlxsw_sp *mlxsw_sp); u32 mlxsw_sp_acl_region_rehash_intrvl_get(struct mlxsw_sp *mlxsw_sp); int mlxsw_sp_acl_region_rehash_intrvl_set(struct mlxsw_sp *mlxsw_sp, u32 val); +struct mlxsw_sp_acl_mangle_action; + +struct mlxsw_sp_acl_rulei_ops { + int (*act_mangle_field)(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_acl_rule_info *rulei, + struct mlxsw_sp_acl_mangle_action *mact, u32 val, + struct netlink_ext_ack *extack); +}; + +extern struct mlxsw_sp_acl_rulei_ops mlxsw_sp1_acl_rulei_ops; +extern struct mlxsw_sp_acl_rulei_ops mlxsw_sp2_acl_rulei_ops; + /* spectrum_acl_tcam.c */ struct mlxsw_sp_acl_tcam; struct mlxsw_sp_acl_tcam_region; @@ -909,6 +973,35 @@ extern const struct mlxsw_afk_ops mlxsw_sp1_afk_ops; extern const struct mlxsw_afk_ops mlxsw_sp2_afk_ops; /* spectrum_matchall.c */ +enum mlxsw_sp_mall_action_type { + MLXSW_SP_MALL_ACTION_TYPE_MIRROR, + MLXSW_SP_MALL_ACTION_TYPE_SAMPLE, + MLXSW_SP_MALL_ACTION_TYPE_TRAP, +}; + +struct mlxsw_sp_mall_mirror_entry { + const struct net_device *to_dev; + int span_id; +}; + +struct mlxsw_sp_mall_trap_entry { + int span_id; +}; + +struct mlxsw_sp_mall_entry { + struct list_head list; + unsigned long cookie; + unsigned int priority; + enum mlxsw_sp_mall_action_type type; + bool ingress; + union { + struct mlxsw_sp_mall_mirror_entry mirror; + struct mlxsw_sp_mall_trap_entry trap; + struct mlxsw_sp_port_sample sample; + }; + struct rcu_head rcu; +}; + int mlxsw_sp_mall_replace(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_flow_block *block, struct tc_cls_matchall_offload *f); @@ -955,6 +1048,8 @@ int mlxsw_sp_setup_tc_tbf(struct mlxsw_sp_port *mlxsw_sp_port, struct tc_tbf_qopt_offload *p); int mlxsw_sp_setup_tc_fifo(struct mlxsw_sp_port *mlxsw_sp_port, struct tc_fifo_qopt_offload *p); +int mlxsw_sp_setup_tc_block_qevent_early_drop(struct mlxsw_sp_port *mlxsw_sp_port, + struct flow_block_offload *f); /* spectrum_fid.c */ bool mlxsw_sp_fid_is_dummy(struct mlxsw_sp *mlxsw_sp, u16 fid_index); @@ -1088,12 +1183,14 @@ void mlxsw_sp_trap_fini(struct mlxsw_core *mlxsw_core, const struct devlink_trap *trap, void *trap_ctx); int mlxsw_sp_trap_action_set(struct mlxsw_core *mlxsw_core, const struct devlink_trap *trap, - enum devlink_trap_action action); + enum devlink_trap_action action, + struct netlink_ext_ack *extack); int mlxsw_sp_trap_group_init(struct mlxsw_core *mlxsw_core, const struct devlink_trap_group *group); int mlxsw_sp_trap_group_set(struct mlxsw_core *mlxsw_core, const struct devlink_trap_group *group, - const struct devlink_trap_policer *policer); + const struct devlink_trap_policer *policer, + struct netlink_ext_ack *extack); int mlxsw_sp_trap_policer_init(struct mlxsw_core *mlxsw_core, const struct devlink_trap_policer *policer); @@ -1107,10 +1204,48 @@ int mlxsw_sp_trap_policer_counter_get(struct mlxsw_core *mlxsw_core, const struct devlink_trap_policer *policer, u64 *p_drops); +int mlxsw_sp_trap_group_policer_hw_id_get(struct mlxsw_sp *mlxsw_sp, u16 id, + bool *p_enabled, u16 *p_hw_id); static inline struct net *mlxsw_sp_net(struct mlxsw_sp *mlxsw_sp) { return mlxsw_core_net(mlxsw_sp->core); } +/* spectrum_ethtool.c */ +extern const struct ethtool_ops mlxsw_sp_port_ethtool_ops; +extern const struct mlxsw_sp_port_type_speed_ops mlxsw_sp1_port_type_speed_ops; +extern const struct mlxsw_sp_port_type_speed_ops mlxsw_sp2_port_type_speed_ops; + +/* spectrum_policer.c */ +extern const struct mlxsw_sp_policer_core_ops mlxsw_sp1_policer_core_ops; +extern const struct mlxsw_sp_policer_core_ops mlxsw_sp2_policer_core_ops; + +enum mlxsw_sp_policer_type { + MLXSW_SP_POLICER_TYPE_SINGLE_RATE, + + __MLXSW_SP_POLICER_TYPE_MAX, + MLXSW_SP_POLICER_TYPE_MAX = __MLXSW_SP_POLICER_TYPE_MAX - 1, +}; + +struct mlxsw_sp_policer_params { + u64 rate; + u64 burst; + bool bytes; +}; + +int mlxsw_sp_policer_add(struct mlxsw_sp *mlxsw_sp, + enum mlxsw_sp_policer_type type, + const struct mlxsw_sp_policer_params *params, + struct netlink_ext_ack *extack, u16 *p_policer_index); +void mlxsw_sp_policer_del(struct mlxsw_sp *mlxsw_sp, + enum mlxsw_sp_policer_type type, + u16 policer_index); +int mlxsw_sp_policer_drops_counter_get(struct mlxsw_sp *mlxsw_sp, + enum mlxsw_sp_policer_type type, + u16 policer_index, u64 *p_drops); +int mlxsw_sp_policers_init(struct mlxsw_sp *mlxsw_sp); +void mlxsw_sp_policers_fini(struct mlxsw_sp *mlxsw_sp); +int mlxsw_sp_policer_resources_register(struct mlxsw_core *mlxsw_core); + #endif diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c index 47da9ee0045d..8cfa03a75374 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c @@ -66,6 +66,7 @@ struct mlxsw_sp_acl_rule { u64 last_used; u64 last_packets; u64 last_bytes; + u64 last_drops; unsigned long priv[]; /* priv has to be always the last item */ }; @@ -508,6 +509,8 @@ enum mlxsw_sp_acl_mangle_field { MLXSW_SP_ACL_MANGLE_FIELD_IP_DSFIELD, MLXSW_SP_ACL_MANGLE_FIELD_IP_DSCP, MLXSW_SP_ACL_MANGLE_FIELD_IP_ECN, + MLXSW_SP_ACL_MANGLE_FIELD_IP_SPORT, + MLXSW_SP_ACL_MANGLE_FIELD_IP_DPORT, }; struct mlxsw_sp_acl_mangle_action { @@ -538,13 +541,26 @@ struct mlxsw_sp_acl_mangle_action { MLXSW_SP_ACL_MANGLE_ACTION(FLOW_ACT_MANGLE_HDR_TYPE_IP6, \ _offset, _mask, _shift, _field) +#define MLXSW_SP_ACL_MANGLE_ACTION_TCP(_offset, _mask, _shift, _field) \ + MLXSW_SP_ACL_MANGLE_ACTION(FLOW_ACT_MANGLE_HDR_TYPE_TCP, _offset, _mask, _shift, _field) + +#define MLXSW_SP_ACL_MANGLE_ACTION_UDP(_offset, _mask, _shift, _field) \ + MLXSW_SP_ACL_MANGLE_ACTION(FLOW_ACT_MANGLE_HDR_TYPE_UDP, _offset, _mask, _shift, _field) + static struct mlxsw_sp_acl_mangle_action mlxsw_sp_acl_mangle_actions[] = { MLXSW_SP_ACL_MANGLE_ACTION_IP4(0, 0xff00ffff, 16, IP_DSFIELD), MLXSW_SP_ACL_MANGLE_ACTION_IP4(0, 0xff03ffff, 18, IP_DSCP), MLXSW_SP_ACL_MANGLE_ACTION_IP4(0, 0xfffcffff, 16, IP_ECN), + MLXSW_SP_ACL_MANGLE_ACTION_IP6(0, 0xf00fffff, 20, IP_DSFIELD), MLXSW_SP_ACL_MANGLE_ACTION_IP6(0, 0xf03fffff, 22, IP_DSCP), MLXSW_SP_ACL_MANGLE_ACTION_IP6(0, 0xffcfffff, 20, IP_ECN), + + MLXSW_SP_ACL_MANGLE_ACTION_TCP(0, 0x0000ffff, 16, IP_SPORT), + MLXSW_SP_ACL_MANGLE_ACTION_TCP(0, 0xffff0000, 0, IP_DPORT), + + MLXSW_SP_ACL_MANGLE_ACTION_UDP(0, 0x0000ffff, 16, IP_SPORT), + MLXSW_SP_ACL_MANGLE_ACTION_UDP(0, 0xffff0000, 0, IP_DPORT), }; static int @@ -563,11 +579,48 @@ mlxsw_sp_acl_rulei_act_mangle_field(struct mlxsw_sp *mlxsw_sp, case MLXSW_SP_ACL_MANGLE_FIELD_IP_ECN: return mlxsw_afa_block_append_qos_ecn(rulei->act_block, val, extack); + default: + return -EOPNOTSUPP; } +} - /* We shouldn't have gotten a match in the first place! */ - WARN_ONCE(1, "Unhandled mangle field"); - return -EINVAL; +static int mlxsw_sp1_acl_rulei_act_mangle_field(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_acl_rule_info *rulei, + struct mlxsw_sp_acl_mangle_action *mact, + u32 val, struct netlink_ext_ack *extack) +{ + int err; + + err = mlxsw_sp_acl_rulei_act_mangle_field(mlxsw_sp, rulei, mact, val, extack); + if (err != -EOPNOTSUPP) + return err; + + NL_SET_ERR_MSG_MOD(extack, "Unsupported mangle field"); + return err; +} + +static int mlxsw_sp2_acl_rulei_act_mangle_field(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_acl_rule_info *rulei, + struct mlxsw_sp_acl_mangle_action *mact, + u32 val, struct netlink_ext_ack *extack) +{ + int err; + + err = mlxsw_sp_acl_rulei_act_mangle_field(mlxsw_sp, rulei, mact, val, extack); + if (err != -EOPNOTSUPP) + return err; + + switch (mact->field) { + case MLXSW_SP_ACL_MANGLE_FIELD_IP_SPORT: + return mlxsw_afa_block_append_l4port(rulei->act_block, false, val, extack); + case MLXSW_SP_ACL_MANGLE_FIELD_IP_DPORT: + return mlxsw_afa_block_append_l4port(rulei->act_block, true, val, extack); + default: + break; + } + + NL_SET_ERR_MSG_MOD(extack, "Unsupported mangle field"); + return err; } int mlxsw_sp_acl_rulei_act_mangle(struct mlxsw_sp *mlxsw_sp, @@ -576,6 +629,7 @@ int mlxsw_sp_acl_rulei_act_mangle(struct mlxsw_sp *mlxsw_sp, u32 offset, u32 mask, u32 val, struct netlink_ext_ack *extack) { + const struct mlxsw_sp_acl_rulei_ops *acl_rulei_ops = mlxsw_sp->acl_rulei_ops; struct mlxsw_sp_acl_mangle_action *mact; size_t i; @@ -585,16 +639,34 @@ int mlxsw_sp_acl_rulei_act_mangle(struct mlxsw_sp *mlxsw_sp, mact->offset == offset && mact->mask == mask) { val >>= mact->shift; - return mlxsw_sp_acl_rulei_act_mangle_field(mlxsw_sp, - rulei, mact, - val, extack); + return acl_rulei_ops->act_mangle_field(mlxsw_sp, + rulei, mact, + val, extack); } } - NL_SET_ERR_MSG_MOD(extack, "Unsupported mangle field"); + NL_SET_ERR_MSG_MOD(extack, "Unknown mangle field"); return -EINVAL; } +int mlxsw_sp_acl_rulei_act_police(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_acl_rule_info *rulei, + u32 index, u64 rate_bytes_ps, + u32 burst, struct netlink_ext_ack *extack) +{ + int err; + + err = mlxsw_afa_block_append_police(rulei->act_block, index, + rate_bytes_ps, burst, + &rulei->policer_index, extack); + if (err) + return err; + + rulei->policer_index_valid = true; + + return 0; +} + int mlxsw_sp_acl_rulei_act_count(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_acl_rule_info *rulei, struct netlink_ext_ack *extack) @@ -815,13 +887,16 @@ static void mlxsw_sp_acl_rule_activity_update_work(struct work_struct *work) int mlxsw_sp_acl_rule_get_stats(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_acl_rule *rule, - u64 *packets, u64 *bytes, u64 *last_use, + u64 *packets, u64 *bytes, u64 *drops, + u64 *last_use, enum flow_action_hw_stats *used_hw_stats) { + enum mlxsw_sp_policer_type type = MLXSW_SP_POLICER_TYPE_SINGLE_RATE; struct mlxsw_sp_acl_rule_info *rulei; u64 current_packets = 0; u64 current_bytes = 0; + u64 current_drops = 0; int err; rulei = mlxsw_sp_acl_rule_rulei(rule); @@ -833,12 +908,21 @@ int mlxsw_sp_acl_rule_get_stats(struct mlxsw_sp *mlxsw_sp, return err; *used_hw_stats = FLOW_ACTION_HW_STATS_IMMEDIATE; } + if (rulei->policer_index_valid) { + err = mlxsw_sp_policer_drops_counter_get(mlxsw_sp, type, + rulei->policer_index, + ¤t_drops); + if (err) + return err; + } *packets = current_packets - rule->last_packets; *bytes = current_bytes - rule->last_bytes; + *drops = current_drops - rule->last_drops; *last_use = rule->last_used; rule->last_bytes = current_bytes; rule->last_packets = current_packets; + rule->last_drops = current_drops; return 0; } @@ -930,3 +1014,11 @@ int mlxsw_sp_acl_region_rehash_intrvl_set(struct mlxsw_sp *mlxsw_sp, u32 val) return mlxsw_sp_acl_tcam_vregion_rehash_intrvl_set(mlxsw_sp, &acl->tcam, val); } + +struct mlxsw_sp_acl_rulei_ops mlxsw_sp1_acl_rulei_ops = { + .act_mangle_field = mlxsw_sp1_acl_rulei_act_mangle_field, +}; + +struct mlxsw_sp_acl_rulei_ops mlxsw_sp2_acl_rulei_ops = { + .act_mangle_field = mlxsw_sp2_acl_rulei_act_mangle_field, +}; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_actions.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_actions.c index 73d56012654b..90372d1c28d4 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_actions.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_actions.c @@ -136,11 +136,13 @@ mlxsw_sp_act_mirror_add(void *priv, u8 local_in_port, const struct net_device *out_dev, bool ingress, int *p_span_id) { + struct mlxsw_sp_span_agent_parms agent_parms = {}; struct mlxsw_sp_port *mlxsw_sp_port; struct mlxsw_sp *mlxsw_sp = priv; int err; - err = mlxsw_sp_span_agent_get(mlxsw_sp, out_dev, p_span_id); + agent_parms.to_dev = out_dev; + err = mlxsw_sp_span_agent_get(mlxsw_sp, p_span_id, &agent_parms); if (err) return err; @@ -167,6 +169,29 @@ mlxsw_sp_act_mirror_del(void *priv, u8 local_in_port, int span_id, bool ingress) mlxsw_sp_span_agent_put(mlxsw_sp, span_id); } +static int mlxsw_sp_act_policer_add(void *priv, u64 rate_bytes_ps, u32 burst, + u16 *p_policer_index, + struct netlink_ext_ack *extack) +{ + struct mlxsw_sp_policer_params params; + struct mlxsw_sp *mlxsw_sp = priv; + + params.rate = rate_bytes_ps; + params.burst = burst; + params.bytes = true; + return mlxsw_sp_policer_add(mlxsw_sp, + MLXSW_SP_POLICER_TYPE_SINGLE_RATE, + ¶ms, extack, p_policer_index); +} + +static void mlxsw_sp_act_policer_del(void *priv, u16 policer_index) +{ + struct mlxsw_sp *mlxsw_sp = priv; + + mlxsw_sp_policer_del(mlxsw_sp, MLXSW_SP_POLICER_TYPE_SINGLE_RATE, + policer_index); +} + const struct mlxsw_afa_ops mlxsw_sp1_act_afa_ops = { .kvdl_set_add = mlxsw_sp1_act_kvdl_set_add, .kvdl_set_del = mlxsw_sp_act_kvdl_set_del, @@ -177,6 +202,8 @@ const struct mlxsw_afa_ops mlxsw_sp1_act_afa_ops = { .counter_index_put = mlxsw_sp_act_counter_index_put, .mirror_add = mlxsw_sp_act_mirror_add, .mirror_del = mlxsw_sp_act_mirror_del, + .policer_add = mlxsw_sp_act_policer_add, + .policer_del = mlxsw_sp_act_policer_del, }; const struct mlxsw_afa_ops mlxsw_sp2_act_afa_ops = { @@ -189,6 +216,8 @@ const struct mlxsw_afa_ops mlxsw_sp2_act_afa_ops = { .counter_index_put = mlxsw_sp_act_counter_index_put, .mirror_add = mlxsw_sp_act_mirror_add, .mirror_del = mlxsw_sp_act_mirror_del, + .policer_add = mlxsw_sp_act_policer_add, + .policer_del = mlxsw_sp_act_policer_del, .dummy_first_set = true, }; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_cnt.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_cnt.c index 7974982533b5..b65b93a2b9bc 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_cnt.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_cnt.c @@ -121,7 +121,6 @@ int mlxsw_sp_counter_pool_init(struct mlxsw_sp *mlxsw_sp) { unsigned int sub_pools_count = ARRAY_SIZE(mlxsw_sp_counter_sub_pools); struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); - struct mlxsw_sp_counter_sub_pool *sub_pool; struct mlxsw_sp_counter_pool *pool; unsigned int map_size; int err; @@ -131,9 +130,9 @@ int mlxsw_sp_counter_pool_init(struct mlxsw_sp *mlxsw_sp) if (!pool) return -ENOMEM; mlxsw_sp->counter_pool = pool; - memcpy(pool->sub_pools, mlxsw_sp_counter_sub_pools, - sub_pools_count * sizeof(*sub_pool)); pool->sub_pools_count = sub_pools_count; + memcpy(pool->sub_pools, mlxsw_sp_counter_sub_pools, + flex_array_size(pool, sub_pools, pool->sub_pools_count)); spin_lock_init(&pool->counter_pool_lock); atomic_set(&pool->active_entries_count, 0); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_dcb.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_dcb.c index 49a72a8f1f57..0d3fb2e51ea5 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_dcb.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_dcb.c @@ -110,8 +110,8 @@ static int mlxsw_sp_port_pg_destroy(struct mlxsw_sp_port *mlxsw_sp_port, return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pbmc), pbmc_pl); } -static int mlxsw_sp_port_headroom_set(struct mlxsw_sp_port *mlxsw_sp_port, - struct ieee_ets *ets) +static int mlxsw_sp_port_headroom_ets_set(struct mlxsw_sp_port *mlxsw_sp_port, + struct ieee_ets *ets) { bool pause_en = mlxsw_sp_port_is_pause_en(mlxsw_sp_port); struct ieee_ets *my_ets = mlxsw_sp_port->dcb.ets; @@ -138,7 +138,7 @@ static int mlxsw_sp_port_headroom_set(struct mlxsw_sp_port *mlxsw_sp_port, err = mlxsw_sp_port_pg_destroy(mlxsw_sp_port, my_ets->prio_tc, ets->prio_tc); if (err) - netdev_warn(dev, "Failed to remove ununsed PGs\n"); + netdev_warn(dev, "Failed to remove unused PGs\n"); return 0; @@ -180,7 +180,7 @@ static int __mlxsw_sp_dcbnl_ieee_setets(struct mlxsw_sp_port *mlxsw_sp_port, } /* Ingress configuration. */ - err = mlxsw_sp_port_headroom_set(mlxsw_sp_port, ets); + err = mlxsw_sp_port_headroom_ets_set(mlxsw_sp_port, ets); if (err) goto err_port_headroom_set; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c new file mode 100644 index 000000000000..14c78f73bb65 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c @@ -0,0 +1,1644 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 +/* Copyright (c) 2020 Mellanox Technologies. All rights reserved */ + +#include "reg.h" +#include "spectrum.h" +#include "core_env.h" + +static const char mlxsw_sp_driver_version[] = "1.0"; + +static void mlxsw_sp_port_get_drvinfo(struct net_device *dev, + struct ethtool_drvinfo *drvinfo) +{ + struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); + struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; + + strlcpy(drvinfo->driver, mlxsw_sp->bus_info->device_kind, + sizeof(drvinfo->driver)); + strlcpy(drvinfo->version, mlxsw_sp_driver_version, + sizeof(drvinfo->version)); + snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), + "%d.%d.%d", + mlxsw_sp->bus_info->fw_rev.major, + mlxsw_sp->bus_info->fw_rev.minor, + mlxsw_sp->bus_info->fw_rev.subminor); + strlcpy(drvinfo->bus_info, mlxsw_sp->bus_info->device_name, + sizeof(drvinfo->bus_info)); +} + +struct mlxsw_sp_ethtool_link_ext_state_opcode_mapping { + u32 status_opcode; + enum ethtool_link_ext_state link_ext_state; + u8 link_ext_substate; +}; + +static const struct mlxsw_sp_ethtool_link_ext_state_opcode_mapping +mlxsw_sp_link_ext_state_opcode_map[] = { + {2, ETHTOOL_LINK_EXT_STATE_AUTONEG, + ETHTOOL_LINK_EXT_SUBSTATE_AN_NO_PARTNER_DETECTED}, + {3, ETHTOOL_LINK_EXT_STATE_AUTONEG, + ETHTOOL_LINK_EXT_SUBSTATE_AN_ACK_NOT_RECEIVED}, + {4, ETHTOOL_LINK_EXT_STATE_AUTONEG, + ETHTOOL_LINK_EXT_SUBSTATE_AN_NEXT_PAGE_EXCHANGE_FAILED}, + {36, ETHTOOL_LINK_EXT_STATE_AUTONEG, + ETHTOOL_LINK_EXT_SUBSTATE_AN_NO_PARTNER_DETECTED_FORCE_MODE}, + {38, ETHTOOL_LINK_EXT_STATE_AUTONEG, + ETHTOOL_LINK_EXT_SUBSTATE_AN_FEC_MISMATCH_DURING_OVERRIDE}, + {39, ETHTOOL_LINK_EXT_STATE_AUTONEG, + ETHTOOL_LINK_EXT_SUBSTATE_AN_NO_HCD}, + + {5, ETHTOOL_LINK_EXT_STATE_LINK_TRAINING_FAILURE, + ETHTOOL_LINK_EXT_SUBSTATE_LT_KR_FRAME_LOCK_NOT_ACQUIRED}, + {6, ETHTOOL_LINK_EXT_STATE_LINK_TRAINING_FAILURE, + ETHTOOL_LINK_EXT_SUBSTATE_LT_KR_LINK_INHIBIT_TIMEOUT}, + {7, ETHTOOL_LINK_EXT_STATE_LINK_TRAINING_FAILURE, + ETHTOOL_LINK_EXT_SUBSTATE_LT_KR_LINK_PARTNER_DID_NOT_SET_RECEIVER_READY}, + {8, ETHTOOL_LINK_EXT_STATE_LINK_TRAINING_FAILURE, 0}, + {14, ETHTOOL_LINK_EXT_STATE_LINK_TRAINING_FAILURE, + ETHTOOL_LINK_EXT_SUBSTATE_LT_REMOTE_FAULT}, + + {9, ETHTOOL_LINK_EXT_STATE_LINK_LOGICAL_MISMATCH, + ETHTOOL_LINK_EXT_SUBSTATE_LLM_PCS_DID_NOT_ACQUIRE_BLOCK_LOCK}, + {10, ETHTOOL_LINK_EXT_STATE_LINK_LOGICAL_MISMATCH, + ETHTOOL_LINK_EXT_SUBSTATE_LLM_PCS_DID_NOT_ACQUIRE_AM_LOCK}, + {11, ETHTOOL_LINK_EXT_STATE_LINK_LOGICAL_MISMATCH, + ETHTOOL_LINK_EXT_SUBSTATE_LLM_PCS_DID_NOT_GET_ALIGN_STATUS}, + {12, ETHTOOL_LINK_EXT_STATE_LINK_LOGICAL_MISMATCH, + ETHTOOL_LINK_EXT_SUBSTATE_LLM_FC_FEC_IS_NOT_LOCKED}, + {13, ETHTOOL_LINK_EXT_STATE_LINK_LOGICAL_MISMATCH, + ETHTOOL_LINK_EXT_SUBSTATE_LLM_RS_FEC_IS_NOT_LOCKED}, + + {15, ETHTOOL_LINK_EXT_STATE_BAD_SIGNAL_INTEGRITY, 0}, + {17, ETHTOOL_LINK_EXT_STATE_BAD_SIGNAL_INTEGRITY, + ETHTOOL_LINK_EXT_SUBSTATE_BSI_LARGE_NUMBER_OF_PHYSICAL_ERRORS}, + {42, ETHTOOL_LINK_EXT_STATE_BAD_SIGNAL_INTEGRITY, + ETHTOOL_LINK_EXT_SUBSTATE_BSI_UNSUPPORTED_RATE}, + + {1024, ETHTOOL_LINK_EXT_STATE_NO_CABLE, 0}, + + {16, ETHTOOL_LINK_EXT_STATE_CABLE_ISSUE, + ETHTOOL_LINK_EXT_SUBSTATE_CI_UNSUPPORTED_CABLE}, + {20, ETHTOOL_LINK_EXT_STATE_CABLE_ISSUE, + ETHTOOL_LINK_EXT_SUBSTATE_CI_UNSUPPORTED_CABLE}, + {29, ETHTOOL_LINK_EXT_STATE_CABLE_ISSUE, + ETHTOOL_LINK_EXT_SUBSTATE_CI_UNSUPPORTED_CABLE}, + {1025, ETHTOOL_LINK_EXT_STATE_CABLE_ISSUE, + ETHTOOL_LINK_EXT_SUBSTATE_CI_UNSUPPORTED_CABLE}, + {1029, ETHTOOL_LINK_EXT_STATE_CABLE_ISSUE, + ETHTOOL_LINK_EXT_SUBSTATE_CI_UNSUPPORTED_CABLE}, + {1031, ETHTOOL_LINK_EXT_STATE_CABLE_ISSUE, 0}, + + {1027, ETHTOOL_LINK_EXT_STATE_EEPROM_ISSUE, 0}, + + {23, ETHTOOL_LINK_EXT_STATE_CALIBRATION_FAILURE, 0}, + + {1032, ETHTOOL_LINK_EXT_STATE_POWER_BUDGET_EXCEEDED, 0}, + + {1030, ETHTOOL_LINK_EXT_STATE_OVERHEAT, 0}, +}; + +static void +mlxsw_sp_port_set_link_ext_state(struct mlxsw_sp_ethtool_link_ext_state_opcode_mapping + link_ext_state_mapping, + struct ethtool_link_ext_state_info *link_ext_state_info) +{ + switch (link_ext_state_mapping.link_ext_state) { + case ETHTOOL_LINK_EXT_STATE_AUTONEG: + link_ext_state_info->autoneg = + link_ext_state_mapping.link_ext_substate; + break; + case ETHTOOL_LINK_EXT_STATE_LINK_TRAINING_FAILURE: + link_ext_state_info->link_training = + link_ext_state_mapping.link_ext_substate; + break; + case ETHTOOL_LINK_EXT_STATE_LINK_LOGICAL_MISMATCH: + link_ext_state_info->link_logical_mismatch = + link_ext_state_mapping.link_ext_substate; + break; + case ETHTOOL_LINK_EXT_STATE_BAD_SIGNAL_INTEGRITY: + link_ext_state_info->bad_signal_integrity = + link_ext_state_mapping.link_ext_substate; + break; + case ETHTOOL_LINK_EXT_STATE_CABLE_ISSUE: + link_ext_state_info->cable_issue = + link_ext_state_mapping.link_ext_substate; + break; + default: + break; + } + + link_ext_state_info->link_ext_state = link_ext_state_mapping.link_ext_state; +} + +static int +mlxsw_sp_port_get_link_ext_state(struct net_device *dev, + struct ethtool_link_ext_state_info *link_ext_state_info) +{ + struct mlxsw_sp_ethtool_link_ext_state_opcode_mapping link_ext_state_mapping; + struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); + char pddr_pl[MLXSW_REG_PDDR_LEN]; + int opcode, err, i; + u32 status_opcode; + + if (netif_carrier_ok(dev)) + return -ENODATA; + + mlxsw_reg_pddr_pack(pddr_pl, mlxsw_sp_port->local_port, + MLXSW_REG_PDDR_PAGE_SELECT_TROUBLESHOOTING_INFO); + + opcode = MLXSW_REG_PDDR_TRBLSH_GROUP_OPCODE_MONITOR; + mlxsw_reg_pddr_trblsh_group_opcode_set(pddr_pl, opcode); + + err = mlxsw_reg_query(mlxsw_sp_port->mlxsw_sp->core, MLXSW_REG(pddr), + pddr_pl); + if (err) + return err; + + status_opcode = mlxsw_reg_pddr_trblsh_status_opcode_get(pddr_pl); + if (!status_opcode) + return -ENODATA; + + for (i = 0; i < ARRAY_SIZE(mlxsw_sp_link_ext_state_opcode_map); i++) { + link_ext_state_mapping = mlxsw_sp_link_ext_state_opcode_map[i]; + if (link_ext_state_mapping.status_opcode == status_opcode) { + mlxsw_sp_port_set_link_ext_state(link_ext_state_mapping, + link_ext_state_info); + return 0; + } + } + + return -ENODATA; +} + +static void mlxsw_sp_port_get_pauseparam(struct net_device *dev, + struct ethtool_pauseparam *pause) +{ + struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); + + pause->rx_pause = mlxsw_sp_port->link.rx_pause; + pause->tx_pause = mlxsw_sp_port->link.tx_pause; +} + +static int mlxsw_sp_port_pause_set(struct mlxsw_sp_port *mlxsw_sp_port, + struct ethtool_pauseparam *pause) +{ + char pfcc_pl[MLXSW_REG_PFCC_LEN]; + + mlxsw_reg_pfcc_pack(pfcc_pl, mlxsw_sp_port->local_port); + mlxsw_reg_pfcc_pprx_set(pfcc_pl, pause->rx_pause); + mlxsw_reg_pfcc_pptx_set(pfcc_pl, pause->tx_pause); + + return mlxsw_reg_write(mlxsw_sp_port->mlxsw_sp->core, MLXSW_REG(pfcc), + pfcc_pl); +} + +static int mlxsw_sp_port_set_pauseparam(struct net_device *dev, + struct ethtool_pauseparam *pause) +{ + struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); + bool pause_en = pause->tx_pause || pause->rx_pause; + int err; + + if (mlxsw_sp_port->dcb.pfc && mlxsw_sp_port->dcb.pfc->pfc_en) { + netdev_err(dev, "PFC already enabled on port\n"); + return -EINVAL; + } + + if (pause->autoneg) { + netdev_err(dev, "PAUSE frames autonegotiation isn't supported\n"); + return -EINVAL; + } + + err = mlxsw_sp_port_headroom_set(mlxsw_sp_port, dev->mtu, pause_en); + if (err) { + netdev_err(dev, "Failed to configure port's headroom\n"); + return err; + } + + err = mlxsw_sp_port_pause_set(mlxsw_sp_port, pause); + if (err) { + netdev_err(dev, "Failed to set PAUSE parameters\n"); + goto err_port_pause_configure; + } + + mlxsw_sp_port->link.rx_pause = pause->rx_pause; + mlxsw_sp_port->link.tx_pause = pause->tx_pause; + + return 0; + +err_port_pause_configure: + pause_en = mlxsw_sp_port_is_pause_en(mlxsw_sp_port); + mlxsw_sp_port_headroom_set(mlxsw_sp_port, dev->mtu, pause_en); + return err; +} + +struct mlxsw_sp_port_hw_stats { + char str[ETH_GSTRING_LEN]; + u64 (*getter)(const char *payload); + bool cells_bytes; +}; + +static struct mlxsw_sp_port_hw_stats mlxsw_sp_port_hw_stats[] = { + { + .str = "a_frames_transmitted_ok", + .getter = mlxsw_reg_ppcnt_a_frames_transmitted_ok_get, + }, + { + .str = "a_frames_received_ok", + .getter = mlxsw_reg_ppcnt_a_frames_received_ok_get, + }, + { + .str = "a_frame_check_sequence_errors", + .getter = mlxsw_reg_ppcnt_a_frame_check_sequence_errors_get, + }, + { + .str = "a_alignment_errors", + .getter = mlxsw_reg_ppcnt_a_alignment_errors_get, + }, + { + .str = "a_octets_transmitted_ok", + .getter = mlxsw_reg_ppcnt_a_octets_transmitted_ok_get, + }, + { + .str = "a_octets_received_ok", + .getter = mlxsw_reg_ppcnt_a_octets_received_ok_get, + }, + { + .str = "a_multicast_frames_xmitted_ok", + .getter = mlxsw_reg_ppcnt_a_multicast_frames_xmitted_ok_get, + }, + { + .str = "a_broadcast_frames_xmitted_ok", + .getter = mlxsw_reg_ppcnt_a_broadcast_frames_xmitted_ok_get, + }, + { + .str = "a_multicast_frames_received_ok", + .getter = mlxsw_reg_ppcnt_a_multicast_frames_received_ok_get, + }, + { + .str = "a_broadcast_frames_received_ok", + .getter = mlxsw_reg_ppcnt_a_broadcast_frames_received_ok_get, + }, + { + .str = "a_in_range_length_errors", + .getter = mlxsw_reg_ppcnt_a_in_range_length_errors_get, + }, + { + .str = "a_out_of_range_length_field", + .getter = mlxsw_reg_ppcnt_a_out_of_range_length_field_get, + }, + { + .str = "a_frame_too_long_errors", + .getter = mlxsw_reg_ppcnt_a_frame_too_long_errors_get, + }, + { + .str = "a_symbol_error_during_carrier", + .getter = mlxsw_reg_ppcnt_a_symbol_error_during_carrier_get, + }, + { + .str = "a_mac_control_frames_transmitted", + .getter = mlxsw_reg_ppcnt_a_mac_control_frames_transmitted_get, + }, + { + .str = "a_mac_control_frames_received", + .getter = mlxsw_reg_ppcnt_a_mac_control_frames_received_get, + }, + { + .str = "a_unsupported_opcodes_received", + .getter = mlxsw_reg_ppcnt_a_unsupported_opcodes_received_get, + }, + { + .str = "a_pause_mac_ctrl_frames_received", + .getter = mlxsw_reg_ppcnt_a_pause_mac_ctrl_frames_received_get, + }, + { + .str = "a_pause_mac_ctrl_frames_xmitted", + .getter = mlxsw_reg_ppcnt_a_pause_mac_ctrl_frames_transmitted_get, + }, +}; + +#define MLXSW_SP_PORT_HW_STATS_LEN ARRAY_SIZE(mlxsw_sp_port_hw_stats) + +static struct mlxsw_sp_port_hw_stats mlxsw_sp_port_hw_rfc_2863_stats[] = { + { + .str = "if_in_discards", + .getter = mlxsw_reg_ppcnt_if_in_discards_get, + }, + { + .str = "if_out_discards", + .getter = mlxsw_reg_ppcnt_if_out_discards_get, + }, + { + .str = "if_out_errors", + .getter = mlxsw_reg_ppcnt_if_out_errors_get, + }, +}; + +#define MLXSW_SP_PORT_HW_RFC_2863_STATS_LEN \ + ARRAY_SIZE(mlxsw_sp_port_hw_rfc_2863_stats) + +static struct mlxsw_sp_port_hw_stats mlxsw_sp_port_hw_rfc_2819_stats[] = { + { + .str = "ether_stats_undersize_pkts", + .getter = mlxsw_reg_ppcnt_ether_stats_undersize_pkts_get, + }, + { + .str = "ether_stats_oversize_pkts", + .getter = mlxsw_reg_ppcnt_ether_stats_oversize_pkts_get, + }, + { + .str = "ether_stats_fragments", + .getter = mlxsw_reg_ppcnt_ether_stats_fragments_get, + }, + { + .str = "ether_pkts64octets", + .getter = mlxsw_reg_ppcnt_ether_stats_pkts64octets_get, + }, + { + .str = "ether_pkts65to127octets", + .getter = mlxsw_reg_ppcnt_ether_stats_pkts65to127octets_get, + }, + { + .str = "ether_pkts128to255octets", + .getter = mlxsw_reg_ppcnt_ether_stats_pkts128to255octets_get, + }, + { + .str = "ether_pkts256to511octets", + .getter = mlxsw_reg_ppcnt_ether_stats_pkts256to511octets_get, + }, + { + .str = "ether_pkts512to1023octets", + .getter = mlxsw_reg_ppcnt_ether_stats_pkts512to1023octets_get, + }, + { + .str = "ether_pkts1024to1518octets", + .getter = mlxsw_reg_ppcnt_ether_stats_pkts1024to1518octets_get, + }, + { + .str = "ether_pkts1519to2047octets", + .getter = mlxsw_reg_ppcnt_ether_stats_pkts1519to2047octets_get, + }, + { + .str = "ether_pkts2048to4095octets", + .getter = mlxsw_reg_ppcnt_ether_stats_pkts2048to4095octets_get, + }, + { + .str = "ether_pkts4096to8191octets", + .getter = mlxsw_reg_ppcnt_ether_stats_pkts4096to8191octets_get, + }, + { + .str = "ether_pkts8192to10239octets", + .getter = mlxsw_reg_ppcnt_ether_stats_pkts8192to10239octets_get, + }, +}; + +#define MLXSW_SP_PORT_HW_RFC_2819_STATS_LEN \ + ARRAY_SIZE(mlxsw_sp_port_hw_rfc_2819_stats) + +static struct mlxsw_sp_port_hw_stats mlxsw_sp_port_hw_rfc_3635_stats[] = { + { + .str = "dot3stats_fcs_errors", + .getter = mlxsw_reg_ppcnt_dot3stats_fcs_errors_get, + }, + { + .str = "dot3stats_symbol_errors", + .getter = mlxsw_reg_ppcnt_dot3stats_symbol_errors_get, + }, + { + .str = "dot3control_in_unknown_opcodes", + .getter = mlxsw_reg_ppcnt_dot3control_in_unknown_opcodes_get, + }, + { + .str = "dot3in_pause_frames", + .getter = mlxsw_reg_ppcnt_dot3in_pause_frames_get, + }, +}; + +#define MLXSW_SP_PORT_HW_RFC_3635_STATS_LEN \ + ARRAY_SIZE(mlxsw_sp_port_hw_rfc_3635_stats) + +static struct mlxsw_sp_port_hw_stats mlxsw_sp_port_hw_ext_stats[] = { + { + .str = "ecn_marked", + .getter = mlxsw_reg_ppcnt_ecn_marked_get, + }, +}; + +#define MLXSW_SP_PORT_HW_EXT_STATS_LEN ARRAY_SIZE(mlxsw_sp_port_hw_ext_stats) + +static struct mlxsw_sp_port_hw_stats mlxsw_sp_port_hw_discard_stats[] = { + { + .str = "discard_ingress_general", + .getter = mlxsw_reg_ppcnt_ingress_general_get, + }, + { + .str = "discard_ingress_policy_engine", + .getter = mlxsw_reg_ppcnt_ingress_policy_engine_get, + }, + { + .str = "discard_ingress_vlan_membership", + .getter = mlxsw_reg_ppcnt_ingress_vlan_membership_get, + }, + { + .str = "discard_ingress_tag_frame_type", + .getter = mlxsw_reg_ppcnt_ingress_tag_frame_type_get, + }, + { + .str = "discard_egress_vlan_membership", + .getter = mlxsw_reg_ppcnt_egress_vlan_membership_get, + }, + { + .str = "discard_loopback_filter", + .getter = mlxsw_reg_ppcnt_loopback_filter_get, + }, + { + .str = "discard_egress_general", + .getter = mlxsw_reg_ppcnt_egress_general_get, + }, + { + .str = "discard_egress_hoq", + .getter = mlxsw_reg_ppcnt_egress_hoq_get, + }, + { + .str = "discard_egress_policy_engine", + .getter = mlxsw_reg_ppcnt_egress_policy_engine_get, + }, + { + .str = "discard_ingress_tx_link_down", + .getter = mlxsw_reg_ppcnt_ingress_tx_link_down_get, + }, + { + .str = "discard_egress_stp_filter", + .getter = mlxsw_reg_ppcnt_egress_stp_filter_get, + }, + { + .str = "discard_egress_sll", + .getter = mlxsw_reg_ppcnt_egress_sll_get, + }, +}; + +#define MLXSW_SP_PORT_HW_DISCARD_STATS_LEN \ + ARRAY_SIZE(mlxsw_sp_port_hw_discard_stats) + +static struct mlxsw_sp_port_hw_stats mlxsw_sp_port_hw_prio_stats[] = { + { + .str = "rx_octets_prio", + .getter = mlxsw_reg_ppcnt_rx_octets_get, + }, + { + .str = "rx_frames_prio", + .getter = mlxsw_reg_ppcnt_rx_frames_get, + }, + { + .str = "tx_octets_prio", + .getter = mlxsw_reg_ppcnt_tx_octets_get, + }, + { + .str = "tx_frames_prio", + .getter = mlxsw_reg_ppcnt_tx_frames_get, + }, + { + .str = "rx_pause_prio", + .getter = mlxsw_reg_ppcnt_rx_pause_get, + }, + { + .str = "rx_pause_duration_prio", + .getter = mlxsw_reg_ppcnt_rx_pause_duration_get, + }, + { + .str = "tx_pause_prio", + .getter = mlxsw_reg_ppcnt_tx_pause_get, + }, + { + .str = "tx_pause_duration_prio", + .getter = mlxsw_reg_ppcnt_tx_pause_duration_get, + }, +}; + +#define MLXSW_SP_PORT_HW_PRIO_STATS_LEN ARRAY_SIZE(mlxsw_sp_port_hw_prio_stats) + +static struct mlxsw_sp_port_hw_stats mlxsw_sp_port_hw_tc_stats[] = { + { + .str = "tc_transmit_queue_tc", + .getter = mlxsw_reg_ppcnt_tc_transmit_queue_get, + .cells_bytes = true, + }, + { + .str = "tc_no_buffer_discard_uc_tc", + .getter = mlxsw_reg_ppcnt_tc_no_buffer_discard_uc_get, + }, +}; + +#define MLXSW_SP_PORT_HW_TC_STATS_LEN ARRAY_SIZE(mlxsw_sp_port_hw_tc_stats) + +#define MLXSW_SP_PORT_ETHTOOL_STATS_LEN (MLXSW_SP_PORT_HW_STATS_LEN + \ + MLXSW_SP_PORT_HW_RFC_2863_STATS_LEN + \ + MLXSW_SP_PORT_HW_RFC_2819_STATS_LEN + \ + MLXSW_SP_PORT_HW_RFC_3635_STATS_LEN + \ + MLXSW_SP_PORT_HW_EXT_STATS_LEN + \ + MLXSW_SP_PORT_HW_DISCARD_STATS_LEN + \ + (MLXSW_SP_PORT_HW_PRIO_STATS_LEN * \ + IEEE_8021QAZ_MAX_TCS) + \ + (MLXSW_SP_PORT_HW_TC_STATS_LEN * \ + TC_MAX_QUEUE)) + +static void mlxsw_sp_port_get_prio_strings(u8 **p, int prio) +{ + int i; + + for (i = 0; i < MLXSW_SP_PORT_HW_PRIO_STATS_LEN; i++) { + snprintf(*p, ETH_GSTRING_LEN, "%.29s_%.1d", + mlxsw_sp_port_hw_prio_stats[i].str, prio); + *p += ETH_GSTRING_LEN; + } +} + +static void mlxsw_sp_port_get_tc_strings(u8 **p, int tc) +{ + int i; + + for (i = 0; i < MLXSW_SP_PORT_HW_TC_STATS_LEN; i++) { + snprintf(*p, ETH_GSTRING_LEN, "%.29s_%.1d", + mlxsw_sp_port_hw_tc_stats[i].str, tc); + *p += ETH_GSTRING_LEN; + } +} + +static void mlxsw_sp_port_get_strings(struct net_device *dev, + u32 stringset, u8 *data) +{ + struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); + u8 *p = data; + int i; + + switch (stringset) { + case ETH_SS_STATS: + for (i = 0; i < MLXSW_SP_PORT_HW_STATS_LEN; i++) { + memcpy(p, mlxsw_sp_port_hw_stats[i].str, + ETH_GSTRING_LEN); + p += ETH_GSTRING_LEN; + } + + for (i = 0; i < MLXSW_SP_PORT_HW_RFC_2863_STATS_LEN; i++) { + memcpy(p, mlxsw_sp_port_hw_rfc_2863_stats[i].str, + ETH_GSTRING_LEN); + p += ETH_GSTRING_LEN; + } + + for (i = 0; i < MLXSW_SP_PORT_HW_RFC_2819_STATS_LEN; i++) { + memcpy(p, mlxsw_sp_port_hw_rfc_2819_stats[i].str, + ETH_GSTRING_LEN); + p += ETH_GSTRING_LEN; + } + + for (i = 0; i < MLXSW_SP_PORT_HW_RFC_3635_STATS_LEN; i++) { + memcpy(p, mlxsw_sp_port_hw_rfc_3635_stats[i].str, + ETH_GSTRING_LEN); + p += ETH_GSTRING_LEN; + } + + for (i = 0; i < MLXSW_SP_PORT_HW_EXT_STATS_LEN; i++) { + memcpy(p, mlxsw_sp_port_hw_ext_stats[i].str, + ETH_GSTRING_LEN); + p += ETH_GSTRING_LEN; + } + + for (i = 0; i < MLXSW_SP_PORT_HW_DISCARD_STATS_LEN; i++) { + memcpy(p, mlxsw_sp_port_hw_discard_stats[i].str, + ETH_GSTRING_LEN); + p += ETH_GSTRING_LEN; + } + + for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) + mlxsw_sp_port_get_prio_strings(&p, i); + + for (i = 0; i < TC_MAX_QUEUE; i++) + mlxsw_sp_port_get_tc_strings(&p, i); + + mlxsw_sp_port->mlxsw_sp->ptp_ops->get_stats_strings(&p); + break; + } +} + +static int mlxsw_sp_port_set_phys_id(struct net_device *dev, + enum ethtool_phys_id_state state) +{ + struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); + struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; + char mlcr_pl[MLXSW_REG_MLCR_LEN]; + bool active; + + switch (state) { + case ETHTOOL_ID_ACTIVE: + active = true; + break; + case ETHTOOL_ID_INACTIVE: + active = false; + break; + default: + return -EOPNOTSUPP; + } + + mlxsw_reg_mlcr_pack(mlcr_pl, mlxsw_sp_port->local_port, active); + return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(mlcr), mlcr_pl); +} + +static int +mlxsw_sp_get_hw_stats_by_group(struct mlxsw_sp_port_hw_stats **p_hw_stats, + int *p_len, enum mlxsw_reg_ppcnt_grp grp) +{ + switch (grp) { + case MLXSW_REG_PPCNT_IEEE_8023_CNT: + *p_hw_stats = mlxsw_sp_port_hw_stats; + *p_len = MLXSW_SP_PORT_HW_STATS_LEN; + break; + case MLXSW_REG_PPCNT_RFC_2863_CNT: + *p_hw_stats = mlxsw_sp_port_hw_rfc_2863_stats; + *p_len = MLXSW_SP_PORT_HW_RFC_2863_STATS_LEN; + break; + case MLXSW_REG_PPCNT_RFC_2819_CNT: + *p_hw_stats = mlxsw_sp_port_hw_rfc_2819_stats; + *p_len = MLXSW_SP_PORT_HW_RFC_2819_STATS_LEN; + break; + case MLXSW_REG_PPCNT_RFC_3635_CNT: + *p_hw_stats = mlxsw_sp_port_hw_rfc_3635_stats; + *p_len = MLXSW_SP_PORT_HW_RFC_3635_STATS_LEN; + break; + case MLXSW_REG_PPCNT_EXT_CNT: + *p_hw_stats = mlxsw_sp_port_hw_ext_stats; + *p_len = MLXSW_SP_PORT_HW_EXT_STATS_LEN; + break; + case MLXSW_REG_PPCNT_DISCARD_CNT: + *p_hw_stats = mlxsw_sp_port_hw_discard_stats; + *p_len = MLXSW_SP_PORT_HW_DISCARD_STATS_LEN; + break; + case MLXSW_REG_PPCNT_PRIO_CNT: + *p_hw_stats = mlxsw_sp_port_hw_prio_stats; + *p_len = MLXSW_SP_PORT_HW_PRIO_STATS_LEN; + break; + case MLXSW_REG_PPCNT_TC_CNT: + *p_hw_stats = mlxsw_sp_port_hw_tc_stats; + *p_len = MLXSW_SP_PORT_HW_TC_STATS_LEN; + break; + default: + WARN_ON(1); + return -EOPNOTSUPP; + } + return 0; +} + +static void __mlxsw_sp_port_get_stats(struct net_device *dev, + enum mlxsw_reg_ppcnt_grp grp, int prio, + u64 *data, int data_index) +{ + struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); + struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; + struct mlxsw_sp_port_hw_stats *hw_stats; + char ppcnt_pl[MLXSW_REG_PPCNT_LEN]; + int i, len; + int err; + + err = mlxsw_sp_get_hw_stats_by_group(&hw_stats, &len, grp); + if (err) + return; + mlxsw_sp_port_get_stats_raw(dev, grp, prio, ppcnt_pl); + for (i = 0; i < len; i++) { + data[data_index + i] = hw_stats[i].getter(ppcnt_pl); + if (!hw_stats[i].cells_bytes) + continue; + data[data_index + i] = mlxsw_sp_cells_bytes(mlxsw_sp, + data[data_index + i]); + } +} + +static void mlxsw_sp_port_get_stats(struct net_device *dev, + struct ethtool_stats *stats, u64 *data) +{ + struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); + int i, data_index = 0; + + /* IEEE 802.3 Counters */ + __mlxsw_sp_port_get_stats(dev, MLXSW_REG_PPCNT_IEEE_8023_CNT, 0, + data, data_index); + data_index = MLXSW_SP_PORT_HW_STATS_LEN; + + /* RFC 2863 Counters */ + __mlxsw_sp_port_get_stats(dev, MLXSW_REG_PPCNT_RFC_2863_CNT, 0, + data, data_index); + data_index += MLXSW_SP_PORT_HW_RFC_2863_STATS_LEN; + + /* RFC 2819 Counters */ + __mlxsw_sp_port_get_stats(dev, MLXSW_REG_PPCNT_RFC_2819_CNT, 0, + data, data_index); + data_index += MLXSW_SP_PORT_HW_RFC_2819_STATS_LEN; + + /* RFC 3635 Counters */ + __mlxsw_sp_port_get_stats(dev, MLXSW_REG_PPCNT_RFC_3635_CNT, 0, + data, data_index); + data_index += MLXSW_SP_PORT_HW_RFC_3635_STATS_LEN; + + /* Extended Counters */ + __mlxsw_sp_port_get_stats(dev, MLXSW_REG_PPCNT_EXT_CNT, 0, + data, data_index); + data_index += MLXSW_SP_PORT_HW_EXT_STATS_LEN; + + /* Discard Counters */ + __mlxsw_sp_port_get_stats(dev, MLXSW_REG_PPCNT_DISCARD_CNT, 0, + data, data_index); + data_index += MLXSW_SP_PORT_HW_DISCARD_STATS_LEN; + + /* Per-Priority Counters */ + for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) { + __mlxsw_sp_port_get_stats(dev, MLXSW_REG_PPCNT_PRIO_CNT, i, + data, data_index); + data_index += MLXSW_SP_PORT_HW_PRIO_STATS_LEN; + } + + /* Per-TC Counters */ + for (i = 0; i < TC_MAX_QUEUE; i++) { + __mlxsw_sp_port_get_stats(dev, MLXSW_REG_PPCNT_TC_CNT, i, + data, data_index); + data_index += MLXSW_SP_PORT_HW_TC_STATS_LEN; + } + + /* PTP counters */ + mlxsw_sp_port->mlxsw_sp->ptp_ops->get_stats(mlxsw_sp_port, + data, data_index); + data_index += mlxsw_sp_port->mlxsw_sp->ptp_ops->get_stats_count(); +} + +static int mlxsw_sp_port_get_sset_count(struct net_device *dev, int sset) +{ + struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); + + switch (sset) { + case ETH_SS_STATS: + return MLXSW_SP_PORT_ETHTOOL_STATS_LEN + + mlxsw_sp_port->mlxsw_sp->ptp_ops->get_stats_count(); + default: + return -EOPNOTSUPP; + } +} + +static void +mlxsw_sp_port_get_link_supported(struct mlxsw_sp *mlxsw_sp, u32 eth_proto_cap, + u8 width, struct ethtool_link_ksettings *cmd) +{ + const struct mlxsw_sp_port_type_speed_ops *ops; + + ops = mlxsw_sp->port_type_speed_ops; + + ethtool_link_ksettings_add_link_mode(cmd, supported, Asym_Pause); + ethtool_link_ksettings_add_link_mode(cmd, supported, Autoneg); + ethtool_link_ksettings_add_link_mode(cmd, supported, Pause); + + ops->from_ptys_supported_port(mlxsw_sp, eth_proto_cap, cmd); + ops->from_ptys_link(mlxsw_sp, eth_proto_cap, width, + cmd->link_modes.supported); +} + +static void +mlxsw_sp_port_get_link_advertise(struct mlxsw_sp *mlxsw_sp, + u32 eth_proto_admin, bool autoneg, u8 width, + struct ethtool_link_ksettings *cmd) +{ + const struct mlxsw_sp_port_type_speed_ops *ops; + + ops = mlxsw_sp->port_type_speed_ops; + + if (!autoneg) + return; + + ethtool_link_ksettings_add_link_mode(cmd, advertising, Autoneg); + ops->from_ptys_link(mlxsw_sp, eth_proto_admin, width, + cmd->link_modes.advertising); +} + +static u8 +mlxsw_sp_port_connector_port(enum mlxsw_reg_ptys_connector_type connector_type) +{ + switch (connector_type) { + case MLXSW_REG_PTYS_CONNECTOR_TYPE_UNKNOWN_OR_NO_CONNECTOR: + return PORT_OTHER; + case MLXSW_REG_PTYS_CONNECTOR_TYPE_PORT_NONE: + return PORT_NONE; + case MLXSW_REG_PTYS_CONNECTOR_TYPE_PORT_TP: + return PORT_TP; + case MLXSW_REG_PTYS_CONNECTOR_TYPE_PORT_AUI: + return PORT_AUI; + case MLXSW_REG_PTYS_CONNECTOR_TYPE_PORT_BNC: + return PORT_BNC; + case MLXSW_REG_PTYS_CONNECTOR_TYPE_PORT_MII: + return PORT_MII; + case MLXSW_REG_PTYS_CONNECTOR_TYPE_PORT_FIBRE: + return PORT_FIBRE; + case MLXSW_REG_PTYS_CONNECTOR_TYPE_PORT_DA: + return PORT_DA; + case MLXSW_REG_PTYS_CONNECTOR_TYPE_PORT_OTHER: + return PORT_OTHER; + default: + WARN_ON_ONCE(1); + return PORT_OTHER; + } +} + +static int mlxsw_sp_port_get_link_ksettings(struct net_device *dev, + struct ethtool_link_ksettings *cmd) +{ + u32 eth_proto_cap, eth_proto_admin, eth_proto_oper; + struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); + struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; + const struct mlxsw_sp_port_type_speed_ops *ops; + char ptys_pl[MLXSW_REG_PTYS_LEN]; + u8 connector_type; + bool autoneg; + int err; + + ops = mlxsw_sp->port_type_speed_ops; + + autoneg = mlxsw_sp_port->link.autoneg; + ops->reg_ptys_eth_pack(mlxsw_sp, ptys_pl, mlxsw_sp_port->local_port, + 0, false); + err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ptys), ptys_pl); + if (err) + return err; + ops->reg_ptys_eth_unpack(mlxsw_sp, ptys_pl, ð_proto_cap, + ð_proto_admin, ð_proto_oper); + + mlxsw_sp_port_get_link_supported(mlxsw_sp, eth_proto_cap, + mlxsw_sp_port->mapping.width, cmd); + + mlxsw_sp_port_get_link_advertise(mlxsw_sp, eth_proto_admin, autoneg, + mlxsw_sp_port->mapping.width, cmd); + + cmd->base.autoneg = autoneg ? AUTONEG_ENABLE : AUTONEG_DISABLE; + connector_type = mlxsw_reg_ptys_connector_type_get(ptys_pl); + cmd->base.port = mlxsw_sp_port_connector_port(connector_type); + ops->from_ptys_speed_duplex(mlxsw_sp, netif_carrier_ok(dev), + eth_proto_oper, cmd); + + return 0; +} + +static int +mlxsw_sp_port_set_link_ksettings(struct net_device *dev, + const struct ethtool_link_ksettings *cmd) +{ + struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); + struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; + const struct mlxsw_sp_port_type_speed_ops *ops; + char ptys_pl[MLXSW_REG_PTYS_LEN]; + u32 eth_proto_cap, eth_proto_new; + bool autoneg; + int err; + + ops = mlxsw_sp->port_type_speed_ops; + + ops->reg_ptys_eth_pack(mlxsw_sp, ptys_pl, mlxsw_sp_port->local_port, + 0, false); + err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ptys), ptys_pl); + if (err) + return err; + ops->reg_ptys_eth_unpack(mlxsw_sp, ptys_pl, ð_proto_cap, NULL, NULL); + + autoneg = cmd->base.autoneg == AUTONEG_ENABLE; + eth_proto_new = autoneg ? + ops->to_ptys_advert_link(mlxsw_sp, mlxsw_sp_port->mapping.width, + cmd) : + ops->to_ptys_speed(mlxsw_sp, mlxsw_sp_port->mapping.width, + cmd->base.speed); + + eth_proto_new = eth_proto_new & eth_proto_cap; + if (!eth_proto_new) { + netdev_err(dev, "No supported speed requested\n"); + return -EINVAL; + } + + ops->reg_ptys_eth_pack(mlxsw_sp, ptys_pl, mlxsw_sp_port->local_port, + eth_proto_new, autoneg); + err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptys), ptys_pl); + if (err) + return err; + + mlxsw_sp_port->link.autoneg = autoneg; + + if (!netif_running(dev)) + return 0; + + mlxsw_sp_port_admin_status_set(mlxsw_sp_port, false); + mlxsw_sp_port_admin_status_set(mlxsw_sp_port, true); + + return 0; +} + +static int mlxsw_sp_get_module_info(struct net_device *netdev, + struct ethtool_modinfo *modinfo) +{ + struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(netdev); + struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; + int err; + + err = mlxsw_env_get_module_info(mlxsw_sp->core, + mlxsw_sp_port->mapping.module, + modinfo); + + return err; +} + +static int mlxsw_sp_get_module_eeprom(struct net_device *netdev, + struct ethtool_eeprom *ee, u8 *data) +{ + struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(netdev); + struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; + int err; + + err = mlxsw_env_get_module_eeprom(netdev, mlxsw_sp->core, + mlxsw_sp_port->mapping.module, ee, + data); + + return err; +} + +static int +mlxsw_sp_get_ts_info(struct net_device *netdev, struct ethtool_ts_info *info) +{ + struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(netdev); + struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; + + return mlxsw_sp->ptp_ops->get_ts_info(mlxsw_sp, info); +} + +const struct ethtool_ops mlxsw_sp_port_ethtool_ops = { + .get_drvinfo = mlxsw_sp_port_get_drvinfo, + .get_link = ethtool_op_get_link, + .get_link_ext_state = mlxsw_sp_port_get_link_ext_state, + .get_pauseparam = mlxsw_sp_port_get_pauseparam, + .set_pauseparam = mlxsw_sp_port_set_pauseparam, + .get_strings = mlxsw_sp_port_get_strings, + .set_phys_id = mlxsw_sp_port_set_phys_id, + .get_ethtool_stats = mlxsw_sp_port_get_stats, + .get_sset_count = mlxsw_sp_port_get_sset_count, + .get_link_ksettings = mlxsw_sp_port_get_link_ksettings, + .set_link_ksettings = mlxsw_sp_port_set_link_ksettings, + .get_module_info = mlxsw_sp_get_module_info, + .get_module_eeprom = mlxsw_sp_get_module_eeprom, + .get_ts_info = mlxsw_sp_get_ts_info, +}; + +struct mlxsw_sp1_port_link_mode { + enum ethtool_link_mode_bit_indices mask_ethtool; + u32 mask; + u32 speed; +}; + +static const struct mlxsw_sp1_port_link_mode mlxsw_sp1_port_link_mode[] = { + { + .mask = MLXSW_REG_PTYS_ETH_SPEED_100BASE_T, + .mask_ethtool = ETHTOOL_LINK_MODE_100baseT_Full_BIT, + .speed = SPEED_100, + }, + { + .mask = MLXSW_REG_PTYS_ETH_SPEED_SGMII | + MLXSW_REG_PTYS_ETH_SPEED_1000BASE_KX, + .mask_ethtool = ETHTOOL_LINK_MODE_1000baseKX_Full_BIT, + .speed = SPEED_1000, + }, + { + .mask = MLXSW_REG_PTYS_ETH_SPEED_10GBASE_T, + .mask_ethtool = ETHTOOL_LINK_MODE_10000baseT_Full_BIT, + .speed = SPEED_10000, + }, + { + .mask = MLXSW_REG_PTYS_ETH_SPEED_10GBASE_CX4 | + MLXSW_REG_PTYS_ETH_SPEED_10GBASE_KX4, + .mask_ethtool = ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT, + .speed = SPEED_10000, + }, + { + .mask = MLXSW_REG_PTYS_ETH_SPEED_10GBASE_KR | + MLXSW_REG_PTYS_ETH_SPEED_10GBASE_CR | + MLXSW_REG_PTYS_ETH_SPEED_10GBASE_SR | + MLXSW_REG_PTYS_ETH_SPEED_10GBASE_ER_LR, + .mask_ethtool = ETHTOOL_LINK_MODE_10000baseKR_Full_BIT, + .speed = SPEED_10000, + }, + { + .mask = MLXSW_REG_PTYS_ETH_SPEED_20GBASE_KR2, + .mask_ethtool = ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT, + .speed = SPEED_20000, + }, + { + .mask = MLXSW_REG_PTYS_ETH_SPEED_40GBASE_CR4, + .mask_ethtool = ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT, + .speed = SPEED_40000, + }, + { + .mask = MLXSW_REG_PTYS_ETH_SPEED_40GBASE_KR4, + .mask_ethtool = ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT, + .speed = SPEED_40000, + }, + { + .mask = MLXSW_REG_PTYS_ETH_SPEED_40GBASE_SR4, + .mask_ethtool = ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT, + .speed = SPEED_40000, + }, + { + .mask = MLXSW_REG_PTYS_ETH_SPEED_40GBASE_LR4_ER4, + .mask_ethtool = ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT, + .speed = SPEED_40000, + }, + { + .mask = MLXSW_REG_PTYS_ETH_SPEED_25GBASE_CR, + .mask_ethtool = ETHTOOL_LINK_MODE_25000baseCR_Full_BIT, + .speed = SPEED_25000, + }, + { + .mask = MLXSW_REG_PTYS_ETH_SPEED_25GBASE_KR, + .mask_ethtool = ETHTOOL_LINK_MODE_25000baseKR_Full_BIT, + .speed = SPEED_25000, + }, + { + .mask = MLXSW_REG_PTYS_ETH_SPEED_25GBASE_SR, + .mask_ethtool = ETHTOOL_LINK_MODE_25000baseSR_Full_BIT, + .speed = SPEED_25000, + }, + { + .mask = MLXSW_REG_PTYS_ETH_SPEED_50GBASE_CR2, + .mask_ethtool = ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT, + .speed = SPEED_50000, + }, + { + .mask = MLXSW_REG_PTYS_ETH_SPEED_50GBASE_KR2, + .mask_ethtool = ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT, + .speed = SPEED_50000, + }, + { + .mask = MLXSW_REG_PTYS_ETH_SPEED_50GBASE_SR2, + .mask_ethtool = ETHTOOL_LINK_MODE_50000baseSR2_Full_BIT, + .speed = SPEED_50000, + }, + { + .mask = MLXSW_REG_PTYS_ETH_SPEED_100GBASE_CR4, + .mask_ethtool = ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT, + .speed = SPEED_100000, + }, + { + .mask = MLXSW_REG_PTYS_ETH_SPEED_100GBASE_SR4, + .mask_ethtool = ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT, + .speed = SPEED_100000, + }, + { + .mask = MLXSW_REG_PTYS_ETH_SPEED_100GBASE_KR4, + .mask_ethtool = ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT, + .speed = SPEED_100000, + }, + { + .mask = MLXSW_REG_PTYS_ETH_SPEED_100GBASE_LR4_ER4, + .mask_ethtool = ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT, + .speed = SPEED_100000, + }, +}; + +#define MLXSW_SP1_PORT_LINK_MODE_LEN ARRAY_SIZE(mlxsw_sp1_port_link_mode) + +static void +mlxsw_sp1_from_ptys_supported_port(struct mlxsw_sp *mlxsw_sp, + u32 ptys_eth_proto, + struct ethtool_link_ksettings *cmd) +{ + if (ptys_eth_proto & (MLXSW_REG_PTYS_ETH_SPEED_10GBASE_CR | + MLXSW_REG_PTYS_ETH_SPEED_10GBASE_SR | + MLXSW_REG_PTYS_ETH_SPEED_40GBASE_CR4 | + MLXSW_REG_PTYS_ETH_SPEED_40GBASE_SR4 | + MLXSW_REG_PTYS_ETH_SPEED_100GBASE_SR4 | + MLXSW_REG_PTYS_ETH_SPEED_SGMII)) + ethtool_link_ksettings_add_link_mode(cmd, supported, FIBRE); + + if (ptys_eth_proto & (MLXSW_REG_PTYS_ETH_SPEED_10GBASE_KR | + MLXSW_REG_PTYS_ETH_SPEED_10GBASE_KX4 | + MLXSW_REG_PTYS_ETH_SPEED_40GBASE_KR4 | + MLXSW_REG_PTYS_ETH_SPEED_100GBASE_KR4 | + MLXSW_REG_PTYS_ETH_SPEED_1000BASE_KX)) + ethtool_link_ksettings_add_link_mode(cmd, supported, Backplane); +} + +static void +mlxsw_sp1_from_ptys_link(struct mlxsw_sp *mlxsw_sp, u32 ptys_eth_proto, + u8 width, unsigned long *mode) +{ + int i; + + for (i = 0; i < MLXSW_SP1_PORT_LINK_MODE_LEN; i++) { + if (ptys_eth_proto & mlxsw_sp1_port_link_mode[i].mask) + __set_bit(mlxsw_sp1_port_link_mode[i].mask_ethtool, + mode); + } +} + +static u32 +mlxsw_sp1_from_ptys_speed(struct mlxsw_sp *mlxsw_sp, u32 ptys_eth_proto) +{ + int i; + + for (i = 0; i < MLXSW_SP1_PORT_LINK_MODE_LEN; i++) { + if (ptys_eth_proto & mlxsw_sp1_port_link_mode[i].mask) + return mlxsw_sp1_port_link_mode[i].speed; + } + + return SPEED_UNKNOWN; +} + +static void +mlxsw_sp1_from_ptys_speed_duplex(struct mlxsw_sp *mlxsw_sp, bool carrier_ok, + u32 ptys_eth_proto, + struct ethtool_link_ksettings *cmd) +{ + cmd->base.speed = SPEED_UNKNOWN; + cmd->base.duplex = DUPLEX_UNKNOWN; + + if (!carrier_ok) + return; + + cmd->base.speed = mlxsw_sp1_from_ptys_speed(mlxsw_sp, ptys_eth_proto); + if (cmd->base.speed != SPEED_UNKNOWN) + cmd->base.duplex = DUPLEX_FULL; +} + +static u32 +mlxsw_sp1_to_ptys_advert_link(struct mlxsw_sp *mlxsw_sp, u8 width, + const struct ethtool_link_ksettings *cmd) +{ + u32 ptys_proto = 0; + int i; + + for (i = 0; i < MLXSW_SP1_PORT_LINK_MODE_LEN; i++) { + if (test_bit(mlxsw_sp1_port_link_mode[i].mask_ethtool, + cmd->link_modes.advertising)) + ptys_proto |= mlxsw_sp1_port_link_mode[i].mask; + } + return ptys_proto; +} + +static u32 mlxsw_sp1_to_ptys_speed(struct mlxsw_sp *mlxsw_sp, u8 width, + u32 speed) +{ + u32 ptys_proto = 0; + int i; + + for (i = 0; i < MLXSW_SP1_PORT_LINK_MODE_LEN; i++) { + if (speed == mlxsw_sp1_port_link_mode[i].speed) + ptys_proto |= mlxsw_sp1_port_link_mode[i].mask; + } + return ptys_proto; +} + +static void +mlxsw_sp1_reg_ptys_eth_pack(struct mlxsw_sp *mlxsw_sp, char *payload, + u8 local_port, u32 proto_admin, bool autoneg) +{ + mlxsw_reg_ptys_eth_pack(payload, local_port, proto_admin, autoneg); +} + +static void +mlxsw_sp1_reg_ptys_eth_unpack(struct mlxsw_sp *mlxsw_sp, char *payload, + u32 *p_eth_proto_cap, u32 *p_eth_proto_admin, + u32 *p_eth_proto_oper) +{ + mlxsw_reg_ptys_eth_unpack(payload, p_eth_proto_cap, p_eth_proto_admin, + p_eth_proto_oper); +} + +const struct mlxsw_sp_port_type_speed_ops mlxsw_sp1_port_type_speed_ops = { + .from_ptys_supported_port = mlxsw_sp1_from_ptys_supported_port, + .from_ptys_link = mlxsw_sp1_from_ptys_link, + .from_ptys_speed = mlxsw_sp1_from_ptys_speed, + .from_ptys_speed_duplex = mlxsw_sp1_from_ptys_speed_duplex, + .to_ptys_advert_link = mlxsw_sp1_to_ptys_advert_link, + .to_ptys_speed = mlxsw_sp1_to_ptys_speed, + .reg_ptys_eth_pack = mlxsw_sp1_reg_ptys_eth_pack, + .reg_ptys_eth_unpack = mlxsw_sp1_reg_ptys_eth_unpack, +}; + +static const enum ethtool_link_mode_bit_indices +mlxsw_sp2_mask_ethtool_sgmii_100m[] = { + ETHTOOL_LINK_MODE_100baseT_Full_BIT, +}; + +#define MLXSW_SP2_MASK_ETHTOOL_SGMII_100M_LEN \ + ARRAY_SIZE(mlxsw_sp2_mask_ethtool_sgmii_100m) + +static const enum ethtool_link_mode_bit_indices +mlxsw_sp2_mask_ethtool_1000base_x_sgmii[] = { + ETHTOOL_LINK_MODE_1000baseT_Full_BIT, + ETHTOOL_LINK_MODE_1000baseKX_Full_BIT, +}; + +#define MLXSW_SP2_MASK_ETHTOOL_1000BASE_X_SGMII_LEN \ + ARRAY_SIZE(mlxsw_sp2_mask_ethtool_1000base_x_sgmii) + +static const enum ethtool_link_mode_bit_indices +mlxsw_sp2_mask_ethtool_2_5gbase_x_2_5gmii[] = { + ETHTOOL_LINK_MODE_2500baseX_Full_BIT, +}; + +#define MLXSW_SP2_MASK_ETHTOOL_2_5GBASE_X_2_5GMII_LEN \ + ARRAY_SIZE(mlxsw_sp2_mask_ethtool_2_5gbase_x_2_5gmii) + +static const enum ethtool_link_mode_bit_indices +mlxsw_sp2_mask_ethtool_5gbase_r[] = { + ETHTOOL_LINK_MODE_5000baseT_Full_BIT, +}; + +#define MLXSW_SP2_MASK_ETHTOOL_5GBASE_R_LEN \ + ARRAY_SIZE(mlxsw_sp2_mask_ethtool_5gbase_r) + +static const enum ethtool_link_mode_bit_indices +mlxsw_sp2_mask_ethtool_xfi_xaui_1_10g[] = { + ETHTOOL_LINK_MODE_10000baseT_Full_BIT, + ETHTOOL_LINK_MODE_10000baseKR_Full_BIT, + ETHTOOL_LINK_MODE_10000baseR_FEC_BIT, + ETHTOOL_LINK_MODE_10000baseCR_Full_BIT, + ETHTOOL_LINK_MODE_10000baseSR_Full_BIT, + ETHTOOL_LINK_MODE_10000baseLR_Full_BIT, + ETHTOOL_LINK_MODE_10000baseER_Full_BIT, +}; + +#define MLXSW_SP2_MASK_ETHTOOL_XFI_XAUI_1_10G_LEN \ + ARRAY_SIZE(mlxsw_sp2_mask_ethtool_xfi_xaui_1_10g) + +static const enum ethtool_link_mode_bit_indices +mlxsw_sp2_mask_ethtool_xlaui_4_xlppi_4_40g[] = { + ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT, + ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT, + ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT, + ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT, +}; + +#define MLXSW_SP2_MASK_ETHTOOL_XLAUI_4_XLPPI_4_40G_LEN \ + ARRAY_SIZE(mlxsw_sp2_mask_ethtool_xlaui_4_xlppi_4_40g) + +static const enum ethtool_link_mode_bit_indices +mlxsw_sp2_mask_ethtool_25gaui_1_25gbase_cr_kr[] = { + ETHTOOL_LINK_MODE_25000baseCR_Full_BIT, + ETHTOOL_LINK_MODE_25000baseKR_Full_BIT, + ETHTOOL_LINK_MODE_25000baseSR_Full_BIT, +}; + +#define MLXSW_SP2_MASK_ETHTOOL_25GAUI_1_25GBASE_CR_KR_LEN \ + ARRAY_SIZE(mlxsw_sp2_mask_ethtool_25gaui_1_25gbase_cr_kr) + +static const enum ethtool_link_mode_bit_indices +mlxsw_sp2_mask_ethtool_50gaui_2_laui_2_50gbase_cr2_kr2[] = { + ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT, + ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT, + ETHTOOL_LINK_MODE_50000baseSR2_Full_BIT, +}; + +#define MLXSW_SP2_MASK_ETHTOOL_50GAUI_2_LAUI_2_50GBASE_CR2_KR2_LEN \ + ARRAY_SIZE(mlxsw_sp2_mask_ethtool_50gaui_2_laui_2_50gbase_cr2_kr2) + +static const enum ethtool_link_mode_bit_indices +mlxsw_sp2_mask_ethtool_50gaui_1_laui_1_50gbase_cr_kr[] = { + ETHTOOL_LINK_MODE_50000baseKR_Full_BIT, + ETHTOOL_LINK_MODE_50000baseSR_Full_BIT, + ETHTOOL_LINK_MODE_50000baseCR_Full_BIT, + ETHTOOL_LINK_MODE_50000baseLR_ER_FR_Full_BIT, + ETHTOOL_LINK_MODE_50000baseDR_Full_BIT, +}; + +#define MLXSW_SP2_MASK_ETHTOOL_50GAUI_1_LAUI_1_50GBASE_CR_KR_LEN \ + ARRAY_SIZE(mlxsw_sp2_mask_ethtool_50gaui_1_laui_1_50gbase_cr_kr) + +static const enum ethtool_link_mode_bit_indices +mlxsw_sp2_mask_ethtool_caui_4_100gbase_cr4_kr4[] = { + ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT, + ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT, + ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT, + ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT, +}; + +#define MLXSW_SP2_MASK_ETHTOOL_CAUI_4_100GBASE_CR4_KR4_LEN \ + ARRAY_SIZE(mlxsw_sp2_mask_ethtool_caui_4_100gbase_cr4_kr4) + +static const enum ethtool_link_mode_bit_indices +mlxsw_sp2_mask_ethtool_100gaui_2_100gbase_cr2_kr2[] = { + ETHTOOL_LINK_MODE_100000baseKR2_Full_BIT, + ETHTOOL_LINK_MODE_100000baseSR2_Full_BIT, + ETHTOOL_LINK_MODE_100000baseCR2_Full_BIT, + ETHTOOL_LINK_MODE_100000baseLR2_ER2_FR2_Full_BIT, + ETHTOOL_LINK_MODE_100000baseDR2_Full_BIT, +}; + +#define MLXSW_SP2_MASK_ETHTOOL_100GAUI_2_100GBASE_CR2_KR2_LEN \ + ARRAY_SIZE(mlxsw_sp2_mask_ethtool_100gaui_2_100gbase_cr2_kr2) + +static const enum ethtool_link_mode_bit_indices +mlxsw_sp2_mask_ethtool_200gaui_4_200gbase_cr4_kr4[] = { + ETHTOOL_LINK_MODE_200000baseKR4_Full_BIT, + ETHTOOL_LINK_MODE_200000baseSR4_Full_BIT, + ETHTOOL_LINK_MODE_200000baseLR4_ER4_FR4_Full_BIT, + ETHTOOL_LINK_MODE_200000baseDR4_Full_BIT, + ETHTOOL_LINK_MODE_200000baseCR4_Full_BIT, +}; + +#define MLXSW_SP2_MASK_ETHTOOL_200GAUI_4_200GBASE_CR4_KR4_LEN \ + ARRAY_SIZE(mlxsw_sp2_mask_ethtool_200gaui_4_200gbase_cr4_kr4) + +static const enum ethtool_link_mode_bit_indices +mlxsw_sp2_mask_ethtool_400gaui_8[] = { + ETHTOOL_LINK_MODE_400000baseKR8_Full_BIT, + ETHTOOL_LINK_MODE_400000baseSR8_Full_BIT, + ETHTOOL_LINK_MODE_400000baseLR8_ER8_FR8_Full_BIT, + ETHTOOL_LINK_MODE_400000baseDR8_Full_BIT, + ETHTOOL_LINK_MODE_400000baseCR8_Full_BIT, +}; + +#define MLXSW_SP2_MASK_ETHTOOL_400GAUI_8_LEN \ + ARRAY_SIZE(mlxsw_sp2_mask_ethtool_400gaui_8) + +#define MLXSW_SP_PORT_MASK_WIDTH_1X BIT(0) +#define MLXSW_SP_PORT_MASK_WIDTH_2X BIT(1) +#define MLXSW_SP_PORT_MASK_WIDTH_4X BIT(2) +#define MLXSW_SP_PORT_MASK_WIDTH_8X BIT(3) + +static u8 mlxsw_sp_port_mask_width_get(u8 width) +{ + switch (width) { + case 1: + return MLXSW_SP_PORT_MASK_WIDTH_1X; + case 2: + return MLXSW_SP_PORT_MASK_WIDTH_2X; + case 4: + return MLXSW_SP_PORT_MASK_WIDTH_4X; + case 8: + return MLXSW_SP_PORT_MASK_WIDTH_8X; + default: + WARN_ON_ONCE(1); + return 0; + } +} + +struct mlxsw_sp2_port_link_mode { + const enum ethtool_link_mode_bit_indices *mask_ethtool; + int m_ethtool_len; + u32 mask; + u32 speed; + u8 mask_width; +}; + +static const struct mlxsw_sp2_port_link_mode mlxsw_sp2_port_link_mode[] = { + { + .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_SGMII_100M, + .mask_ethtool = mlxsw_sp2_mask_ethtool_sgmii_100m, + .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_SGMII_100M_LEN, + .mask_width = MLXSW_SP_PORT_MASK_WIDTH_1X | + MLXSW_SP_PORT_MASK_WIDTH_2X | + MLXSW_SP_PORT_MASK_WIDTH_4X | + MLXSW_SP_PORT_MASK_WIDTH_8X, + .speed = SPEED_100, + }, + { + .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_1000BASE_X_SGMII, + .mask_ethtool = mlxsw_sp2_mask_ethtool_1000base_x_sgmii, + .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_1000BASE_X_SGMII_LEN, + .mask_width = MLXSW_SP_PORT_MASK_WIDTH_1X | + MLXSW_SP_PORT_MASK_WIDTH_2X | + MLXSW_SP_PORT_MASK_WIDTH_4X | + MLXSW_SP_PORT_MASK_WIDTH_8X, + .speed = SPEED_1000, + }, + { + .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_2_5GBASE_X_2_5GMII, + .mask_ethtool = mlxsw_sp2_mask_ethtool_2_5gbase_x_2_5gmii, + .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_2_5GBASE_X_2_5GMII_LEN, + .mask_width = MLXSW_SP_PORT_MASK_WIDTH_1X | + MLXSW_SP_PORT_MASK_WIDTH_2X | + MLXSW_SP_PORT_MASK_WIDTH_4X | + MLXSW_SP_PORT_MASK_WIDTH_8X, + .speed = SPEED_2500, + }, + { + .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_5GBASE_R, + .mask_ethtool = mlxsw_sp2_mask_ethtool_5gbase_r, + .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_5GBASE_R_LEN, + .mask_width = MLXSW_SP_PORT_MASK_WIDTH_1X | + MLXSW_SP_PORT_MASK_WIDTH_2X | + MLXSW_SP_PORT_MASK_WIDTH_4X | + MLXSW_SP_PORT_MASK_WIDTH_8X, + .speed = SPEED_5000, + }, + { + .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_XFI_XAUI_1_10G, + .mask_ethtool = mlxsw_sp2_mask_ethtool_xfi_xaui_1_10g, + .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_XFI_XAUI_1_10G_LEN, + .mask_width = MLXSW_SP_PORT_MASK_WIDTH_1X | + MLXSW_SP_PORT_MASK_WIDTH_2X | + MLXSW_SP_PORT_MASK_WIDTH_4X | + MLXSW_SP_PORT_MASK_WIDTH_8X, + .speed = SPEED_10000, + }, + { + .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_XLAUI_4_XLPPI_4_40G, + .mask_ethtool = mlxsw_sp2_mask_ethtool_xlaui_4_xlppi_4_40g, + .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_XLAUI_4_XLPPI_4_40G_LEN, + .mask_width = MLXSW_SP_PORT_MASK_WIDTH_4X | + MLXSW_SP_PORT_MASK_WIDTH_8X, + .speed = SPEED_40000, + }, + { + .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_25GAUI_1_25GBASE_CR_KR, + .mask_ethtool = mlxsw_sp2_mask_ethtool_25gaui_1_25gbase_cr_kr, + .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_25GAUI_1_25GBASE_CR_KR_LEN, + .mask_width = MLXSW_SP_PORT_MASK_WIDTH_1X | + MLXSW_SP_PORT_MASK_WIDTH_2X | + MLXSW_SP_PORT_MASK_WIDTH_4X | + MLXSW_SP_PORT_MASK_WIDTH_8X, + .speed = SPEED_25000, + }, + { + .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_50GAUI_2_LAUI_2_50GBASE_CR2_KR2, + .mask_ethtool = mlxsw_sp2_mask_ethtool_50gaui_2_laui_2_50gbase_cr2_kr2, + .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_50GAUI_2_LAUI_2_50GBASE_CR2_KR2_LEN, + .mask_width = MLXSW_SP_PORT_MASK_WIDTH_2X | + MLXSW_SP_PORT_MASK_WIDTH_4X | + MLXSW_SP_PORT_MASK_WIDTH_8X, + .speed = SPEED_50000, + }, + { + .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_50GAUI_1_LAUI_1_50GBASE_CR_KR, + .mask_ethtool = mlxsw_sp2_mask_ethtool_50gaui_1_laui_1_50gbase_cr_kr, + .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_50GAUI_1_LAUI_1_50GBASE_CR_KR_LEN, + .mask_width = MLXSW_SP_PORT_MASK_WIDTH_1X, + .speed = SPEED_50000, + }, + { + .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_CAUI_4_100GBASE_CR4_KR4, + .mask_ethtool = mlxsw_sp2_mask_ethtool_caui_4_100gbase_cr4_kr4, + .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_CAUI_4_100GBASE_CR4_KR4_LEN, + .mask_width = MLXSW_SP_PORT_MASK_WIDTH_4X | + MLXSW_SP_PORT_MASK_WIDTH_8X, + .speed = SPEED_100000, + }, + { + .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_100GAUI_2_100GBASE_CR2_KR2, + .mask_ethtool = mlxsw_sp2_mask_ethtool_100gaui_2_100gbase_cr2_kr2, + .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_100GAUI_2_100GBASE_CR2_KR2_LEN, + .mask_width = MLXSW_SP_PORT_MASK_WIDTH_2X, + .speed = SPEED_100000, + }, + { + .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_200GAUI_4_200GBASE_CR4_KR4, + .mask_ethtool = mlxsw_sp2_mask_ethtool_200gaui_4_200gbase_cr4_kr4, + .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_200GAUI_4_200GBASE_CR4_KR4_LEN, + .mask_width = MLXSW_SP_PORT_MASK_WIDTH_4X | + MLXSW_SP_PORT_MASK_WIDTH_8X, + .speed = SPEED_200000, + }, + { + .mask = MLXSW_REG_PTYS_EXT_ETH_SPEED_400GAUI_8, + .mask_ethtool = mlxsw_sp2_mask_ethtool_400gaui_8, + .m_ethtool_len = MLXSW_SP2_MASK_ETHTOOL_400GAUI_8_LEN, + .mask_width = MLXSW_SP_PORT_MASK_WIDTH_8X, + .speed = SPEED_400000, + }, +}; + +#define MLXSW_SP2_PORT_LINK_MODE_LEN ARRAY_SIZE(mlxsw_sp2_port_link_mode) + +static void +mlxsw_sp2_from_ptys_supported_port(struct mlxsw_sp *mlxsw_sp, + u32 ptys_eth_proto, + struct ethtool_link_ksettings *cmd) +{ + ethtool_link_ksettings_add_link_mode(cmd, supported, FIBRE); + ethtool_link_ksettings_add_link_mode(cmd, supported, Backplane); +} + +static void +mlxsw_sp2_set_bit_ethtool(const struct mlxsw_sp2_port_link_mode *link_mode, + unsigned long *mode) +{ + int i; + + for (i = 0; i < link_mode->m_ethtool_len; i++) + __set_bit(link_mode->mask_ethtool[i], mode); +} + +static void +mlxsw_sp2_from_ptys_link(struct mlxsw_sp *mlxsw_sp, u32 ptys_eth_proto, + u8 width, unsigned long *mode) +{ + u8 mask_width = mlxsw_sp_port_mask_width_get(width); + int i; + + for (i = 0; i < MLXSW_SP2_PORT_LINK_MODE_LEN; i++) { + if ((ptys_eth_proto & mlxsw_sp2_port_link_mode[i].mask) && + (mask_width & mlxsw_sp2_port_link_mode[i].mask_width)) + mlxsw_sp2_set_bit_ethtool(&mlxsw_sp2_port_link_mode[i], + mode); + } +} + +static u32 +mlxsw_sp2_from_ptys_speed(struct mlxsw_sp *mlxsw_sp, u32 ptys_eth_proto) +{ + int i; + + for (i = 0; i < MLXSW_SP2_PORT_LINK_MODE_LEN; i++) { + if (ptys_eth_proto & mlxsw_sp2_port_link_mode[i].mask) + return mlxsw_sp2_port_link_mode[i].speed; + } + + return SPEED_UNKNOWN; +} + +static void +mlxsw_sp2_from_ptys_speed_duplex(struct mlxsw_sp *mlxsw_sp, bool carrier_ok, + u32 ptys_eth_proto, + struct ethtool_link_ksettings *cmd) +{ + cmd->base.speed = SPEED_UNKNOWN; + cmd->base.duplex = DUPLEX_UNKNOWN; + + if (!carrier_ok) + return; + + cmd->base.speed = mlxsw_sp2_from_ptys_speed(mlxsw_sp, ptys_eth_proto); + if (cmd->base.speed != SPEED_UNKNOWN) + cmd->base.duplex = DUPLEX_FULL; +} + +static bool +mlxsw_sp2_test_bit_ethtool(const struct mlxsw_sp2_port_link_mode *link_mode, + const unsigned long *mode) +{ + int cnt = 0; + int i; + + for (i = 0; i < link_mode->m_ethtool_len; i++) { + if (test_bit(link_mode->mask_ethtool[i], mode)) + cnt++; + } + + return cnt == link_mode->m_ethtool_len; +} + +static u32 +mlxsw_sp2_to_ptys_advert_link(struct mlxsw_sp *mlxsw_sp, u8 width, + const struct ethtool_link_ksettings *cmd) +{ + u8 mask_width = mlxsw_sp_port_mask_width_get(width); + u32 ptys_proto = 0; + int i; + + for (i = 0; i < MLXSW_SP2_PORT_LINK_MODE_LEN; i++) { + if ((mask_width & mlxsw_sp2_port_link_mode[i].mask_width) && + mlxsw_sp2_test_bit_ethtool(&mlxsw_sp2_port_link_mode[i], + cmd->link_modes.advertising)) + ptys_proto |= mlxsw_sp2_port_link_mode[i].mask; + } + return ptys_proto; +} + +static u32 mlxsw_sp2_to_ptys_speed(struct mlxsw_sp *mlxsw_sp, + u8 width, u32 speed) +{ + u8 mask_width = mlxsw_sp_port_mask_width_get(width); + u32 ptys_proto = 0; + int i; + + for (i = 0; i < MLXSW_SP2_PORT_LINK_MODE_LEN; i++) { + if ((speed == mlxsw_sp2_port_link_mode[i].speed) && + (mask_width & mlxsw_sp2_port_link_mode[i].mask_width)) + ptys_proto |= mlxsw_sp2_port_link_mode[i].mask; + } + return ptys_proto; +} + +static void +mlxsw_sp2_reg_ptys_eth_pack(struct mlxsw_sp *mlxsw_sp, char *payload, + u8 local_port, u32 proto_admin, + bool autoneg) +{ + mlxsw_reg_ptys_ext_eth_pack(payload, local_port, proto_admin, autoneg); +} + +static void +mlxsw_sp2_reg_ptys_eth_unpack(struct mlxsw_sp *mlxsw_sp, char *payload, + u32 *p_eth_proto_cap, u32 *p_eth_proto_admin, + u32 *p_eth_proto_oper) +{ + mlxsw_reg_ptys_ext_eth_unpack(payload, p_eth_proto_cap, + p_eth_proto_admin, p_eth_proto_oper); +} + +const struct mlxsw_sp_port_type_speed_ops mlxsw_sp2_port_type_speed_ops = { + .from_ptys_supported_port = mlxsw_sp2_from_ptys_supported_port, + .from_ptys_link = mlxsw_sp2_from_ptys_link, + .from_ptys_speed = mlxsw_sp2_from_ptys_speed, + .from_ptys_speed_duplex = mlxsw_sp2_from_ptys_speed_duplex, + .to_ptys_advert_link = mlxsw_sp2_to_ptys_advert_link, + .to_ptys_speed = mlxsw_sp2_to_ptys_speed, + .reg_ptys_eth_pack = mlxsw_sp2_reg_ptys_eth_pack, + .reg_ptys_eth_unpack = mlxsw_sp2_reg_ptys_eth_unpack, +}; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_flow.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_flow.c index 47b66f347ff1..0456cda33808 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_flow.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_flow.c @@ -219,8 +219,7 @@ static int mlxsw_sp_setup_tc_block_bind(struct mlxsw_sp_port *mlxsw_sp_port, mlxsw_sp_tc_block_release); if (IS_ERR(block_cb)) { mlxsw_sp_flow_block_destroy(flow_block); - err = PTR_ERR(block_cb); - goto err_cb_register; + return PTR_ERR(block_cb); } register_block = true; } else { @@ -247,7 +246,6 @@ static int mlxsw_sp_setup_tc_block_bind(struct mlxsw_sp_port *mlxsw_sp_port, err_block_bind: if (!flow_block_cb_decref(block_cb)) flow_block_cb_free(block_cb); -err_cb_register: return err; } @@ -279,18 +277,10 @@ static void mlxsw_sp_setup_tc_block_unbind(struct mlxsw_sp_port *mlxsw_sp_port, } } -int mlxsw_sp_setup_tc_block(struct mlxsw_sp_port *mlxsw_sp_port, - struct flow_block_offload *f) +int mlxsw_sp_setup_tc_block_clsact(struct mlxsw_sp_port *mlxsw_sp_port, + struct flow_block_offload *f, + bool ingress) { - bool ingress; - - if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) - ingress = true; - else if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS) - ingress = false; - else - return -EOPNOTSUPP; - f->driver_block_list = &mlxsw_sp_block_cb_list; switch (f->command) { diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c index 51e1b3930c56..41855e58564b 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c @@ -4,6 +4,7 @@ #include <linux/kernel.h> #include <linux/errno.h> #include <linux/netdevice.h> +#include <linux/log2.h> #include <net/net_namespace.h> #include <net/flow_dissector.h> #include <net/pkt_cls.h> @@ -22,6 +23,7 @@ static int mlxsw_sp_flower_parse_actions(struct mlxsw_sp *mlxsw_sp, { const struct flow_action_entry *act; int mirror_act_count = 0; + int police_act_count = 0; int err, i; if (!flow_action_has_entries(flow_action)) @@ -180,6 +182,28 @@ static int mlxsw_sp_flower_parse_actions(struct mlxsw_sp *mlxsw_sp, return err; break; } + case FLOW_ACTION_POLICE: { + u32 burst; + + if (police_act_count++) { + NL_SET_ERR_MSG_MOD(extack, "Multiple police actions per rule are not supported"); + return -EOPNOTSUPP; + } + + /* The kernel might adjust the requested burst size so + * that it is not exactly a power of two. Re-adjust it + * here since the hardware only supports burst sizes + * that are a power of two. + */ + burst = roundup_pow_of_two(act->police.burst); + err = mlxsw_sp_acl_rulei_act_police(mlxsw_sp, rulei, + act->police.index, + act->police.rate_bytes_ps, + burst, extack); + if (err) + return err; + break; + } default: NL_SET_ERR_MSG_MOD(extack, "Unsupported action"); dev_err(mlxsw_sp->bus_info->dev, "Unsupported action\n"); @@ -616,6 +640,7 @@ int mlxsw_sp_flower_stats(struct mlxsw_sp *mlxsw_sp, u64 packets; u64 lastuse; u64 bytes; + u64 drops; int err; ruleset = mlxsw_sp_acl_ruleset_get(mlxsw_sp, block, @@ -629,11 +654,12 @@ int mlxsw_sp_flower_stats(struct mlxsw_sp *mlxsw_sp, return -EINVAL; err = mlxsw_sp_acl_rule_get_stats(mlxsw_sp, rule, &packets, &bytes, - &lastuse, &used_hw_stats); + &drops, &lastuse, &used_hw_stats); if (err) goto err_rule_get_stats; - flow_stats_update(&f->stats, bytes, packets, lastuse, used_hw_stats); + flow_stats_update(&f->stats, bytes, packets, drops, lastuse, + used_hw_stats); mlxsw_sp_acl_ruleset_put(mlxsw_sp, ruleset); return 0; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_matchall.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_matchall.c index f1a44a8eda55..f30599ad6019 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_matchall.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_matchall.c @@ -10,29 +10,6 @@ #include "spectrum_span.h" #include "reg.h" -enum mlxsw_sp_mall_action_type { - MLXSW_SP_MALL_ACTION_TYPE_MIRROR, - MLXSW_SP_MALL_ACTION_TYPE_SAMPLE, -}; - -struct mlxsw_sp_mall_mirror_entry { - const struct net_device *to_dev; - int span_id; -}; - -struct mlxsw_sp_mall_entry { - struct list_head list; - unsigned long cookie; - unsigned int priority; - enum mlxsw_sp_mall_action_type type; - bool ingress; - union { - struct mlxsw_sp_mall_mirror_entry mirror; - struct mlxsw_sp_port_sample sample; - }; - struct rcu_head rcu; -}; - static struct mlxsw_sp_mall_entry * mlxsw_sp_mall_entry_find(struct mlxsw_sp_flow_block *block, unsigned long cookie) { @@ -50,6 +27,7 @@ mlxsw_sp_mall_port_mirror_add(struct mlxsw_sp_port *mlxsw_sp_port, struct mlxsw_sp_mall_entry *mall_entry) { struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; + struct mlxsw_sp_span_agent_parms agent_parms = {}; struct mlxsw_sp_span_trigger_parms parms; enum mlxsw_sp_span_trigger trigger; int err; @@ -59,8 +37,9 @@ mlxsw_sp_mall_port_mirror_add(struct mlxsw_sp_port *mlxsw_sp_port, return -EINVAL; } - err = mlxsw_sp_span_agent_get(mlxsw_sp, mall_entry->mirror.to_dev, - &mall_entry->mirror.span_id); + agent_parms.to_dev = mall_entry->mirror.to_dev; + err = mlxsw_sp_span_agent_get(mlxsw_sp, &mall_entry->mirror.span_id, + &agent_parms); if (err) return err; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_policer.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_policer.c new file mode 100644 index 000000000000..39052e5c12fd --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_policer.c @@ -0,0 +1,468 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 +/* Copyright (c) 2020 Mellanox Technologies. All rights reserved */ + +#include <linux/idr.h> +#include <linux/log2.h> +#include <linux/mutex.h> +#include <linux/netlink.h> +#include <net/devlink.h> + +#include "spectrum.h" + +struct mlxsw_sp_policer_family { + enum mlxsw_sp_policer_type type; + enum mlxsw_reg_qpcr_g qpcr_type; + struct mlxsw_sp *mlxsw_sp; + u16 start_index; /* Inclusive */ + u16 end_index; /* Exclusive */ + struct idr policer_idr; + struct mutex lock; /* Protects policer_idr */ + atomic_t policers_count; + const struct mlxsw_sp_policer_family_ops *ops; +}; + +struct mlxsw_sp_policer { + struct mlxsw_sp_policer_params params; + u16 index; +}; + +struct mlxsw_sp_policer_family_ops { + int (*init)(struct mlxsw_sp_policer_family *family); + void (*fini)(struct mlxsw_sp_policer_family *family); + int (*policer_index_alloc)(struct mlxsw_sp_policer_family *family, + struct mlxsw_sp_policer *policer); + struct mlxsw_sp_policer * (*policer_index_free)(struct mlxsw_sp_policer_family *family, + u16 policer_index); + int (*policer_init)(struct mlxsw_sp_policer_family *family, + const struct mlxsw_sp_policer *policer); + int (*policer_params_check)(const struct mlxsw_sp_policer_family *family, + const struct mlxsw_sp_policer_params *params, + struct netlink_ext_ack *extack); +}; + +struct mlxsw_sp_policer_core { + struct mlxsw_sp_policer_family *family_arr[MLXSW_SP_POLICER_TYPE_MAX + 1]; + const struct mlxsw_sp_policer_core_ops *ops; + u8 lowest_bs_bits; + u8 highest_bs_bits; +}; + +struct mlxsw_sp_policer_core_ops { + int (*init)(struct mlxsw_sp_policer_core *policer_core); +}; + +static u64 mlxsw_sp_policer_rate_bytes_ps_kbps(u64 rate_bytes_ps) +{ + return div_u64(rate_bytes_ps, 1000) * BITS_PER_BYTE; +} + +static u8 mlxsw_sp_policer_burst_bytes_hw_units(u64 burst_bytes) +{ + /* Provided burst size is in bytes. The ASIC burst size value is + * (2 ^ bs) * 512 bits. Convert the provided size to 512-bit units. + */ + u64 bs512 = div_u64(burst_bytes, 64); + + if (!bs512) + return 0; + + return fls64(bs512) - 1; +} + +static u64 mlxsw_sp_policer_single_rate_occ_get(void *priv) +{ + struct mlxsw_sp_policer_family *family = priv; + + return atomic_read(&family->policers_count); +} + +static int +mlxsw_sp_policer_single_rate_family_init(struct mlxsw_sp_policer_family *family) +{ + struct mlxsw_core *core = family->mlxsw_sp->core; + struct devlink *devlink; + + /* CPU policers are allocated from the first N policers in the global + * range, so skip them. + */ + if (!MLXSW_CORE_RES_VALID(core, MAX_GLOBAL_POLICERS) || + !MLXSW_CORE_RES_VALID(core, MAX_CPU_POLICERS)) + return -EIO; + + family->start_index = MLXSW_CORE_RES_GET(core, MAX_CPU_POLICERS); + family->end_index = MLXSW_CORE_RES_GET(core, MAX_GLOBAL_POLICERS); + + atomic_set(&family->policers_count, 0); + devlink = priv_to_devlink(core); + devlink_resource_occ_get_register(devlink, + MLXSW_SP_RESOURCE_SINGLE_RATE_POLICERS, + mlxsw_sp_policer_single_rate_occ_get, + family); + + return 0; +} + +static void +mlxsw_sp_policer_single_rate_family_fini(struct mlxsw_sp_policer_family *family) +{ + struct devlink *devlink = priv_to_devlink(family->mlxsw_sp->core); + + devlink_resource_occ_get_unregister(devlink, + MLXSW_SP_RESOURCE_SINGLE_RATE_POLICERS); + WARN_ON(atomic_read(&family->policers_count) != 0); +} + +static int +mlxsw_sp_policer_single_rate_index_alloc(struct mlxsw_sp_policer_family *family, + struct mlxsw_sp_policer *policer) +{ + int id; + + mutex_lock(&family->lock); + id = idr_alloc(&family->policer_idr, policer, family->start_index, + family->end_index, GFP_KERNEL); + mutex_unlock(&family->lock); + + if (id < 0) + return id; + + atomic_inc(&family->policers_count); + policer->index = id; + + return 0; +} + +static struct mlxsw_sp_policer * +mlxsw_sp_policer_single_rate_index_free(struct mlxsw_sp_policer_family *family, + u16 policer_index) +{ + struct mlxsw_sp_policer *policer; + + atomic_dec(&family->policers_count); + + mutex_lock(&family->lock); + policer = idr_remove(&family->policer_idr, policer_index); + mutex_unlock(&family->lock); + + WARN_ON(!policer); + + return policer; +} + +static int +mlxsw_sp_policer_single_rate_init(struct mlxsw_sp_policer_family *family, + const struct mlxsw_sp_policer *policer) +{ + u64 rate_kbps = mlxsw_sp_policer_rate_bytes_ps_kbps(policer->params.rate); + u8 bs = mlxsw_sp_policer_burst_bytes_hw_units(policer->params.burst); + struct mlxsw_sp *mlxsw_sp = family->mlxsw_sp; + char qpcr_pl[MLXSW_REG_QPCR_LEN]; + + mlxsw_reg_qpcr_pack(qpcr_pl, policer->index, MLXSW_REG_QPCR_IR_UNITS_K, + true, rate_kbps, bs); + mlxsw_reg_qpcr_clear_counter_set(qpcr_pl, true); + + return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(qpcr), qpcr_pl); +} + +static int +mlxsw_sp_policer_single_rate_params_check(const struct mlxsw_sp_policer_family *family, + const struct mlxsw_sp_policer_params *params, + struct netlink_ext_ack *extack) +{ + struct mlxsw_sp_policer_core *policer_core = family->mlxsw_sp->policer_core; + u64 rate_bps = params->rate * BITS_PER_BYTE; + u8 bs; + + if (!params->bytes) { + NL_SET_ERR_MSG_MOD(extack, "Only bandwidth policing is currently supported by single rate policers"); + return -EINVAL; + } + + if (!is_power_of_2(params->burst)) { + NL_SET_ERR_MSG_MOD(extack, "Policer burst size is not power of two"); + return -EINVAL; + } + + bs = mlxsw_sp_policer_burst_bytes_hw_units(params->burst); + + if (bs < policer_core->lowest_bs_bits) { + NL_SET_ERR_MSG_MOD(extack, "Policer burst size lower than limit"); + return -EINVAL; + } + + if (bs > policer_core->highest_bs_bits) { + NL_SET_ERR_MSG_MOD(extack, "Policer burst size higher than limit"); + return -EINVAL; + } + + if (rate_bps < MLXSW_REG_QPCR_LOWEST_CIR_BITS) { + NL_SET_ERR_MSG_MOD(extack, "Policer rate lower than limit"); + return -EINVAL; + } + + if (rate_bps > MLXSW_REG_QPCR_HIGHEST_CIR_BITS) { + NL_SET_ERR_MSG_MOD(extack, "Policer rate higher than limit"); + return -EINVAL; + } + + return 0; +} + +static const struct mlxsw_sp_policer_family_ops mlxsw_sp_policer_single_rate_ops = { + .init = mlxsw_sp_policer_single_rate_family_init, + .fini = mlxsw_sp_policer_single_rate_family_fini, + .policer_index_alloc = mlxsw_sp_policer_single_rate_index_alloc, + .policer_index_free = mlxsw_sp_policer_single_rate_index_free, + .policer_init = mlxsw_sp_policer_single_rate_init, + .policer_params_check = mlxsw_sp_policer_single_rate_params_check, +}; + +static const struct mlxsw_sp_policer_family mlxsw_sp_policer_single_rate_family = { + .type = MLXSW_SP_POLICER_TYPE_SINGLE_RATE, + .qpcr_type = MLXSW_REG_QPCR_G_GLOBAL, + .ops = &mlxsw_sp_policer_single_rate_ops, +}; + +static const struct mlxsw_sp_policer_family *mlxsw_sp_policer_family_arr[] = { + [MLXSW_SP_POLICER_TYPE_SINGLE_RATE] = &mlxsw_sp_policer_single_rate_family, +}; + +int mlxsw_sp_policer_add(struct mlxsw_sp *mlxsw_sp, + enum mlxsw_sp_policer_type type, + const struct mlxsw_sp_policer_params *params, + struct netlink_ext_ack *extack, u16 *p_policer_index) +{ + struct mlxsw_sp_policer_family *family; + struct mlxsw_sp_policer *policer; + int err; + + family = mlxsw_sp->policer_core->family_arr[type]; + + err = family->ops->policer_params_check(family, params, extack); + if (err) + return err; + + policer = kmalloc(sizeof(*policer), GFP_KERNEL); + if (!policer) + return -ENOMEM; + policer->params = *params; + + err = family->ops->policer_index_alloc(family, policer); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Failed to allocate policer index"); + goto err_policer_index_alloc; + } + + err = family->ops->policer_init(family, policer); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Failed to initialize policer"); + goto err_policer_init; + } + + *p_policer_index = policer->index; + + return 0; + +err_policer_init: + family->ops->policer_index_free(family, policer->index); +err_policer_index_alloc: + kfree(policer); + return err; +} + +void mlxsw_sp_policer_del(struct mlxsw_sp *mlxsw_sp, + enum mlxsw_sp_policer_type type, u16 policer_index) +{ + struct mlxsw_sp_policer_family *family; + struct mlxsw_sp_policer *policer; + + family = mlxsw_sp->policer_core->family_arr[type]; + policer = family->ops->policer_index_free(family, policer_index); + kfree(policer); +} + +int mlxsw_sp_policer_drops_counter_get(struct mlxsw_sp *mlxsw_sp, + enum mlxsw_sp_policer_type type, + u16 policer_index, u64 *p_drops) +{ + struct mlxsw_sp_policer_family *family; + char qpcr_pl[MLXSW_REG_QPCR_LEN]; + int err; + + family = mlxsw_sp->policer_core->family_arr[type]; + + MLXSW_REG_ZERO(qpcr, qpcr_pl); + mlxsw_reg_qpcr_pid_set(qpcr_pl, policer_index); + mlxsw_reg_qpcr_g_set(qpcr_pl, family->qpcr_type); + err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(qpcr), qpcr_pl); + if (err) + return err; + + *p_drops = mlxsw_reg_qpcr_violate_count_get(qpcr_pl); + + return 0; +} + +static int +mlxsw_sp_policer_family_register(struct mlxsw_sp *mlxsw_sp, + const struct mlxsw_sp_policer_family *tmpl) +{ + struct mlxsw_sp_policer_family *family; + int err; + + family = kmemdup(tmpl, sizeof(*family), GFP_KERNEL); + if (!family) + return -ENOMEM; + + family->mlxsw_sp = mlxsw_sp; + idr_init(&family->policer_idr); + mutex_init(&family->lock); + + err = family->ops->init(family); + if (err) + goto err_family_init; + + if (WARN_ON(family->start_index >= family->end_index)) { + err = -EINVAL; + goto err_index_check; + } + + mlxsw_sp->policer_core->family_arr[tmpl->type] = family; + + return 0; + +err_index_check: + family->ops->fini(family); +err_family_init: + mutex_destroy(&family->lock); + idr_destroy(&family->policer_idr); + kfree(family); + return err; +} + +static void +mlxsw_sp_policer_family_unregister(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_policer_family *family) +{ + family->ops->fini(family); + mutex_destroy(&family->lock); + WARN_ON(!idr_is_empty(&family->policer_idr)); + idr_destroy(&family->policer_idr); + kfree(family); +} + +int mlxsw_sp_policers_init(struct mlxsw_sp *mlxsw_sp) +{ + struct mlxsw_sp_policer_core *policer_core; + int i, err; + + policer_core = kzalloc(sizeof(*policer_core), GFP_KERNEL); + if (!policer_core) + return -ENOMEM; + mlxsw_sp->policer_core = policer_core; + policer_core->ops = mlxsw_sp->policer_core_ops; + + err = policer_core->ops->init(policer_core); + if (err) + goto err_init; + + for (i = 0; i < MLXSW_SP_POLICER_TYPE_MAX + 1; i++) { + err = mlxsw_sp_policer_family_register(mlxsw_sp, mlxsw_sp_policer_family_arr[i]); + if (err) + goto err_family_register; + } + + return 0; + +err_family_register: + for (i--; i >= 0; i--) { + struct mlxsw_sp_policer_family *family; + + family = mlxsw_sp->policer_core->family_arr[i]; + mlxsw_sp_policer_family_unregister(mlxsw_sp, family); + } +err_init: + kfree(mlxsw_sp->policer_core); + return err; +} + +void mlxsw_sp_policers_fini(struct mlxsw_sp *mlxsw_sp) +{ + int i; + + for (i = MLXSW_SP_POLICER_TYPE_MAX; i >= 0; i--) { + struct mlxsw_sp_policer_family *family; + + family = mlxsw_sp->policer_core->family_arr[i]; + mlxsw_sp_policer_family_unregister(mlxsw_sp, family); + } + + kfree(mlxsw_sp->policer_core); +} + +int mlxsw_sp_policer_resources_register(struct mlxsw_core *mlxsw_core) +{ + u64 global_policers, cpu_policers, single_rate_policers; + struct devlink *devlink = priv_to_devlink(mlxsw_core); + struct devlink_resource_size_params size_params; + int err; + + if (!MLXSW_CORE_RES_VALID(mlxsw_core, MAX_GLOBAL_POLICERS) || + !MLXSW_CORE_RES_VALID(mlxsw_core, MAX_CPU_POLICERS)) + return -EIO; + + global_policers = MLXSW_CORE_RES_GET(mlxsw_core, MAX_GLOBAL_POLICERS); + cpu_policers = MLXSW_CORE_RES_GET(mlxsw_core, MAX_CPU_POLICERS); + single_rate_policers = global_policers - cpu_policers; + + devlink_resource_size_params_init(&size_params, global_policers, + global_policers, 1, + DEVLINK_RESOURCE_UNIT_ENTRY); + err = devlink_resource_register(devlink, "global_policers", + global_policers, + MLXSW_SP_RESOURCE_GLOBAL_POLICERS, + DEVLINK_RESOURCE_ID_PARENT_TOP, + &size_params); + if (err) + return err; + + devlink_resource_size_params_init(&size_params, single_rate_policers, + single_rate_policers, 1, + DEVLINK_RESOURCE_UNIT_ENTRY); + err = devlink_resource_register(devlink, "single_rate_policers", + single_rate_policers, + MLXSW_SP_RESOURCE_SINGLE_RATE_POLICERS, + MLXSW_SP_RESOURCE_GLOBAL_POLICERS, + &size_params); + if (err) + return err; + + return 0; +} + +static int +mlxsw_sp1_policer_core_init(struct mlxsw_sp_policer_core *policer_core) +{ + policer_core->lowest_bs_bits = MLXSW_REG_QPCR_LOWEST_CBS_BITS_SP1; + policer_core->highest_bs_bits = MLXSW_REG_QPCR_HIGHEST_CBS_BITS_SP1; + + return 0; +} + +const struct mlxsw_sp_policer_core_ops mlxsw_sp1_policer_core_ops = { + .init = mlxsw_sp1_policer_core_init, +}; + +static int +mlxsw_sp2_policer_core_init(struct mlxsw_sp_policer_core *policer_core) +{ + policer_core->lowest_bs_bits = MLXSW_REG_QPCR_LOWEST_CBS_BITS_SP2; + policer_core->highest_bs_bits = MLXSW_REG_QPCR_HIGHEST_CBS_BITS_SP2; + + return 0; +} + +const struct mlxsw_sp_policer_core_ops mlxsw_sp2_policer_core_ops = { + .init = mlxsw_sp2_policer_core_init, +}; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_qdisc.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_qdisc.c index 670a43fe2a00..964fd444bb10 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_qdisc.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_qdisc.c @@ -8,6 +8,7 @@ #include <net/red.h> #include "spectrum.h" +#include "spectrum_span.h" #include "reg.h" #define MLXSW_SP_PRIO_BAND_TO_TCLASS(band) (IEEE_8021QAZ_MAX_TCS - band - 1) @@ -1272,6 +1273,529 @@ int mlxsw_sp_setup_tc_ets(struct mlxsw_sp_port *mlxsw_sp_port, } } +struct mlxsw_sp_qevent_block { + struct list_head binding_list; + struct list_head mall_entry_list; + struct mlxsw_sp *mlxsw_sp; +}; + +struct mlxsw_sp_qevent_binding { + struct list_head list; + struct mlxsw_sp_port *mlxsw_sp_port; + u32 handle; + int tclass_num; + enum mlxsw_sp_span_trigger span_trigger; +}; + +static LIST_HEAD(mlxsw_sp_qevent_block_cb_list); + +static int mlxsw_sp_qevent_span_configure(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_mall_entry *mall_entry, + struct mlxsw_sp_qevent_binding *qevent_binding, + const struct mlxsw_sp_span_agent_parms *agent_parms, + int *p_span_id) +{ + struct mlxsw_sp_port *mlxsw_sp_port = qevent_binding->mlxsw_sp_port; + struct mlxsw_sp_span_trigger_parms trigger_parms = {}; + int span_id; + int err; + + err = mlxsw_sp_span_agent_get(mlxsw_sp, &span_id, agent_parms); + if (err) + return err; + + err = mlxsw_sp_span_analyzed_port_get(mlxsw_sp_port, true); + if (err) + goto err_analyzed_port_get; + + trigger_parms.span_id = span_id; + err = mlxsw_sp_span_agent_bind(mlxsw_sp, qevent_binding->span_trigger, mlxsw_sp_port, + &trigger_parms); + if (err) + goto err_agent_bind; + + err = mlxsw_sp_span_trigger_enable(mlxsw_sp_port, qevent_binding->span_trigger, + qevent_binding->tclass_num); + if (err) + goto err_trigger_enable; + + *p_span_id = span_id; + return 0; + +err_trigger_enable: + mlxsw_sp_span_agent_unbind(mlxsw_sp, qevent_binding->span_trigger, mlxsw_sp_port, + &trigger_parms); +err_agent_bind: + mlxsw_sp_span_analyzed_port_put(mlxsw_sp_port, true); +err_analyzed_port_get: + mlxsw_sp_span_agent_put(mlxsw_sp, span_id); + return err; +} + +static void mlxsw_sp_qevent_span_deconfigure(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_qevent_binding *qevent_binding, + int span_id) +{ + struct mlxsw_sp_port *mlxsw_sp_port = qevent_binding->mlxsw_sp_port; + struct mlxsw_sp_span_trigger_parms trigger_parms = { + .span_id = span_id, + }; + + mlxsw_sp_span_trigger_disable(mlxsw_sp_port, qevent_binding->span_trigger, + qevent_binding->tclass_num); + mlxsw_sp_span_agent_unbind(mlxsw_sp, qevent_binding->span_trigger, mlxsw_sp_port, + &trigger_parms); + mlxsw_sp_span_analyzed_port_put(mlxsw_sp_port, true); + mlxsw_sp_span_agent_put(mlxsw_sp, span_id); +} + +static int mlxsw_sp_qevent_mirror_configure(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_mall_entry *mall_entry, + struct mlxsw_sp_qevent_binding *qevent_binding) +{ + struct mlxsw_sp_span_agent_parms agent_parms = { + .to_dev = mall_entry->mirror.to_dev, + }; + + return mlxsw_sp_qevent_span_configure(mlxsw_sp, mall_entry, qevent_binding, + &agent_parms, &mall_entry->mirror.span_id); +} + +static void mlxsw_sp_qevent_mirror_deconfigure(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_mall_entry *mall_entry, + struct mlxsw_sp_qevent_binding *qevent_binding) +{ + mlxsw_sp_qevent_span_deconfigure(mlxsw_sp, qevent_binding, mall_entry->mirror.span_id); +} + +static int mlxsw_sp_qevent_trap_configure(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_mall_entry *mall_entry, + struct mlxsw_sp_qevent_binding *qevent_binding) +{ + struct mlxsw_sp_span_agent_parms agent_parms = {}; + int err; + + err = mlxsw_sp_trap_group_policer_hw_id_get(mlxsw_sp, + DEVLINK_TRAP_GROUP_GENERIC_ID_BUFFER_DROPS, + &agent_parms.policer_enable, + &agent_parms.policer_id); + if (err) + return err; + + return mlxsw_sp_qevent_span_configure(mlxsw_sp, mall_entry, qevent_binding, + &agent_parms, &mall_entry->trap.span_id); +} + +static void mlxsw_sp_qevent_trap_deconfigure(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_mall_entry *mall_entry, + struct mlxsw_sp_qevent_binding *qevent_binding) +{ + mlxsw_sp_qevent_span_deconfigure(mlxsw_sp, qevent_binding, mall_entry->trap.span_id); +} + +static int mlxsw_sp_qevent_entry_configure(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_mall_entry *mall_entry, + struct mlxsw_sp_qevent_binding *qevent_binding) +{ + switch (mall_entry->type) { + case MLXSW_SP_MALL_ACTION_TYPE_MIRROR: + return mlxsw_sp_qevent_mirror_configure(mlxsw_sp, mall_entry, qevent_binding); + case MLXSW_SP_MALL_ACTION_TYPE_TRAP: + return mlxsw_sp_qevent_trap_configure(mlxsw_sp, mall_entry, qevent_binding); + default: + /* This should have been validated away. */ + WARN_ON(1); + return -EOPNOTSUPP; + } +} + +static void mlxsw_sp_qevent_entry_deconfigure(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_mall_entry *mall_entry, + struct mlxsw_sp_qevent_binding *qevent_binding) +{ + switch (mall_entry->type) { + case MLXSW_SP_MALL_ACTION_TYPE_MIRROR: + return mlxsw_sp_qevent_mirror_deconfigure(mlxsw_sp, mall_entry, qevent_binding); + case MLXSW_SP_MALL_ACTION_TYPE_TRAP: + return mlxsw_sp_qevent_trap_deconfigure(mlxsw_sp, mall_entry, qevent_binding); + default: + WARN_ON(1); + return; + } +} + +static int mlxsw_sp_qevent_binding_configure(struct mlxsw_sp_qevent_block *qevent_block, + struct mlxsw_sp_qevent_binding *qevent_binding) +{ + struct mlxsw_sp_mall_entry *mall_entry; + int err; + + list_for_each_entry(mall_entry, &qevent_block->mall_entry_list, list) { + err = mlxsw_sp_qevent_entry_configure(qevent_block->mlxsw_sp, mall_entry, + qevent_binding); + if (err) + goto err_entry_configure; + } + + return 0; + +err_entry_configure: + list_for_each_entry_continue_reverse(mall_entry, &qevent_block->mall_entry_list, list) + mlxsw_sp_qevent_entry_deconfigure(qevent_block->mlxsw_sp, mall_entry, + qevent_binding); + return err; +} + +static void mlxsw_sp_qevent_binding_deconfigure(struct mlxsw_sp_qevent_block *qevent_block, + struct mlxsw_sp_qevent_binding *qevent_binding) +{ + struct mlxsw_sp_mall_entry *mall_entry; + + list_for_each_entry(mall_entry, &qevent_block->mall_entry_list, list) + mlxsw_sp_qevent_entry_deconfigure(qevent_block->mlxsw_sp, mall_entry, + qevent_binding); +} + +static int mlxsw_sp_qevent_block_configure(struct mlxsw_sp_qevent_block *qevent_block) +{ + struct mlxsw_sp_qevent_binding *qevent_binding; + int err; + + list_for_each_entry(qevent_binding, &qevent_block->binding_list, list) { + err = mlxsw_sp_qevent_binding_configure(qevent_block, qevent_binding); + if (err) + goto err_binding_configure; + } + + return 0; + +err_binding_configure: + list_for_each_entry_continue_reverse(qevent_binding, &qevent_block->binding_list, list) + mlxsw_sp_qevent_binding_deconfigure(qevent_block, qevent_binding); + return err; +} + +static void mlxsw_sp_qevent_block_deconfigure(struct mlxsw_sp_qevent_block *qevent_block) +{ + struct mlxsw_sp_qevent_binding *qevent_binding; + + list_for_each_entry(qevent_binding, &qevent_block->binding_list, list) + mlxsw_sp_qevent_binding_deconfigure(qevent_block, qevent_binding); +} + +static struct mlxsw_sp_mall_entry * +mlxsw_sp_qevent_mall_entry_find(struct mlxsw_sp_qevent_block *block, unsigned long cookie) +{ + struct mlxsw_sp_mall_entry *mall_entry; + + list_for_each_entry(mall_entry, &block->mall_entry_list, list) + if (mall_entry->cookie == cookie) + return mall_entry; + + return NULL; +} + +static int mlxsw_sp_qevent_mall_replace(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_qevent_block *qevent_block, + struct tc_cls_matchall_offload *f) +{ + struct mlxsw_sp_mall_entry *mall_entry; + struct flow_action_entry *act; + int err; + + /* It should not currently be possible to replace a matchall rule. So + * this must be a new rule. + */ + if (!list_empty(&qevent_block->mall_entry_list)) { + NL_SET_ERR_MSG(f->common.extack, "At most one filter supported"); + return -EOPNOTSUPP; + } + if (f->rule->action.num_entries != 1) { + NL_SET_ERR_MSG(f->common.extack, "Only singular actions supported"); + return -EOPNOTSUPP; + } + if (f->common.chain_index) { + NL_SET_ERR_MSG(f->common.extack, "Only chain 0 is supported"); + return -EOPNOTSUPP; + } + if (f->common.protocol != htons(ETH_P_ALL)) { + NL_SET_ERR_MSG(f->common.extack, "Protocol matching not supported"); + return -EOPNOTSUPP; + } + + act = &f->rule->action.entries[0]; + if (!(act->hw_stats & FLOW_ACTION_HW_STATS_DISABLED)) { + NL_SET_ERR_MSG(f->common.extack, "HW counters not supported on qevents"); + return -EOPNOTSUPP; + } + + mall_entry = kzalloc(sizeof(*mall_entry), GFP_KERNEL); + if (!mall_entry) + return -ENOMEM; + mall_entry->cookie = f->cookie; + + if (act->id == FLOW_ACTION_MIRRED) { + mall_entry->type = MLXSW_SP_MALL_ACTION_TYPE_MIRROR; + mall_entry->mirror.to_dev = act->dev; + } else if (act->id == FLOW_ACTION_TRAP) { + mall_entry->type = MLXSW_SP_MALL_ACTION_TYPE_TRAP; + } else { + NL_SET_ERR_MSG(f->common.extack, "Unsupported action"); + err = -EOPNOTSUPP; + goto err_unsupported_action; + } + + list_add_tail(&mall_entry->list, &qevent_block->mall_entry_list); + + err = mlxsw_sp_qevent_block_configure(qevent_block); + if (err) + goto err_block_configure; + + return 0; + +err_block_configure: + list_del(&mall_entry->list); +err_unsupported_action: + kfree(mall_entry); + return err; +} + +static void mlxsw_sp_qevent_mall_destroy(struct mlxsw_sp_qevent_block *qevent_block, + struct tc_cls_matchall_offload *f) +{ + struct mlxsw_sp_mall_entry *mall_entry; + + mall_entry = mlxsw_sp_qevent_mall_entry_find(qevent_block, f->cookie); + if (!mall_entry) + return; + + mlxsw_sp_qevent_block_deconfigure(qevent_block); + + list_del(&mall_entry->list); + kfree(mall_entry); +} + +static int mlxsw_sp_qevent_block_mall_cb(struct mlxsw_sp_qevent_block *qevent_block, + struct tc_cls_matchall_offload *f) +{ + struct mlxsw_sp *mlxsw_sp = qevent_block->mlxsw_sp; + + switch (f->command) { + case TC_CLSMATCHALL_REPLACE: + return mlxsw_sp_qevent_mall_replace(mlxsw_sp, qevent_block, f); + case TC_CLSMATCHALL_DESTROY: + mlxsw_sp_qevent_mall_destroy(qevent_block, f); + return 0; + default: + return -EOPNOTSUPP; + } +} + +static int mlxsw_sp_qevent_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv) +{ + struct mlxsw_sp_qevent_block *qevent_block = cb_priv; + + switch (type) { + case TC_SETUP_CLSMATCHALL: + return mlxsw_sp_qevent_block_mall_cb(qevent_block, type_data); + default: + return -EOPNOTSUPP; + } +} + +static struct mlxsw_sp_qevent_block *mlxsw_sp_qevent_block_create(struct mlxsw_sp *mlxsw_sp, + struct net *net) +{ + struct mlxsw_sp_qevent_block *qevent_block; + + qevent_block = kzalloc(sizeof(*qevent_block), GFP_KERNEL); + if (!qevent_block) + return NULL; + + INIT_LIST_HEAD(&qevent_block->binding_list); + INIT_LIST_HEAD(&qevent_block->mall_entry_list); + qevent_block->mlxsw_sp = mlxsw_sp; + return qevent_block; +} + +static void +mlxsw_sp_qevent_block_destroy(struct mlxsw_sp_qevent_block *qevent_block) +{ + WARN_ON(!list_empty(&qevent_block->binding_list)); + WARN_ON(!list_empty(&qevent_block->mall_entry_list)); + kfree(qevent_block); +} + +static void mlxsw_sp_qevent_block_release(void *cb_priv) +{ + struct mlxsw_sp_qevent_block *qevent_block = cb_priv; + + mlxsw_sp_qevent_block_destroy(qevent_block); +} + +static struct mlxsw_sp_qevent_binding * +mlxsw_sp_qevent_binding_create(struct mlxsw_sp_port *mlxsw_sp_port, u32 handle, int tclass_num, + enum mlxsw_sp_span_trigger span_trigger) +{ + struct mlxsw_sp_qevent_binding *binding; + + binding = kzalloc(sizeof(*binding), GFP_KERNEL); + if (!binding) + return ERR_PTR(-ENOMEM); + + binding->mlxsw_sp_port = mlxsw_sp_port; + binding->handle = handle; + binding->tclass_num = tclass_num; + binding->span_trigger = span_trigger; + return binding; +} + +static void +mlxsw_sp_qevent_binding_destroy(struct mlxsw_sp_qevent_binding *binding) +{ + kfree(binding); +} + +static struct mlxsw_sp_qevent_binding * +mlxsw_sp_qevent_binding_lookup(struct mlxsw_sp_qevent_block *block, + struct mlxsw_sp_port *mlxsw_sp_port, + u32 handle, + enum mlxsw_sp_span_trigger span_trigger) +{ + struct mlxsw_sp_qevent_binding *qevent_binding; + + list_for_each_entry(qevent_binding, &block->binding_list, list) + if (qevent_binding->mlxsw_sp_port == mlxsw_sp_port && + qevent_binding->handle == handle && + qevent_binding->span_trigger == span_trigger) + return qevent_binding; + return NULL; +} + +static int mlxsw_sp_setup_tc_block_qevent_bind(struct mlxsw_sp_port *mlxsw_sp_port, + struct flow_block_offload *f, + enum mlxsw_sp_span_trigger span_trigger) +{ + struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; + struct mlxsw_sp_qevent_binding *qevent_binding; + struct mlxsw_sp_qevent_block *qevent_block; + struct flow_block_cb *block_cb; + struct mlxsw_sp_qdisc *qdisc; + bool register_block = false; + int err; + + block_cb = flow_block_cb_lookup(f->block, mlxsw_sp_qevent_block_cb, mlxsw_sp); + if (!block_cb) { + qevent_block = mlxsw_sp_qevent_block_create(mlxsw_sp, f->net); + if (!qevent_block) + return -ENOMEM; + block_cb = flow_block_cb_alloc(mlxsw_sp_qevent_block_cb, mlxsw_sp, qevent_block, + mlxsw_sp_qevent_block_release); + if (IS_ERR(block_cb)) { + mlxsw_sp_qevent_block_destroy(qevent_block); + return PTR_ERR(block_cb); + } + register_block = true; + } else { + qevent_block = flow_block_cb_priv(block_cb); + } + flow_block_cb_incref(block_cb); + + qdisc = mlxsw_sp_qdisc_find_by_handle(mlxsw_sp_port, f->sch->handle); + if (!qdisc) { + NL_SET_ERR_MSG(f->extack, "Qdisc not offloaded"); + err = -ENOENT; + goto err_find_qdisc; + } + + if (WARN_ON(mlxsw_sp_qevent_binding_lookup(qevent_block, mlxsw_sp_port, f->sch->handle, + span_trigger))) { + err = -EEXIST; + goto err_binding_exists; + } + + qevent_binding = mlxsw_sp_qevent_binding_create(mlxsw_sp_port, f->sch->handle, + qdisc->tclass_num, span_trigger); + if (IS_ERR(qevent_binding)) { + err = PTR_ERR(qevent_binding); + goto err_binding_create; + } + + err = mlxsw_sp_qevent_binding_configure(qevent_block, qevent_binding); + if (err) + goto err_binding_configure; + + list_add(&qevent_binding->list, &qevent_block->binding_list); + + if (register_block) { + flow_block_cb_add(block_cb, f); + list_add_tail(&block_cb->driver_list, &mlxsw_sp_qevent_block_cb_list); + } + + return 0; + +err_binding_configure: + mlxsw_sp_qevent_binding_destroy(qevent_binding); +err_binding_create: +err_binding_exists: +err_find_qdisc: + if (!flow_block_cb_decref(block_cb)) + flow_block_cb_free(block_cb); + return err; +} + +static void mlxsw_sp_setup_tc_block_qevent_unbind(struct mlxsw_sp_port *mlxsw_sp_port, + struct flow_block_offload *f, + enum mlxsw_sp_span_trigger span_trigger) +{ + struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; + struct mlxsw_sp_qevent_binding *qevent_binding; + struct mlxsw_sp_qevent_block *qevent_block; + struct flow_block_cb *block_cb; + + block_cb = flow_block_cb_lookup(f->block, mlxsw_sp_qevent_block_cb, mlxsw_sp); + if (!block_cb) + return; + qevent_block = flow_block_cb_priv(block_cb); + + qevent_binding = mlxsw_sp_qevent_binding_lookup(qevent_block, mlxsw_sp_port, f->sch->handle, + span_trigger); + if (!qevent_binding) + return; + + list_del(&qevent_binding->list); + mlxsw_sp_qevent_binding_deconfigure(qevent_block, qevent_binding); + mlxsw_sp_qevent_binding_destroy(qevent_binding); + + if (!flow_block_cb_decref(block_cb)) { + flow_block_cb_remove(block_cb, f); + list_del(&block_cb->driver_list); + } +} + +static int mlxsw_sp_setup_tc_block_qevent(struct mlxsw_sp_port *mlxsw_sp_port, + struct flow_block_offload *f, + enum mlxsw_sp_span_trigger span_trigger) +{ + f->driver_block_list = &mlxsw_sp_qevent_block_cb_list; + + switch (f->command) { + case FLOW_BLOCK_BIND: + return mlxsw_sp_setup_tc_block_qevent_bind(mlxsw_sp_port, f, span_trigger); + case FLOW_BLOCK_UNBIND: + mlxsw_sp_setup_tc_block_qevent_unbind(mlxsw_sp_port, f, span_trigger); + return 0; + default: + return -EOPNOTSUPP; + } +} + +int mlxsw_sp_setup_tc_block_qevent_early_drop(struct mlxsw_sp_port *mlxsw_sp_port, + struct flow_block_offload *f) +{ + return mlxsw_sp_setup_tc_block_qevent(mlxsw_sp_port, f, MLXSW_SP_SPAN_TRIGGER_EARLY_DROP); +} + int mlxsw_sp_tc_qdisc_init(struct mlxsw_sp_port *mlxsw_sp_port) { struct mlxsw_sp_qdisc_state *qdisc_state; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c index 019ed503aadf..0521e9d48c45 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c @@ -5001,15 +5001,6 @@ static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp, static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt) { - /* Packets with link-local destination IP arriving to the router - * are trapped to the CPU, so no need to program specific routes - * for them. Only allow prefix routes (usually one fe80::/64) so - * that packets are trapped for the right reason. - */ - if ((ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_LINKLOCAL) && - (rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST))) - return true; - /* Multicast routes aren't supported, so ignore them. Neighbour * Discovery packets are specifically trapped. */ @@ -8078,16 +8069,6 @@ int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp, mlxsw_sp->router = router; router->mlxsw_sp = mlxsw_sp; - router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event; - err = register_inetaddr_notifier(&router->inetaddr_nb); - if (err) - goto err_register_inetaddr_notifier; - - router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event; - err = register_inet6addr_notifier(&router->inet6addr_nb); - if (err) - goto err_register_inet6addr_notifier; - INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list); err = __mlxsw_sp_router_init(mlxsw_sp); if (err) @@ -8128,12 +8109,6 @@ int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp, if (err) goto err_neigh_init; - mlxsw_sp->router->netevent_nb.notifier_call = - mlxsw_sp_router_netevent_event; - err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb); - if (err) - goto err_register_netevent_notifier; - err = mlxsw_sp_mp_hash_init(mlxsw_sp); if (err) goto err_mp_hash_init; @@ -8142,6 +8117,22 @@ int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp, if (err) goto err_dscp_init; + router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event; + err = register_inetaddr_notifier(&router->inetaddr_nb); + if (err) + goto err_register_inetaddr_notifier; + + router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event; + err = register_inet6addr_notifier(&router->inet6addr_nb); + if (err) + goto err_register_inet6addr_notifier; + + mlxsw_sp->router->netevent_nb.notifier_call = + mlxsw_sp_router_netevent_event; + err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb); + if (err) + goto err_register_netevent_notifier; + mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event; err = register_fib_notifier(mlxsw_sp_net(mlxsw_sp), &mlxsw_sp->router->fib_nb, @@ -8152,10 +8143,15 @@ int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp, return 0; err_register_fib_notifier: -err_dscp_init: -err_mp_hash_init: unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); err_register_netevent_notifier: + unregister_inet6addr_notifier(&router->inet6addr_nb); +err_register_inet6addr_notifier: + unregister_inetaddr_notifier(&router->inetaddr_nb); +err_register_inetaddr_notifier: + mlxsw_core_flush_owq(); +err_dscp_init: +err_mp_hash_init: mlxsw_sp_neigh_fini(mlxsw_sp); err_neigh_init: mlxsw_sp_vrs_fini(mlxsw_sp); @@ -8174,10 +8170,6 @@ err_ipips_init: err_rifs_init: __mlxsw_sp_router_fini(mlxsw_sp); err_router_init: - unregister_inet6addr_notifier(&router->inet6addr_nb); -err_register_inet6addr_notifier: - unregister_inetaddr_notifier(&router->inetaddr_nb); -err_register_inetaddr_notifier: mutex_destroy(&mlxsw_sp->router->lock); kfree(mlxsw_sp->router); return err; @@ -8188,6 +8180,9 @@ void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp), &mlxsw_sp->router->fib_nb); unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); + unregister_inet6addr_notifier(&mlxsw_sp->router->inet6addr_nb); + unregister_inetaddr_notifier(&mlxsw_sp->router->inetaddr_nb); + mlxsw_core_flush_owq(); mlxsw_sp_neigh_fini(mlxsw_sp); mlxsw_sp_vrs_fini(mlxsw_sp); mlxsw_sp_mr_fini(mlxsw_sp); @@ -8197,8 +8192,6 @@ void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) mlxsw_sp_ipips_fini(mlxsw_sp); mlxsw_sp_rifs_fini(mlxsw_sp); __mlxsw_sp_router_fini(mlxsw_sp); - unregister_inet6addr_notifier(&mlxsw_sp->router->inet6addr_nb); - unregister_inetaddr_notifier(&mlxsw_sp->router->inetaddr_nb); mutex_destroy(&mlxsw_sp->router->lock); kfree(mlxsw_sp->router); } diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c index 92351a79addc..5c959a995199 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c @@ -21,9 +21,14 @@ struct mlxsw_sp_span { struct work_struct work; struct mlxsw_sp *mlxsw_sp; + const struct mlxsw_sp_span_trigger_ops **span_trigger_ops_arr; + const struct mlxsw_sp_span_entry_ops **span_entry_ops_arr; + size_t span_entry_ops_arr_size; struct list_head analyzed_ports_list; struct mutex analyzed_ports_lock; /* Protects analyzed_ports_list */ struct list_head trigger_entries_list; + u16 policer_id_base; + refcount_t policer_id_base_ref_count; atomic_t active_entries_count; int entries_count; struct mlxsw_sp_span_entry entries[]; @@ -38,12 +43,31 @@ struct mlxsw_sp_span_analyzed_port { struct mlxsw_sp_span_trigger_entry { struct list_head list; /* Member of trigger_entries_list */ + struct mlxsw_sp_span *span; + const struct mlxsw_sp_span_trigger_ops *ops; refcount_t ref_count; u8 local_port; enum mlxsw_sp_span_trigger trigger; struct mlxsw_sp_span_trigger_parms parms; }; +enum mlxsw_sp_span_trigger_type { + MLXSW_SP_SPAN_TRIGGER_TYPE_PORT, + MLXSW_SP_SPAN_TRIGGER_TYPE_GLOBAL, +}; + +struct mlxsw_sp_span_trigger_ops { + int (*bind)(struct mlxsw_sp_span_trigger_entry *trigger_entry); + void (*unbind)(struct mlxsw_sp_span_trigger_entry *trigger_entry); + bool (*matches)(struct mlxsw_sp_span_trigger_entry *trigger_entry, + enum mlxsw_sp_span_trigger trigger, + struct mlxsw_sp_port *mlxsw_sp_port); + int (*enable)(struct mlxsw_sp_span_trigger_entry *trigger_entry, + struct mlxsw_sp_port *mlxsw_sp_port, u8 tc); + void (*disable)(struct mlxsw_sp_span_trigger_entry *trigger_entry, + struct mlxsw_sp_port *mlxsw_sp_port, u8 tc); +}; + static void mlxsw_sp_span_respin_work(struct work_struct *work); static u64 mlxsw_sp_span_occ_get(void *priv) @@ -57,7 +81,7 @@ int mlxsw_sp_span_init(struct mlxsw_sp *mlxsw_sp) { struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); struct mlxsw_sp_span *span; - int i, entries_count; + int i, entries_count, err; if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_SPAN)) return -EIO; @@ -66,6 +90,7 @@ int mlxsw_sp_span_init(struct mlxsw_sp *mlxsw_sp) span = kzalloc(struct_size(span, entries, entries_count), GFP_KERNEL); if (!span) return -ENOMEM; + refcount_set(&span->policer_id_base_ref_count, 0); span->entries_count = entries_count; atomic_set(&span->active_entries_count, 0); mutex_init(&span->analyzed_ports_lock); @@ -77,11 +102,20 @@ int mlxsw_sp_span_init(struct mlxsw_sp *mlxsw_sp) for (i = 0; i < mlxsw_sp->span->entries_count; i++) mlxsw_sp->span->entries[i].id = i; + err = mlxsw_sp->span_ops->init(mlxsw_sp); + if (err) + goto err_init; + devlink_resource_occ_get_register(devlink, MLXSW_SP_RESOURCE_SPAN, mlxsw_sp_span_occ_get, mlxsw_sp); INIT_WORK(&span->work, mlxsw_sp_span_respin_work); return 0; + +err_init: + mutex_destroy(&mlxsw_sp->span->analyzed_ports_lock); + kfree(mlxsw_sp->span); + return err; } void mlxsw_sp_span_fini(struct mlxsw_sp *mlxsw_sp) @@ -97,8 +131,41 @@ void mlxsw_sp_span_fini(struct mlxsw_sp *mlxsw_sp) kfree(mlxsw_sp->span); } +static bool mlxsw_sp1_span_cpu_can_handle(const struct net_device *dev) +{ + return !dev; +} + +static int mlxsw_sp1_span_entry_cpu_parms(struct mlxsw_sp *mlxsw_sp, + const struct net_device *to_dev, + struct mlxsw_sp_span_parms *sparmsp) +{ + return -EOPNOTSUPP; +} + +static int +mlxsw_sp1_span_entry_cpu_configure(struct mlxsw_sp_span_entry *span_entry, + struct mlxsw_sp_span_parms sparms) +{ + return -EOPNOTSUPP; +} + +static void +mlxsw_sp1_span_entry_cpu_deconfigure(struct mlxsw_sp_span_entry *span_entry) +{ +} + +static const +struct mlxsw_sp_span_entry_ops mlxsw_sp1_span_entry_ops_cpu = { + .can_handle = mlxsw_sp1_span_cpu_can_handle, + .parms_set = mlxsw_sp1_span_entry_cpu_parms, + .configure = mlxsw_sp1_span_entry_cpu_configure, + .deconfigure = mlxsw_sp1_span_entry_cpu_deconfigure, +}; + static int -mlxsw_sp_span_entry_phys_parms(const struct net_device *to_dev, +mlxsw_sp_span_entry_phys_parms(struct mlxsw_sp *mlxsw_sp, + const struct net_device *to_dev, struct mlxsw_sp_span_parms *sparmsp) { sparmsp->dest_port = netdev_priv(to_dev); @@ -118,6 +185,8 @@ mlxsw_sp_span_entry_phys_configure(struct mlxsw_sp_span_entry *span_entry, /* Create a new port analayzer entry for local_port. */ mlxsw_reg_mpat_pack(mpat_pl, pa_id, local_port, true, MLXSW_REG_MPAT_SPAN_TYPE_LOCAL_ETH); + mlxsw_reg_mpat_pide_set(mpat_pl, sparms.policer_enable); + mlxsw_reg_mpat_pid_set(mpat_pl, sparms.policer_id); return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(mpat), mpat_pl); } @@ -374,7 +443,8 @@ out: } static int -mlxsw_sp_span_entry_gretap4_parms(const struct net_device *to_dev, +mlxsw_sp_span_entry_gretap4_parms(struct mlxsw_sp *mlxsw_sp, + const struct net_device *to_dev, struct mlxsw_sp_span_parms *sparmsp) { struct ip_tunnel_parm tparm = mlxsw_sp_ipip_netdev_parms4(to_dev); @@ -413,6 +483,8 @@ mlxsw_sp_span_entry_gretap4_configure(struct mlxsw_sp_span_entry *span_entry, /* Create a new port analayzer entry for local_port. */ mlxsw_reg_mpat_pack(mpat_pl, pa_id, local_port, true, MLXSW_REG_MPAT_SPAN_TYPE_REMOTE_ETH_L3); + mlxsw_reg_mpat_pide_set(mpat_pl, sparms.policer_enable); + mlxsw_reg_mpat_pid_set(mpat_pl, sparms.policer_id); mlxsw_reg_mpat_eth_rspan_pack(mpat_pl, sparms.vid); mlxsw_reg_mpat_eth_rspan_l2_pack(mpat_pl, MLXSW_REG_MPAT_ETH_RSPAN_VERSION_NO_HEADER, @@ -475,7 +547,8 @@ out: } static int -mlxsw_sp_span_entry_gretap6_parms(const struct net_device *to_dev, +mlxsw_sp_span_entry_gretap6_parms(struct mlxsw_sp *mlxsw_sp, + const struct net_device *to_dev, struct mlxsw_sp_span_parms *sparmsp) { struct __ip6_tnl_parm tparm = mlxsw_sp_ipip_netdev_parms6(to_dev); @@ -514,6 +587,8 @@ mlxsw_sp_span_entry_gretap6_configure(struct mlxsw_sp_span_entry *span_entry, /* Create a new port analayzer entry for local_port. */ mlxsw_reg_mpat_pack(mpat_pl, pa_id, local_port, true, MLXSW_REG_MPAT_SPAN_TYPE_REMOTE_ETH_L3); + mlxsw_reg_mpat_pide_set(mpat_pl, sparms.policer_enable); + mlxsw_reg_mpat_pid_set(mpat_pl, sparms.policer_id); mlxsw_reg_mpat_eth_rspan_pack(mpat_pl, sparms.vid); mlxsw_reg_mpat_eth_rspan_l2_pack(mpat_pl, MLXSW_REG_MPAT_ETH_RSPAN_VERSION_NO_HEADER, @@ -549,7 +624,8 @@ mlxsw_sp_span_vlan_can_handle(const struct net_device *dev) } static int -mlxsw_sp_span_entry_vlan_parms(const struct net_device *to_dev, +mlxsw_sp_span_entry_vlan_parms(struct mlxsw_sp *mlxsw_sp, + const struct net_device *to_dev, struct mlxsw_sp_span_parms *sparmsp) { struct net_device *real_dev; @@ -576,6 +652,8 @@ mlxsw_sp_span_entry_vlan_configure(struct mlxsw_sp_span_entry *span_entry, mlxsw_reg_mpat_pack(mpat_pl, pa_id, local_port, true, MLXSW_REG_MPAT_SPAN_TYPE_REMOTE_ETH); + mlxsw_reg_mpat_pide_set(mpat_pl, sparms.policer_enable); + mlxsw_reg_mpat_pid_set(mpat_pl, sparms.policer_id); mlxsw_reg_mpat_eth_rspan_pack(mpat_pl, sparms.vid); return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(mpat), mpat_pl); @@ -597,7 +675,61 @@ struct mlxsw_sp_span_entry_ops mlxsw_sp_span_entry_ops_vlan = { }; static const -struct mlxsw_sp_span_entry_ops *const mlxsw_sp_span_entry_types[] = { +struct mlxsw_sp_span_entry_ops *mlxsw_sp1_span_entry_ops_arr[] = { + &mlxsw_sp1_span_entry_ops_cpu, + &mlxsw_sp_span_entry_ops_phys, +#if IS_ENABLED(CONFIG_NET_IPGRE) + &mlxsw_sp_span_entry_ops_gretap4, +#endif +#if IS_ENABLED(CONFIG_IPV6_GRE) + &mlxsw_sp_span_entry_ops_gretap6, +#endif + &mlxsw_sp_span_entry_ops_vlan, +}; + +static bool mlxsw_sp2_span_cpu_can_handle(const struct net_device *dev) +{ + return !dev; +} + +static int mlxsw_sp2_span_entry_cpu_parms(struct mlxsw_sp *mlxsw_sp, + const struct net_device *to_dev, + struct mlxsw_sp_span_parms *sparmsp) +{ + sparmsp->dest_port = mlxsw_sp->ports[MLXSW_PORT_CPU_PORT]; + return 0; +} + +static int +mlxsw_sp2_span_entry_cpu_configure(struct mlxsw_sp_span_entry *span_entry, + struct mlxsw_sp_span_parms sparms) +{ + /* Mirroring to the CPU port is like mirroring to any other physical + * port. Its local port is used instead of that of the physical port. + */ + return mlxsw_sp_span_entry_phys_configure(span_entry, sparms); +} + +static void +mlxsw_sp2_span_entry_cpu_deconfigure(struct mlxsw_sp_span_entry *span_entry) +{ + enum mlxsw_reg_mpat_span_type span_type; + + span_type = MLXSW_REG_MPAT_SPAN_TYPE_LOCAL_ETH; + mlxsw_sp_span_entry_deconfigure_common(span_entry, span_type); +} + +static const +struct mlxsw_sp_span_entry_ops mlxsw_sp2_span_entry_ops_cpu = { + .can_handle = mlxsw_sp2_span_cpu_can_handle, + .parms_set = mlxsw_sp2_span_entry_cpu_parms, + .configure = mlxsw_sp2_span_entry_cpu_configure, + .deconfigure = mlxsw_sp2_span_entry_cpu_deconfigure, +}; + +static const +struct mlxsw_sp_span_entry_ops *mlxsw_sp2_span_entry_ops_arr[] = { + &mlxsw_sp2_span_entry_ops_cpu, &mlxsw_sp_span_entry_ops_phys, #if IS_ENABLED(CONFIG_NET_IPGRE) &mlxsw_sp_span_entry_ops_gretap4, @@ -609,7 +741,8 @@ struct mlxsw_sp_span_entry_ops *const mlxsw_sp_span_entry_types[] = { }; static int -mlxsw_sp_span_entry_nop_parms(const struct net_device *to_dev, +mlxsw_sp_span_entry_nop_parms(struct mlxsw_sp *mlxsw_sp, + const struct net_device *to_dev, struct mlxsw_sp_span_parms *sparmsp) { return mlxsw_sp_span_entry_unoffloadable(sparmsp); @@ -644,16 +777,15 @@ mlxsw_sp_span_entry_configure(struct mlxsw_sp *mlxsw_sp, goto set_parms; if (sparms.dest_port->mlxsw_sp != mlxsw_sp) { - netdev_err(span_entry->to_dev, "Cannot mirror to %s, which belongs to a different mlxsw instance", - sparms.dest_port->dev->name); + dev_err(mlxsw_sp->bus_info->dev, + "Cannot mirror to a port which belongs to a different mlxsw instance\n"); sparms.dest_port = NULL; goto set_parms; } err = span_entry->ops->configure(span_entry, sparms); if (err) { - netdev_err(span_entry->to_dev, "Failed to offload mirror to %s", - sparms.dest_port->dev->name); + dev_err(mlxsw_sp->bus_info->dev, "Failed to offload mirror\n"); sparms.dest_port = NULL; goto set_parms; } @@ -669,6 +801,46 @@ mlxsw_sp_span_entry_deconfigure(struct mlxsw_sp_span_entry *span_entry) span_entry->ops->deconfigure(span_entry); } +static int mlxsw_sp_span_policer_id_base_set(struct mlxsw_sp_span *span, + u16 policer_id) +{ + struct mlxsw_sp *mlxsw_sp = span->mlxsw_sp; + u16 policer_id_base; + int err; + + /* Policers set on SPAN agents must be in the range of + * `policer_id_base .. policer_id_base + max_span_agents - 1`. If the + * base is set and the new policer is not within the range, then we + * must error out. + */ + if (refcount_read(&span->policer_id_base_ref_count)) { + if (policer_id < span->policer_id_base || + policer_id >= span->policer_id_base + span->entries_count) + return -EINVAL; + + refcount_inc(&span->policer_id_base_ref_count); + return 0; + } + + /* Base must be even. */ + policer_id_base = policer_id % 2 == 0 ? policer_id : policer_id - 1; + err = mlxsw_sp->span_ops->policer_id_base_set(mlxsw_sp, + policer_id_base); + if (err) + return err; + + span->policer_id_base = policer_id_base; + refcount_set(&span->policer_id_base_ref_count, 1); + + return 0; +} + +static void mlxsw_sp_span_policer_id_base_unset(struct mlxsw_sp_span *span) +{ + if (refcount_dec_and_test(&span->policer_id_base_ref_count)) + span->policer_id_base = 0; +} + static struct mlxsw_sp_span_entry * mlxsw_sp_span_entry_create(struct mlxsw_sp *mlxsw_sp, const struct net_device *to_dev, @@ -688,6 +860,15 @@ mlxsw_sp_span_entry_create(struct mlxsw_sp *mlxsw_sp, if (!span_entry) return NULL; + if (sparms.policer_enable) { + int err; + + err = mlxsw_sp_span_policer_id_base_set(mlxsw_sp->span, + sparms.policer_id); + if (err) + return NULL; + } + atomic_inc(&mlxsw_sp->span->active_entries_count); span_entry->ops = ops; refcount_set(&span_entry->ref_count, 1); @@ -702,6 +883,8 @@ static void mlxsw_sp_span_entry_destroy(struct mlxsw_sp *mlxsw_sp, { mlxsw_sp_span_entry_deconfigure(span_entry); atomic_dec(&mlxsw_sp->span->active_entries_count); + if (span_entry->parms.policer_enable) + mlxsw_sp_span_policer_id_base_unset(mlxsw_sp->span); } struct mlxsw_sp_span_entry * @@ -741,6 +924,24 @@ mlxsw_sp_span_entry_find_by_id(struct mlxsw_sp *mlxsw_sp, int span_id) } static struct mlxsw_sp_span_entry * +mlxsw_sp_span_entry_find_by_parms(struct mlxsw_sp *mlxsw_sp, + const struct net_device *to_dev, + const struct mlxsw_sp_span_parms *sparms) +{ + int i; + + for (i = 0; i < mlxsw_sp->span->entries_count; i++) { + struct mlxsw_sp_span_entry *curr = &mlxsw_sp->span->entries[i]; + + if (refcount_read(&curr->ref_count) && curr->to_dev == to_dev && + curr->parms.policer_enable == sparms->policer_enable && + curr->parms.policer_id == sparms->policer_id) + return curr; + } + return NULL; +} + +static struct mlxsw_sp_span_entry * mlxsw_sp_span_entry_get(struct mlxsw_sp *mlxsw_sp, const struct net_device *to_dev, const struct mlxsw_sp_span_entry_ops *ops, @@ -748,7 +949,8 @@ mlxsw_sp_span_entry_get(struct mlxsw_sp *mlxsw_sp, { struct mlxsw_sp_span_entry *span_entry; - span_entry = mlxsw_sp_span_entry_find_by_port(mlxsw_sp, to_dev); + span_entry = mlxsw_sp_span_entry_find_by_parms(mlxsw_sp, to_dev, + &sparms); if (span_entry) { /* Already exists, just take a reference */ refcount_inc(&span_entry->ref_count); @@ -766,6 +968,14 @@ static int mlxsw_sp_span_entry_put(struct mlxsw_sp *mlxsw_sp, return 0; } +static u32 mlxsw_sp_span_buffsize_get(struct mlxsw_sp *mlxsw_sp, int mtu, + u32 speed) +{ + u32 buffsize = mlxsw_sp->span_ops->buffsize_get(speed, mtu); + + return mlxsw_sp_bytes_cells(mlxsw_sp, buffsize) + 1; +} + static int mlxsw_sp_span_port_buffer_update(struct mlxsw_sp_port *mlxsw_sp_port, u16 mtu) { @@ -857,11 +1067,12 @@ static const struct mlxsw_sp_span_entry_ops * mlxsw_sp_span_entry_ops(struct mlxsw_sp *mlxsw_sp, const struct net_device *to_dev) { + struct mlxsw_sp_span *span = mlxsw_sp->span; size_t i; - for (i = 0; i < ARRAY_SIZE(mlxsw_sp_span_entry_types); ++i) - if (mlxsw_sp_span_entry_types[i]->can_handle(to_dev)) - return mlxsw_sp_span_entry_types[i]; + for (i = 0; i < span->span_entry_ops_arr_size; ++i) + if (span->span_entry_ops_arr[i]->can_handle(to_dev)) + return span->span_entry_ops_arr[i]; return NULL; } @@ -883,7 +1094,7 @@ static void mlxsw_sp_span_respin_work(struct work_struct *work) if (!refcount_read(&curr->ref_count)) continue; - err = curr->ops->parms_set(curr->to_dev, &sparms); + err = curr->ops->parms_set(mlxsw_sp, curr->to_dev, &sparms); if (err) continue; @@ -902,9 +1113,10 @@ void mlxsw_sp_span_respin(struct mlxsw_sp *mlxsw_sp) mlxsw_core_schedule_work(&mlxsw_sp->span->work); } -int mlxsw_sp_span_agent_get(struct mlxsw_sp *mlxsw_sp, - const struct net_device *to_dev, int *p_span_id) +int mlxsw_sp_span_agent_get(struct mlxsw_sp *mlxsw_sp, int *p_span_id, + const struct mlxsw_sp_span_agent_parms *parms) { + const struct net_device *to_dev = parms->to_dev; const struct mlxsw_sp_span_entry_ops *ops; struct mlxsw_sp_span_entry *span_entry; struct mlxsw_sp_span_parms sparms; @@ -919,10 +1131,12 @@ int mlxsw_sp_span_agent_get(struct mlxsw_sp *mlxsw_sp, } memset(&sparms, 0, sizeof(sparms)); - err = ops->parms_set(to_dev, &sparms); + err = ops->parms_set(mlxsw_sp, to_dev, &sparms); if (err) return err; + sparms.policer_id = parms->policer_id; + sparms.policer_enable = parms->policer_enable; span_entry = mlxsw_sp_span_entry_get(mlxsw_sp, to_dev, ops, sparms); if (!span_entry) return -ENOBUFS; @@ -1051,9 +1265,9 @@ out_unlock: } static int -__mlxsw_sp_span_trigger_entry_bind(struct mlxsw_sp_span *span, - struct mlxsw_sp_span_trigger_entry * - trigger_entry, bool enable) +__mlxsw_sp_span_trigger_port_bind(struct mlxsw_sp_span *span, + struct mlxsw_sp_span_trigger_entry * + trigger_entry, bool enable) { char mpar_pl[MLXSW_REG_MPAR_LEN]; enum mlxsw_reg_mpar_i_e i_e; @@ -1076,19 +1290,254 @@ __mlxsw_sp_span_trigger_entry_bind(struct mlxsw_sp_span *span, } static int -mlxsw_sp_span_trigger_entry_bind(struct mlxsw_sp_span *span, - struct mlxsw_sp_span_trigger_entry * - trigger_entry) +mlxsw_sp_span_trigger_port_bind(struct mlxsw_sp_span_trigger_entry * + trigger_entry) +{ + return __mlxsw_sp_span_trigger_port_bind(trigger_entry->span, + trigger_entry, true); +} + +static void +mlxsw_sp_span_trigger_port_unbind(struct mlxsw_sp_span_trigger_entry * + trigger_entry) +{ + __mlxsw_sp_span_trigger_port_bind(trigger_entry->span, trigger_entry, + false); +} + +static bool +mlxsw_sp_span_trigger_port_matches(struct mlxsw_sp_span_trigger_entry * + trigger_entry, + enum mlxsw_sp_span_trigger trigger, + struct mlxsw_sp_port *mlxsw_sp_port) +{ + return trigger_entry->trigger == trigger && + trigger_entry->local_port == mlxsw_sp_port->local_port; +} + +static int +mlxsw_sp_span_trigger_port_enable(struct mlxsw_sp_span_trigger_entry * + trigger_entry, + struct mlxsw_sp_port *mlxsw_sp_port, u8 tc) { - return __mlxsw_sp_span_trigger_entry_bind(span, trigger_entry, true); + /* Port trigger are enabled during binding. */ + return 0; } static void -mlxsw_sp_span_trigger_entry_unbind(struct mlxsw_sp_span *span, - struct mlxsw_sp_span_trigger_entry * +mlxsw_sp_span_trigger_port_disable(struct mlxsw_sp_span_trigger_entry * + trigger_entry, + struct mlxsw_sp_port *mlxsw_sp_port, u8 tc) +{ +} + +static const struct mlxsw_sp_span_trigger_ops +mlxsw_sp_span_trigger_port_ops = { + .bind = mlxsw_sp_span_trigger_port_bind, + .unbind = mlxsw_sp_span_trigger_port_unbind, + .matches = mlxsw_sp_span_trigger_port_matches, + .enable = mlxsw_sp_span_trigger_port_enable, + .disable = mlxsw_sp_span_trigger_port_disable, +}; + +static int +mlxsw_sp1_span_trigger_global_bind(struct mlxsw_sp_span_trigger_entry * trigger_entry) { - __mlxsw_sp_span_trigger_entry_bind(span, trigger_entry, false); + return -EOPNOTSUPP; +} + +static void +mlxsw_sp1_span_trigger_global_unbind(struct mlxsw_sp_span_trigger_entry * + trigger_entry) +{ +} + +static bool +mlxsw_sp1_span_trigger_global_matches(struct mlxsw_sp_span_trigger_entry * + trigger_entry, + enum mlxsw_sp_span_trigger trigger, + struct mlxsw_sp_port *mlxsw_sp_port) +{ + WARN_ON_ONCE(1); + return false; +} + +static int +mlxsw_sp1_span_trigger_global_enable(struct mlxsw_sp_span_trigger_entry * + trigger_entry, + struct mlxsw_sp_port *mlxsw_sp_port, + u8 tc) +{ + return -EOPNOTSUPP; +} + +static void +mlxsw_sp1_span_trigger_global_disable(struct mlxsw_sp_span_trigger_entry * + trigger_entry, + struct mlxsw_sp_port *mlxsw_sp_port, + u8 tc) +{ +} + +static const struct mlxsw_sp_span_trigger_ops +mlxsw_sp1_span_trigger_global_ops = { + .bind = mlxsw_sp1_span_trigger_global_bind, + .unbind = mlxsw_sp1_span_trigger_global_unbind, + .matches = mlxsw_sp1_span_trigger_global_matches, + .enable = mlxsw_sp1_span_trigger_global_enable, + .disable = mlxsw_sp1_span_trigger_global_disable, +}; + +static const struct mlxsw_sp_span_trigger_ops * +mlxsw_sp1_span_trigger_ops_arr[] = { + [MLXSW_SP_SPAN_TRIGGER_TYPE_PORT] = &mlxsw_sp_span_trigger_port_ops, + [MLXSW_SP_SPAN_TRIGGER_TYPE_GLOBAL] = + &mlxsw_sp1_span_trigger_global_ops, +}; + +static int +mlxsw_sp2_span_trigger_global_bind(struct mlxsw_sp_span_trigger_entry * + trigger_entry) +{ + struct mlxsw_sp *mlxsw_sp = trigger_entry->span->mlxsw_sp; + enum mlxsw_reg_mpagr_trigger trigger; + char mpagr_pl[MLXSW_REG_MPAGR_LEN]; + + switch (trigger_entry->trigger) { + case MLXSW_SP_SPAN_TRIGGER_TAIL_DROP: + trigger = MLXSW_REG_MPAGR_TRIGGER_INGRESS_SHARED_BUFFER; + break; + case MLXSW_SP_SPAN_TRIGGER_EARLY_DROP: + trigger = MLXSW_REG_MPAGR_TRIGGER_INGRESS_WRED; + break; + case MLXSW_SP_SPAN_TRIGGER_ECN: + trigger = MLXSW_REG_MPAGR_TRIGGER_EGRESS_ECN; + break; + default: + WARN_ON_ONCE(1); + return -EINVAL; + } + + mlxsw_reg_mpagr_pack(mpagr_pl, trigger, trigger_entry->parms.span_id, + 1); + return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(mpagr), mpagr_pl); +} + +static void +mlxsw_sp2_span_trigger_global_unbind(struct mlxsw_sp_span_trigger_entry * + trigger_entry) +{ + /* There is no unbinding for global triggers. The trigger should be + * disabled on all ports by now. + */ +} + +static bool +mlxsw_sp2_span_trigger_global_matches(struct mlxsw_sp_span_trigger_entry * + trigger_entry, + enum mlxsw_sp_span_trigger trigger, + struct mlxsw_sp_port *mlxsw_sp_port) +{ + return trigger_entry->trigger == trigger; +} + +static int +__mlxsw_sp2_span_trigger_global_enable(struct mlxsw_sp_span_trigger_entry * + trigger_entry, + struct mlxsw_sp_port *mlxsw_sp_port, + u8 tc, bool enable) +{ + struct mlxsw_sp *mlxsw_sp = trigger_entry->span->mlxsw_sp; + char momte_pl[MLXSW_REG_MOMTE_LEN]; + enum mlxsw_reg_momte_type type; + int err; + + switch (trigger_entry->trigger) { + case MLXSW_SP_SPAN_TRIGGER_TAIL_DROP: + type = MLXSW_REG_MOMTE_TYPE_SHARED_BUFFER_TCLASS; + break; + case MLXSW_SP_SPAN_TRIGGER_EARLY_DROP: + type = MLXSW_REG_MOMTE_TYPE_WRED; + break; + case MLXSW_SP_SPAN_TRIGGER_ECN: + type = MLXSW_REG_MOMTE_TYPE_ECN; + break; + default: + WARN_ON_ONCE(1); + return -EINVAL; + } + + /* Query existing configuration in order to only change the state of + * the specified traffic class. + */ + mlxsw_reg_momte_pack(momte_pl, mlxsw_sp_port->local_port, type); + err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(momte), momte_pl); + if (err) + return err; + + mlxsw_reg_momte_tclass_en_set(momte_pl, tc, enable); + return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(momte), momte_pl); +} + +static int +mlxsw_sp2_span_trigger_global_enable(struct mlxsw_sp_span_trigger_entry * + trigger_entry, + struct mlxsw_sp_port *mlxsw_sp_port, + u8 tc) +{ + return __mlxsw_sp2_span_trigger_global_enable(trigger_entry, + mlxsw_sp_port, tc, true); +} + +static void +mlxsw_sp2_span_trigger_global_disable(struct mlxsw_sp_span_trigger_entry * + trigger_entry, + struct mlxsw_sp_port *mlxsw_sp_port, + u8 tc) +{ + __mlxsw_sp2_span_trigger_global_enable(trigger_entry, mlxsw_sp_port, tc, + false); +} + +static const struct mlxsw_sp_span_trigger_ops +mlxsw_sp2_span_trigger_global_ops = { + .bind = mlxsw_sp2_span_trigger_global_bind, + .unbind = mlxsw_sp2_span_trigger_global_unbind, + .matches = mlxsw_sp2_span_trigger_global_matches, + .enable = mlxsw_sp2_span_trigger_global_enable, + .disable = mlxsw_sp2_span_trigger_global_disable, +}; + +static const struct mlxsw_sp_span_trigger_ops * +mlxsw_sp2_span_trigger_ops_arr[] = { + [MLXSW_SP_SPAN_TRIGGER_TYPE_PORT] = &mlxsw_sp_span_trigger_port_ops, + [MLXSW_SP_SPAN_TRIGGER_TYPE_GLOBAL] = + &mlxsw_sp2_span_trigger_global_ops, +}; + +static void +mlxsw_sp_span_trigger_ops_set(struct mlxsw_sp_span_trigger_entry *trigger_entry) +{ + struct mlxsw_sp_span *span = trigger_entry->span; + enum mlxsw_sp_span_trigger_type type; + + switch (trigger_entry->trigger) { + case MLXSW_SP_SPAN_TRIGGER_INGRESS: /* fall-through */ + case MLXSW_SP_SPAN_TRIGGER_EGRESS: + type = MLXSW_SP_SPAN_TRIGGER_TYPE_PORT; + break; + case MLXSW_SP_SPAN_TRIGGER_TAIL_DROP: /* fall-through */ + case MLXSW_SP_SPAN_TRIGGER_EARLY_DROP: /* fall-through */ + case MLXSW_SP_SPAN_TRIGGER_ECN: + type = MLXSW_SP_SPAN_TRIGGER_TYPE_GLOBAL; + break; + default: + WARN_ON_ONCE(1); + return; + } + + trigger_entry->ops = span->span_trigger_ops_arr[type]; } static struct mlxsw_sp_span_trigger_entry * @@ -1106,12 +1555,15 @@ mlxsw_sp_span_trigger_entry_create(struct mlxsw_sp_span *span, return ERR_PTR(-ENOMEM); refcount_set(&trigger_entry->ref_count, 1); - trigger_entry->local_port = mlxsw_sp_port->local_port; + trigger_entry->local_port = mlxsw_sp_port ? mlxsw_sp_port->local_port : + 0; trigger_entry->trigger = trigger; memcpy(&trigger_entry->parms, parms, sizeof(trigger_entry->parms)); + trigger_entry->span = span; + mlxsw_sp_span_trigger_ops_set(trigger_entry); list_add_tail(&trigger_entry->list, &span->trigger_entries_list); - err = mlxsw_sp_span_trigger_entry_bind(span, trigger_entry); + err = trigger_entry->ops->bind(trigger_entry); if (err) goto err_trigger_entry_bind; @@ -1128,7 +1580,7 @@ mlxsw_sp_span_trigger_entry_destroy(struct mlxsw_sp_span *span, struct mlxsw_sp_span_trigger_entry * trigger_entry) { - mlxsw_sp_span_trigger_entry_unbind(span, trigger_entry); + trigger_entry->ops->unbind(trigger_entry); list_del(&trigger_entry->list); kfree(trigger_entry); } @@ -1141,8 +1593,8 @@ mlxsw_sp_span_trigger_entry_find(struct mlxsw_sp_span *span, struct mlxsw_sp_span_trigger_entry *trigger_entry; list_for_each_entry(trigger_entry, &span->trigger_entries_list, list) { - if (trigger_entry->trigger == trigger && - trigger_entry->local_port == mlxsw_sp_port->local_port) + if (trigger_entry->ops->matches(trigger_entry, trigger, + mlxsw_sp_port)) return trigger_entry; } @@ -1207,3 +1659,138 @@ void mlxsw_sp_span_agent_unbind(struct mlxsw_sp *mlxsw_sp, mlxsw_sp_span_trigger_entry_destroy(mlxsw_sp->span, trigger_entry); } + +int mlxsw_sp_span_trigger_enable(struct mlxsw_sp_port *mlxsw_sp_port, + enum mlxsw_sp_span_trigger trigger, u8 tc) +{ + struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; + struct mlxsw_sp_span_trigger_entry *trigger_entry; + + ASSERT_RTNL(); + + trigger_entry = mlxsw_sp_span_trigger_entry_find(mlxsw_sp->span, + trigger, + mlxsw_sp_port); + if (WARN_ON_ONCE(!trigger_entry)) + return -EINVAL; + + return trigger_entry->ops->enable(trigger_entry, mlxsw_sp_port, tc); +} + +void mlxsw_sp_span_trigger_disable(struct mlxsw_sp_port *mlxsw_sp_port, + enum mlxsw_sp_span_trigger trigger, u8 tc) +{ + struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; + struct mlxsw_sp_span_trigger_entry *trigger_entry; + + ASSERT_RTNL(); + + trigger_entry = mlxsw_sp_span_trigger_entry_find(mlxsw_sp->span, + trigger, + mlxsw_sp_port); + if (WARN_ON_ONCE(!trigger_entry)) + return; + + return trigger_entry->ops->disable(trigger_entry, mlxsw_sp_port, tc); +} + +static int mlxsw_sp1_span_init(struct mlxsw_sp *mlxsw_sp) +{ + size_t arr_size = ARRAY_SIZE(mlxsw_sp1_span_entry_ops_arr); + + /* Must be first to avoid NULL pointer dereference by subsequent + * can_handle() callbacks. + */ + if (WARN_ON(mlxsw_sp1_span_entry_ops_arr[0] != + &mlxsw_sp1_span_entry_ops_cpu)) + return -EINVAL; + + mlxsw_sp->span->span_trigger_ops_arr = mlxsw_sp1_span_trigger_ops_arr; + mlxsw_sp->span->span_entry_ops_arr = mlxsw_sp1_span_entry_ops_arr; + mlxsw_sp->span->span_entry_ops_arr_size = arr_size; + + return 0; +} + +static u32 mlxsw_sp1_span_buffsize_get(int mtu, u32 speed) +{ + return mtu * 5 / 2; +} + +static int mlxsw_sp1_span_policer_id_base_set(struct mlxsw_sp *mlxsw_sp, + u16 policer_id_base) +{ + return -EOPNOTSUPP; +} + +const struct mlxsw_sp_span_ops mlxsw_sp1_span_ops = { + .init = mlxsw_sp1_span_init, + .buffsize_get = mlxsw_sp1_span_buffsize_get, + .policer_id_base_set = mlxsw_sp1_span_policer_id_base_set, +}; + +static int mlxsw_sp2_span_init(struct mlxsw_sp *mlxsw_sp) +{ + size_t arr_size = ARRAY_SIZE(mlxsw_sp2_span_entry_ops_arr); + + /* Must be first to avoid NULL pointer dereference by subsequent + * can_handle() callbacks. + */ + if (WARN_ON(mlxsw_sp2_span_entry_ops_arr[0] != + &mlxsw_sp2_span_entry_ops_cpu)) + return -EINVAL; + + mlxsw_sp->span->span_trigger_ops_arr = mlxsw_sp2_span_trigger_ops_arr; + mlxsw_sp->span->span_entry_ops_arr = mlxsw_sp2_span_entry_ops_arr; + mlxsw_sp->span->span_entry_ops_arr_size = arr_size; + + return 0; +} + +#define MLXSW_SP2_SPAN_EG_MIRROR_BUFFER_FACTOR 38 +#define MLXSW_SP3_SPAN_EG_MIRROR_BUFFER_FACTOR 50 + +static u32 __mlxsw_sp_span_buffsize_get(int mtu, u32 speed, u32 buffer_factor) +{ + return 3 * mtu + buffer_factor * speed / 1000; +} + +static u32 mlxsw_sp2_span_buffsize_get(int mtu, u32 speed) +{ + int factor = MLXSW_SP2_SPAN_EG_MIRROR_BUFFER_FACTOR; + + return __mlxsw_sp_span_buffsize_get(mtu, speed, factor); +} + +static int mlxsw_sp2_span_policer_id_base_set(struct mlxsw_sp *mlxsw_sp, + u16 policer_id_base) +{ + char mogcr_pl[MLXSW_REG_MOGCR_LEN]; + int err; + + err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(mogcr), mogcr_pl); + if (err) + return err; + + mlxsw_reg_mogcr_mirroring_pid_base_set(mogcr_pl, policer_id_base); + return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(mogcr), mogcr_pl); +} + +const struct mlxsw_sp_span_ops mlxsw_sp2_span_ops = { + .init = mlxsw_sp2_span_init, + .buffsize_get = mlxsw_sp2_span_buffsize_get, + .policer_id_base_set = mlxsw_sp2_span_policer_id_base_set, +}; + +static u32 mlxsw_sp3_span_buffsize_get(int mtu, u32 speed) +{ + int factor = MLXSW_SP3_SPAN_EG_MIRROR_BUFFER_FACTOR; + + return __mlxsw_sp_span_buffsize_get(mtu, speed, factor); +} + +const struct mlxsw_sp_span_ops mlxsw_sp3_span_ops = { + .init = mlxsw_sp2_span_init, + .buffsize_get = mlxsw_sp3_span_buffsize_get, + .policer_id_base_set = mlxsw_sp2_span_policer_id_base_set, +}; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.h index 9f6dd2d0f4e6..1c746dd3b1bd 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.h +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.h @@ -21,19 +21,37 @@ struct mlxsw_sp_span_parms { union mlxsw_sp_l3addr daddr; union mlxsw_sp_l3addr saddr; u16 vid; + u16 policer_id; + bool policer_enable; }; enum mlxsw_sp_span_trigger { MLXSW_SP_SPAN_TRIGGER_INGRESS, MLXSW_SP_SPAN_TRIGGER_EGRESS, + MLXSW_SP_SPAN_TRIGGER_TAIL_DROP, + MLXSW_SP_SPAN_TRIGGER_EARLY_DROP, + MLXSW_SP_SPAN_TRIGGER_ECN, }; struct mlxsw_sp_span_trigger_parms { int span_id; }; +struct mlxsw_sp_span_agent_parms { + const struct net_device *to_dev; + u16 policer_id; + bool policer_enable; +}; + struct mlxsw_sp_span_entry_ops; +struct mlxsw_sp_span_ops { + int (*init)(struct mlxsw_sp *mlxsw_sp); + u32 (*buffsize_get)(int mtu, u32 speed); + int (*policer_id_base_set)(struct mlxsw_sp *mlxsw_sp, + u16 policer_id_base); +}; + struct mlxsw_sp_span_entry { const struct net_device *to_dev; const struct mlxsw_sp_span_entry_ops *ops; @@ -44,7 +62,8 @@ struct mlxsw_sp_span_entry { struct mlxsw_sp_span_entry_ops { bool (*can_handle)(const struct net_device *to_dev); - int (*parms_set)(const struct net_device *to_dev, + int (*parms_set)(struct mlxsw_sp *mlxsw_sp, + const struct net_device *to_dev, struct mlxsw_sp_span_parms *sparmsp); int (*configure)(struct mlxsw_sp_span_entry *span_entry, struct mlxsw_sp_span_parms sparms); @@ -65,8 +84,8 @@ void mlxsw_sp_span_entry_invalidate(struct mlxsw_sp *mlxsw_sp, int mlxsw_sp_span_port_mtu_update(struct mlxsw_sp_port *port, u16 mtu); void mlxsw_sp_span_speed_update_work(struct work_struct *work); -int mlxsw_sp_span_agent_get(struct mlxsw_sp *mlxsw_sp, - const struct net_device *to_dev, int *p_span_id); +int mlxsw_sp_span_agent_get(struct mlxsw_sp *mlxsw_sp, int *p_span_id, + const struct mlxsw_sp_span_agent_parms *parms); void mlxsw_sp_span_agent_put(struct mlxsw_sp *mlxsw_sp, int span_id); int mlxsw_sp_span_analyzed_port_get(struct mlxsw_sp_port *mlxsw_sp_port, bool ingress); @@ -81,5 +100,13 @@ mlxsw_sp_span_agent_unbind(struct mlxsw_sp *mlxsw_sp, enum mlxsw_sp_span_trigger trigger, struct mlxsw_sp_port *mlxsw_sp_port, const struct mlxsw_sp_span_trigger_parms *parms); +int mlxsw_sp_span_trigger_enable(struct mlxsw_sp_port *mlxsw_sp_port, + enum mlxsw_sp_span_trigger trigger, u8 tc); +void mlxsw_sp_span_trigger_disable(struct mlxsw_sp_port *mlxsw_sp_port, + enum mlxsw_sp_span_trigger trigger, u8 tc); + +extern const struct mlxsw_sp_span_ops mlxsw_sp1_span_ops; +extern const struct mlxsw_sp_span_ops mlxsw_sp2_span_ops; +extern const struct mlxsw_sp_span_ops mlxsw_sp3_span_ops; #endif diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_trap.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_trap.c index 157a42c63066..2e41c5519c1b 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_trap.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_trap.c @@ -21,6 +21,7 @@ struct mlxsw_sp_trap_group_item { struct devlink_trap_group group; u16 hw_group_id; u8 priority; + u8 fixed_policer:1; /* Whether policer binding can change */ }; #define MLXSW_SP_TRAP_LISTENERS_MAX 3 @@ -28,6 +29,7 @@ struct mlxsw_sp_trap_group_item { struct mlxsw_sp_trap_item { struct devlink_trap trap; struct mlxsw_listener listeners_arr[MLXSW_SP_TRAP_LISTENERS_MAX]; + u8 is_source:1; }; /* All driver-specific traps must be documented in @@ -46,6 +48,11 @@ enum { #define MLXSW_SP_TRAP_METADATA DEVLINK_TRAP_METADATA_TYPE_F_IN_PORT +enum { + /* Packet was early dropped. */ + MLXSW_SP_MIRROR_REASON_INGRESS_WRED = 9, +}; + static int mlxsw_sp_rx_listener(struct mlxsw_sp *mlxsw_sp, struct sk_buff *skb, u8 local_port, struct mlxsw_sp_port *mlxsw_sp_port) @@ -222,6 +229,11 @@ static void mlxsw_sp_rx_sample_listener(struct sk_buff *skb, u8 local_port, DEVLINK_TRAP_GROUP_GENERIC_ID_##_group_id, \ MLXSW_SP_TRAP_METADATA | (_metadata)) +#define MLXSW_SP_TRAP_BUFFER_DROP(_id) \ + DEVLINK_TRAP_GENERIC(DROP, TRAP, _id, \ + DEVLINK_TRAP_GROUP_GENERIC_ID_BUFFER_DROPS, \ + MLXSW_SP_TRAP_METADATA) + #define MLXSW_SP_TRAP_DRIVER_DROP(_id, _group_id) \ DEVLINK_TRAP_DRIVER(DROP, DROP, DEVLINK_MLXSW_TRAP_ID_##_id, \ DEVLINK_MLXSW_TRAP_NAME_##_id, \ @@ -248,6 +260,10 @@ static void mlxsw_sp_rx_sample_listener(struct sk_buff *skb, u8 local_port, TRAP_EXCEPTION_TO_CPU, false, SP_##_en_group_id, \ SET_FW_DEFAULT, SP_##_dis_group_id) +#define MLXSW_SP_RXL_BUFFER_DISCARD(_mirror_reason) \ + MLXSW_RXL_MIRROR(mlxsw_sp_rx_drop_listener, 0, SP_BUFFER_DISCARDS, \ + MLXSW_SP_MIRROR_REASON_##_mirror_reason) + #define MLXSW_SP_RXL_EXCEPTION(_id, _group_id, _action) \ MLXSW_RXL(mlxsw_sp_rx_mark_listener, _id, \ _action, false, SP_##_group_id, SET_FW_DEFAULT) @@ -328,6 +344,12 @@ mlxsw_sp_trap_policer_items_arr[] = { { .policer = MLXSW_SP_TRAP_POLICER(18, 1024, 128), }, + { + .policer = MLXSW_SP_TRAP_POLICER(19, 1024, 512), + }, + { + .policer = MLXSW_SP_TRAP_POLICER(20, 10240, 4096), + }, }; static const struct mlxsw_sp_trap_group_item mlxsw_sp_trap_group_items_arr[] = { @@ -422,6 +444,11 @@ static const struct mlxsw_sp_trap_group_item mlxsw_sp_trap_group_items_arr[] = { .priority = 2, }, { + .group = DEVLINK_TRAP_GROUP_GENERIC(EXTERNAL_DELIVERY, 19), + .hw_group_id = MLXSW_REG_HTGT_TRAP_GROUP_SP_EXTERNAL_ROUTE, + .priority = 1, + }, + { .group = DEVLINK_TRAP_GROUP_GENERIC(IPV6, 15), .hw_group_id = MLXSW_REG_HTGT_TRAP_GROUP_SP_IPV6, .priority = 2, @@ -882,11 +909,11 @@ static const struct mlxsw_sp_trap_item mlxsw_sp_trap_items_arr[] = { }, }, { - .trap = MLXSW_SP_TRAP_CONTROL(EXTERNAL_ROUTE, LOCAL_DELIVERY, + .trap = MLXSW_SP_TRAP_CONTROL(EXTERNAL_ROUTE, EXTERNAL_DELIVERY, TRAP), .listeners_arr = { - MLXSW_SP_RXL_MARK(RTR_INGRESS0, IP2ME, TRAP_TO_CPU, - false), + MLXSW_SP_RXL_MARK(RTR_INGRESS0, EXTERNAL_ROUTE, + TRAP_TO_CPU, false), }, }, { @@ -1055,10 +1082,10 @@ static int mlxsw_sp_trap_dummy_group_init(struct mlxsw_sp *mlxsw_sp) static int mlxsw_sp_trap_policer_items_arr_init(struct mlxsw_sp *mlxsw_sp) { + size_t arr_size = ARRAY_SIZE(mlxsw_sp_trap_policer_items_arr); size_t elem_size = sizeof(struct mlxsw_sp_trap_policer_item); - u64 arr_size = ARRAY_SIZE(mlxsw_sp_trap_policer_items_arr); struct mlxsw_sp_trap *trap = mlxsw_sp->trap; - u64 free_policers = 0; + size_t free_policers = 0; u32 last_id; int i; @@ -1151,6 +1178,43 @@ static void mlxsw_sp_trap_policers_fini(struct mlxsw_sp *mlxsw_sp) mlxsw_sp_trap_policer_items_arr_fini(mlxsw_sp); } +static int mlxsw_sp_trap_group_items_arr_init(struct mlxsw_sp *mlxsw_sp) +{ + size_t common_groups_count = ARRAY_SIZE(mlxsw_sp_trap_group_items_arr); + const struct mlxsw_sp_trap_group_item *spec_group_items_arr; + size_t elem_size = sizeof(struct mlxsw_sp_trap_group_item); + struct mlxsw_sp_trap *trap = mlxsw_sp->trap; + size_t groups_count, spec_groups_count; + int err; + + err = mlxsw_sp->trap_ops->groups_init(mlxsw_sp, &spec_group_items_arr, + &spec_groups_count); + if (err) + return err; + + /* The group items array is created by concatenating the common trap + * group items and the ASIC-specific trap group items. + */ + groups_count = common_groups_count + spec_groups_count; + trap->group_items_arr = kcalloc(groups_count, elem_size, GFP_KERNEL); + if (!trap->group_items_arr) + return -ENOMEM; + + memcpy(trap->group_items_arr, mlxsw_sp_trap_group_items_arr, + elem_size * common_groups_count); + memcpy(trap->group_items_arr + common_groups_count, + spec_group_items_arr, elem_size * spec_groups_count); + + trap->groups_count = groups_count; + + return 0; +} + +static void mlxsw_sp_trap_group_items_arr_fini(struct mlxsw_sp *mlxsw_sp) +{ + kfree(mlxsw_sp->trap->group_items_arr); +} + static int mlxsw_sp_trap_groups_init(struct mlxsw_sp *mlxsw_sp) { struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); @@ -1158,13 +1222,9 @@ static int mlxsw_sp_trap_groups_init(struct mlxsw_sp *mlxsw_sp) struct mlxsw_sp_trap *trap = mlxsw_sp->trap; int err, i; - trap->group_items_arr = kmemdup(mlxsw_sp_trap_group_items_arr, - sizeof(mlxsw_sp_trap_group_items_arr), - GFP_KERNEL); - if (!trap->group_items_arr) - return -ENOMEM; - - trap->groups_count = ARRAY_SIZE(mlxsw_sp_trap_group_items_arr); + err = mlxsw_sp_trap_group_items_arr_init(mlxsw_sp); + if (err) + return err; for (i = 0; i < trap->groups_count; i++) { group_item = &trap->group_items_arr[i]; @@ -1181,7 +1241,7 @@ err_trap_group_register: group_item = &trap->group_items_arr[i]; devlink_trap_groups_unregister(devlink, &group_item->group, 1); } - kfree(trap->group_items_arr); + mlxsw_sp_trap_group_items_arr_fini(mlxsw_sp); return err; } @@ -1197,7 +1257,7 @@ static void mlxsw_sp_trap_groups_fini(struct mlxsw_sp *mlxsw_sp) group_item = &trap->group_items_arr[i]; devlink_trap_groups_unregister(devlink, &group_item->group, 1); } - kfree(trap->group_items_arr); + mlxsw_sp_trap_group_items_arr_fini(mlxsw_sp); } static bool @@ -1206,6 +1266,43 @@ mlxsw_sp_trap_listener_is_valid(const struct mlxsw_listener *listener) return listener->trap_id != 0; } +static int mlxsw_sp_trap_items_arr_init(struct mlxsw_sp *mlxsw_sp) +{ + size_t common_traps_count = ARRAY_SIZE(mlxsw_sp_trap_items_arr); + const struct mlxsw_sp_trap_item *spec_trap_items_arr; + size_t elem_size = sizeof(struct mlxsw_sp_trap_item); + struct mlxsw_sp_trap *trap = mlxsw_sp->trap; + size_t traps_count, spec_traps_count; + int err; + + err = mlxsw_sp->trap_ops->traps_init(mlxsw_sp, &spec_trap_items_arr, + &spec_traps_count); + if (err) + return err; + + /* The trap items array is created by concatenating the common trap + * items and the ASIC-specific trap items. + */ + traps_count = common_traps_count + spec_traps_count; + trap->trap_items_arr = kcalloc(traps_count, elem_size, GFP_KERNEL); + if (!trap->trap_items_arr) + return -ENOMEM; + + memcpy(trap->trap_items_arr, mlxsw_sp_trap_items_arr, + elem_size * common_traps_count); + memcpy(trap->trap_items_arr + common_traps_count, + spec_trap_items_arr, elem_size * spec_traps_count); + + trap->traps_count = traps_count; + + return 0; +} + +static void mlxsw_sp_trap_items_arr_fini(struct mlxsw_sp *mlxsw_sp) +{ + kfree(mlxsw_sp->trap->trap_items_arr); +} + static int mlxsw_sp_traps_init(struct mlxsw_sp *mlxsw_sp) { struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); @@ -1213,13 +1310,9 @@ static int mlxsw_sp_traps_init(struct mlxsw_sp *mlxsw_sp) const struct mlxsw_sp_trap_item *trap_item; int err, i; - trap->trap_items_arr = kmemdup(mlxsw_sp_trap_items_arr, - sizeof(mlxsw_sp_trap_items_arr), - GFP_KERNEL); - if (!trap->trap_items_arr) - return -ENOMEM; - - trap->traps_count = ARRAY_SIZE(mlxsw_sp_trap_items_arr); + err = mlxsw_sp_trap_items_arr_init(mlxsw_sp); + if (err) + return err; for (i = 0; i < trap->traps_count; i++) { trap_item = &trap->trap_items_arr[i]; @@ -1236,7 +1329,7 @@ err_trap_register: trap_item = &trap->trap_items_arr[i]; devlink_traps_unregister(devlink, &trap_item->trap, 1); } - kfree(trap->trap_items_arr); + mlxsw_sp_trap_items_arr_fini(mlxsw_sp); return err; } @@ -1252,7 +1345,7 @@ static void mlxsw_sp_traps_fini(struct mlxsw_sp *mlxsw_sp) trap_item = &trap->trap_items_arr[i]; devlink_traps_unregister(devlink, &trap_item->trap, 1); } - kfree(trap->trap_items_arr); + mlxsw_sp_trap_items_arr_fini(mlxsw_sp); } int mlxsw_sp_devlink_traps_init(struct mlxsw_sp *mlxsw_sp) @@ -1344,7 +1437,8 @@ void mlxsw_sp_trap_fini(struct mlxsw_core *mlxsw_core, int mlxsw_sp_trap_action_set(struct mlxsw_core *mlxsw_core, const struct devlink_trap *trap, - enum devlink_trap_action action) + enum devlink_trap_action action, + struct netlink_ext_ack *extack) { struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core); const struct mlxsw_sp_trap_item *trap_item; @@ -1354,6 +1448,11 @@ int mlxsw_sp_trap_action_set(struct mlxsw_core *mlxsw_core, if (WARN_ON(!trap_item)) return -EINVAL; + if (trap_item->is_source) { + NL_SET_ERR_MSG_MOD(extack, "Changing the action of source traps is not supported"); + return -EOPNOTSUPP; + } + for (i = 0; i < MLXSW_SP_TRAP_LISTENERS_MAX; i++) { const struct mlxsw_listener *listener; bool enabled; @@ -1384,7 +1483,7 @@ int mlxsw_sp_trap_action_set(struct mlxsw_core *mlxsw_core, static int __mlxsw_sp_trap_group_init(struct mlxsw_core *mlxsw_core, const struct devlink_trap_group *group, - u32 policer_id) + u32 policer_id, struct netlink_ext_ack *extack) { struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core); u16 hw_policer_id = MLXSW_REG_HTGT_INVALID_POLICER; @@ -1395,6 +1494,11 @@ __mlxsw_sp_trap_group_init(struct mlxsw_core *mlxsw_core, if (WARN_ON(!group_item)) return -EINVAL; + if (group_item->fixed_policer && policer_id != group->init_policer_id) { + NL_SET_ERR_MSG_MOD(extack, "Changing the policer binding of this group is not supported"); + return -EOPNOTSUPP; + } + if (policer_id) { struct mlxsw_sp_trap_policer_item *policer_item; @@ -1414,16 +1518,18 @@ int mlxsw_sp_trap_group_init(struct mlxsw_core *mlxsw_core, const struct devlink_trap_group *group) { return __mlxsw_sp_trap_group_init(mlxsw_core, group, - group->init_policer_id); + group->init_policer_id, NULL); } int mlxsw_sp_trap_group_set(struct mlxsw_core *mlxsw_core, const struct devlink_trap_group *group, - const struct devlink_trap_policer *policer) + const struct devlink_trap_policer *policer, + struct netlink_ext_ack *extack) { u32 policer_id = policer ? policer->id : 0; - return __mlxsw_sp_trap_group_init(mlxsw_core, group, policer_id); + return __mlxsw_sp_trap_group_init(mlxsw_core, group, policer_id, + extack); } static int @@ -1568,3 +1674,110 @@ mlxsw_sp_trap_policer_counter_get(struct mlxsw_core *mlxsw_core, return 0; } + +int mlxsw_sp_trap_group_policer_hw_id_get(struct mlxsw_sp *mlxsw_sp, u16 id, + bool *p_enabled, u16 *p_hw_id) +{ + struct mlxsw_sp_trap_policer_item *pol_item; + struct mlxsw_sp_trap_group_item *gr_item; + u32 pol_id; + + gr_item = mlxsw_sp_trap_group_item_lookup(mlxsw_sp, id); + if (!gr_item) + return -ENOENT; + + pol_id = gr_item->group.init_policer_id; + if (!pol_id) { + *p_enabled = false; + return 0; + } + + pol_item = mlxsw_sp_trap_policer_item_lookup(mlxsw_sp, pol_id); + if (WARN_ON(!pol_item)) + return -ENOENT; + + *p_enabled = true; + *p_hw_id = pol_item->hw_id; + return 0; +} + +static const struct mlxsw_sp_trap_group_item +mlxsw_sp1_trap_group_items_arr[] = { +}; + +static const struct mlxsw_sp_trap_item +mlxsw_sp1_trap_items_arr[] = { +}; + +static int +mlxsw_sp1_trap_groups_init(struct mlxsw_sp *mlxsw_sp, + const struct mlxsw_sp_trap_group_item **arr, + size_t *p_groups_count) +{ + *arr = mlxsw_sp1_trap_group_items_arr; + *p_groups_count = ARRAY_SIZE(mlxsw_sp1_trap_group_items_arr); + + return 0; +} + +static int mlxsw_sp1_traps_init(struct mlxsw_sp *mlxsw_sp, + const struct mlxsw_sp_trap_item **arr, + size_t *p_traps_count) +{ + *arr = mlxsw_sp1_trap_items_arr; + *p_traps_count = ARRAY_SIZE(mlxsw_sp1_trap_items_arr); + + return 0; +} + +const struct mlxsw_sp_trap_ops mlxsw_sp1_trap_ops = { + .groups_init = mlxsw_sp1_trap_groups_init, + .traps_init = mlxsw_sp1_traps_init, +}; + +static const struct mlxsw_sp_trap_group_item +mlxsw_sp2_trap_group_items_arr[] = { + { + .group = DEVLINK_TRAP_GROUP_GENERIC(BUFFER_DROPS, 20), + .hw_group_id = MLXSW_REG_HTGT_TRAP_GROUP_SP_BUFFER_DISCARDS, + .priority = 0, + .fixed_policer = true, + }, +}; + +static const struct mlxsw_sp_trap_item +mlxsw_sp2_trap_items_arr[] = { + { + .trap = MLXSW_SP_TRAP_BUFFER_DROP(EARLY_DROP), + .listeners_arr = { + MLXSW_SP_RXL_BUFFER_DISCARD(INGRESS_WRED), + }, + .is_source = true, + }, +}; + +static int +mlxsw_sp2_trap_groups_init(struct mlxsw_sp *mlxsw_sp, + const struct mlxsw_sp_trap_group_item **arr, + size_t *p_groups_count) +{ + *arr = mlxsw_sp2_trap_group_items_arr; + *p_groups_count = ARRAY_SIZE(mlxsw_sp2_trap_group_items_arr); + + return 0; +} + +static int mlxsw_sp2_traps_init(struct mlxsw_sp *mlxsw_sp, + const struct mlxsw_sp_trap_item **arr, + size_t *p_traps_count) +{ + *arr = mlxsw_sp2_trap_items_arr; + *p_traps_count = ARRAY_SIZE(mlxsw_sp2_trap_items_arr); + + return 0; +} + +const struct mlxsw_sp_trap_ops mlxsw_sp2_trap_ops = { + .groups_init = mlxsw_sp2_trap_groups_init, + .traps_init = mlxsw_sp2_traps_init, +}; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_trap.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum_trap.h index 13ac412f4d53..b8df684bedef 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_trap.h +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_trap.h @@ -9,13 +9,13 @@ struct mlxsw_sp_trap { struct mlxsw_sp_trap_policer_item *policer_items_arr; - u64 policers_count; /* Number of registered policers */ + size_t policers_count; /* Number of registered policers */ struct mlxsw_sp_trap_group_item *group_items_arr; - u64 groups_count; /* Number of registered groups */ + size_t groups_count; /* Number of registered groups */ struct mlxsw_sp_trap_item *trap_items_arr; - u64 traps_count; /* Number of registered traps */ + size_t traps_count; /* Number of registered traps */ u16 thin_policer_hw_id; @@ -23,4 +23,16 @@ struct mlxsw_sp_trap { unsigned long policers_usage[]; /* Usage bitmap */ }; +struct mlxsw_sp_trap_ops { + int (*groups_init)(struct mlxsw_sp *mlxsw_sp, + const struct mlxsw_sp_trap_group_item **arr, + size_t *p_groups_count); + int (*traps_init)(struct mlxsw_sp *mlxsw_sp, + const struct mlxsw_sp_trap_item **arr, + size_t *p_traps_count); +}; + +extern const struct mlxsw_sp_trap_ops mlxsw_sp1_trap_ops; +extern const struct mlxsw_sp_trap_ops mlxsw_sp2_trap_ops; + #endif diff --git a/drivers/net/ethernet/mellanox/mlxsw/switchib.c b/drivers/net/ethernet/mellanox/mlxsw/switchib.c index 4ff1e623aa76..1e561132eb1e 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/switchib.c +++ b/drivers/net/ethernet/mellanox/mlxsw/switchib.c @@ -281,7 +281,7 @@ static int mlxsw_sib_port_create(struct mlxsw_sib *mlxsw_sib, u8 local_port, int err; err = mlxsw_core_port_init(mlxsw_sib->core, local_port, - module + 1, false, 0, + module + 1, false, 0, false, 0, mlxsw_sib->hw_id, sizeof(mlxsw_sib->hw_id)); if (err) { dev_err(mlxsw_sib->bus_info->dev, "Port %d: Failed to init core port\n", diff --git a/drivers/net/ethernet/mellanox/mlxsw/switchx2.c b/drivers/net/ethernet/mellanox/mlxsw/switchx2.c index b438f5576e18..6f9a725662fb 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/switchx2.c +++ b/drivers/net/ethernet/mellanox/mlxsw/switchx2.c @@ -1107,7 +1107,7 @@ static int mlxsw_sx_port_eth_create(struct mlxsw_sx *mlxsw_sx, u8 local_port, int err; err = mlxsw_core_port_init(mlxsw_sx->core, local_port, - module + 1, false, 0, + module + 1, false, 0, false, 0, mlxsw_sx->hw_id, sizeof(mlxsw_sx->hw_id)); if (err) { dev_err(mlxsw_sx->bus_info->dev, "Port %d: Failed to init core port\n", diff --git a/drivers/net/ethernet/mellanox/mlxsw/trap.h b/drivers/net/ethernet/mellanox/mlxsw/trap.h index 28e60697d14e..33909887d0ac 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/trap.h +++ b/drivers/net/ethernet/mellanox/mlxsw/trap.h @@ -107,8 +107,16 @@ enum { MLXSW_TRAP_ID_ACL2 = 0x1C2, MLXSW_TRAP_ID_DISCARD_INGRESS_ACL = 0x1C3, MLXSW_TRAP_ID_DISCARD_EGRESS_ACL = 0x1C4, + MLXSW_TRAP_ID_MIRROR_SESSION0 = 0x220, + MLXSW_TRAP_ID_MIRROR_SESSION1 = 0x221, + MLXSW_TRAP_ID_MIRROR_SESSION2 = 0x222, + MLXSW_TRAP_ID_MIRROR_SESSION3 = 0x223, + MLXSW_TRAP_ID_MIRROR_SESSION4 = 0x224, + MLXSW_TRAP_ID_MIRROR_SESSION5 = 0x225, + MLXSW_TRAP_ID_MIRROR_SESSION6 = 0x226, + MLXSW_TRAP_ID_MIRROR_SESSION7 = 0x227, - MLXSW_TRAP_ID_MAX = 0x1FF + MLXSW_TRAP_ID_MAX = 0x3FF, }; enum mlxsw_event_trap_id { |