diff options
Diffstat (limited to 'drivers/net/ethernet/mellanox/mlx5/core/en_accel')
13 files changed, 252 insertions, 247 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h index 3022463f2284..fac145dcf2ce 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h @@ -42,6 +42,8 @@ #include "en/txrx.h" #if IS_ENABLED(CONFIG_GENEVE) +#include <net/geneve.h> + static inline bool mlx5_geneve_tx_allowed(struct mlx5_core_dev *mdev) { return mlx5_tx_swp_supported(mdev); @@ -100,33 +102,49 @@ mlx5e_udp_gso_handle_tx_skb(struct sk_buff *skb) udp_hdr(skb)->len = htons(payload_len); } -static inline struct sk_buff * -mlx5e_accel_handle_tx(struct sk_buff *skb, - struct mlx5e_txqsq *sq, - struct net_device *dev, - struct mlx5e_tx_wqe **wqe, - u16 *pi) +struct mlx5e_accel_tx_state { +#ifdef CONFIG_MLX5_EN_TLS + struct mlx5e_accel_tx_tls_state tls; +#endif +}; + +static inline bool mlx5e_accel_tx_begin(struct net_device *dev, + struct mlx5e_txqsq *sq, + struct sk_buff *skb, + struct mlx5e_accel_tx_state *state) { + if (skb_is_gso(skb) && skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) + mlx5e_udp_gso_handle_tx_skb(skb); + #ifdef CONFIG_MLX5_EN_TLS if (test_bit(MLX5E_SQ_STATE_TLS, &sq->state)) { - skb = mlx5e_tls_handle_tx_skb(dev, sq, skb, wqe, pi); - if (unlikely(!skb)) - return NULL; + /* May send SKBs and WQEs. */ + if (unlikely(!mlx5e_tls_handle_tx_skb(dev, sq, skb, &state->tls))) + return false; } #endif + return true; +} + +static inline bool mlx5e_accel_tx_finish(struct mlx5e_priv *priv, + struct mlx5e_txqsq *sq, + struct sk_buff *skb, + struct mlx5e_tx_wqe *wqe, + struct mlx5e_accel_tx_state *state) +{ +#ifdef CONFIG_MLX5_EN_TLS + mlx5e_tls_handle_tx_wqe(sq, &wqe->ctrl, &state->tls); +#endif + #ifdef CONFIG_MLX5_EN_IPSEC if (test_bit(MLX5E_SQ_STATE_IPSEC, &sq->state)) { - skb = mlx5e_ipsec_handle_tx_skb(dev, *wqe, skb); - if (unlikely(!skb)) - return NULL; + if (unlikely(!mlx5e_ipsec_handle_tx_skb(priv, &wqe->eth, skb))) + return false; } #endif - if (skb_is_gso(skb) && skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) - mlx5e_udp_gso_handle_tx_skb(skb); - - return skb; + return true; } #endif /* __MLX5E_EN_ACCEL_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index 29626c6c9c25..92eb3bad4acd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -75,18 +75,23 @@ struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *ipsec, return ret; } -static int mlx5e_ipsec_sadb_rx_add(struct mlx5e_ipsec_sa_entry *sa_entry) +static int mlx5e_ipsec_sadb_rx_add(struct mlx5e_ipsec_sa_entry *sa_entry, + unsigned int handle) { struct mlx5e_ipsec *ipsec = sa_entry->ipsec; + struct mlx5e_ipsec_sa_entry *_sa_entry; unsigned long flags; - int ret; - ret = ida_simple_get(&ipsec->halloc, 1, 0, GFP_KERNEL); - if (ret < 0) - return ret; + rcu_read_lock(); + hash_for_each_possible_rcu(ipsec->sadb_rx, _sa_entry, hlist, handle) + if (_sa_entry->handle == handle) { + rcu_read_unlock(); + return -EEXIST; + } + rcu_read_unlock(); spin_lock_irqsave(&ipsec->sadb_rx_lock, flags); - sa_entry->handle = ret; + sa_entry->handle = handle; hash_add_rcu(ipsec->sadb_rx, &sa_entry->hlist, sa_entry->handle); spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags); @@ -103,15 +108,6 @@ static void mlx5e_ipsec_sadb_rx_del(struct mlx5e_ipsec_sa_entry *sa_entry) spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags); } -static void mlx5e_ipsec_sadb_rx_free(struct mlx5e_ipsec_sa_entry *sa_entry) -{ - struct mlx5e_ipsec *ipsec = sa_entry->ipsec; - - /* xfrm already doing sync rcu between del and free callbacks */ - - ida_simple_remove(&ipsec->halloc, sa_entry->handle); -} - static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry) { struct xfrm_replay_state_esn *replay_esn; @@ -199,6 +195,14 @@ mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, attrs->flags |= (x->props.mode == XFRM_MODE_TRANSPORT) ? MLX5_ACCEL_ESP_FLAGS_TRANSPORT : MLX5_ACCEL_ESP_FLAGS_TUNNEL; + + /* spi */ + attrs->spi = x->id.spi; + + /* source , destination ips */ + memcpy(&attrs->saddr, x->props.saddr.a6, sizeof(attrs->saddr)); + memcpy(&attrs->daddr, x->id.daddr.a6, sizeof(attrs->daddr)); + attrs->is_ipv6 = (x->props.family != AF_INET); } static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) @@ -284,8 +288,7 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) struct net_device *netdev = x->xso.dev; struct mlx5_accel_esp_xfrm_attrs attrs; struct mlx5e_priv *priv; - __be32 saddr[4] = {0}, daddr[4] = {0}, spi; - bool is_ipv6 = false; + unsigned int sa_handle; int err; priv = netdev_priv(netdev); @@ -303,20 +306,6 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) sa_entry->x = x; sa_entry->ipsec = priv->ipsec; - /* Add the SA to handle processed incoming packets before the add SA - * completion was received - */ - if (x->xso.flags & XFRM_OFFLOAD_INBOUND) { - err = mlx5e_ipsec_sadb_rx_add(sa_entry); - if (err) { - netdev_info(netdev, "Failed adding to SADB_RX: %d\n", err); - goto err_entry; - } - } else { - sa_entry->set_iv_op = (x->props.flags & XFRM_STATE_ESN) ? - mlx5e_ipsec_set_iv_esn : mlx5e_ipsec_set_iv; - } - /* check esn */ mlx5e_ipsec_update_esn_state(sa_entry); @@ -327,41 +316,38 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) MLX5_ACCEL_XFRM_FLAG_REQUIRE_METADATA); if (IS_ERR(sa_entry->xfrm)) { err = PTR_ERR(sa_entry->xfrm); - goto err_sadb_rx; + goto err_sa_entry; } /* create hw context */ - if (x->props.family == AF_INET) { - saddr[3] = x->props.saddr.a4; - daddr[3] = x->id.daddr.a4; - } else { - memcpy(saddr, x->props.saddr.a6, sizeof(saddr)); - memcpy(daddr, x->id.daddr.a6, sizeof(daddr)); - is_ipv6 = true; - } - spi = x->id.spi; sa_entry->hw_context = mlx5_accel_esp_create_hw_context(priv->mdev, sa_entry->xfrm, - saddr, daddr, spi, - is_ipv6); + &sa_handle); if (IS_ERR(sa_entry->hw_context)) { err = PTR_ERR(sa_entry->hw_context); goto err_xfrm; } + if (x->xso.flags & XFRM_OFFLOAD_INBOUND) { + err = mlx5e_ipsec_sadb_rx_add(sa_entry, sa_handle); + if (err) + goto err_hw_ctx; + } else { + sa_entry->set_iv_op = (x->props.flags & XFRM_STATE_ESN) ? + mlx5e_ipsec_set_iv_esn : mlx5e_ipsec_set_iv; + } + x->xso.offload_handle = (unsigned long)sa_entry; goto out; +err_hw_ctx: + mlx5_accel_esp_free_hw_context(sa_entry->hw_context); err_xfrm: mlx5_accel_esp_destroy_xfrm(sa_entry->xfrm); -err_sadb_rx: - if (x->xso.flags & XFRM_OFFLOAD_INBOUND) { - mlx5e_ipsec_sadb_rx_del(sa_entry); - mlx5e_ipsec_sadb_rx_free(sa_entry); - } -err_entry: +err_sa_entry: kfree(sa_entry); + out: return err; } @@ -390,9 +376,6 @@ static void mlx5e_xfrm_free_state(struct xfrm_state *x) mlx5_accel_esp_destroy_xfrm(sa_entry->xfrm); } - if (x->xso.flags & XFRM_OFFLOAD_INBOUND) - mlx5e_ipsec_sadb_rx_free(sa_entry); - kfree(sa_entry); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index 93bf10e6508c..c85151a1e008 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -109,11 +109,6 @@ int mlx5e_ipsec_init(struct mlx5e_priv *priv); void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv); void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv); -int mlx5e_ipsec_get_count(struct mlx5e_priv *priv); -int mlx5e_ipsec_get_strings(struct mlx5e_priv *priv, uint8_t *data); -void mlx5e_ipsec_update_stats(struct mlx5e_priv *priv); -int mlx5e_ipsec_get_stats(struct mlx5e_priv *priv, u64 *data); - struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *dev, unsigned int handle); @@ -136,26 +131,6 @@ static inline void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv) { } -static inline int mlx5e_ipsec_get_count(struct mlx5e_priv *priv) -{ - return 0; -} - -static inline int mlx5e_ipsec_get_strings(struct mlx5e_priv *priv, - uint8_t *data) -{ - return 0; -} - -static inline void mlx5e_ipsec_update_stats(struct mlx5e_priv *priv) -{ -} - -static inline int mlx5e_ipsec_get_stats(struct mlx5e_priv *priv, u64 *data) -{ - return 0; -} - #endif #endif /* __MLX5E_IPSEC_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c index 0dd17514caae..824b87ac8f9e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c @@ -233,11 +233,10 @@ static void mlx5e_ipsec_set_metadata(struct sk_buff *skb, ntohs(mdata->content.tx.seq)); } -struct sk_buff *mlx5e_ipsec_handle_tx_skb(struct net_device *netdev, - struct mlx5e_tx_wqe *wqe, - struct sk_buff *skb) +bool mlx5e_ipsec_handle_tx_skb(struct mlx5e_priv *priv, + struct mlx5_wqe_eth_seg *eseg, + struct sk_buff *skb) { - struct mlx5e_priv *priv = netdev_priv(netdev); struct xfrm_offload *xo = xfrm_offload(skb); struct mlx5e_ipsec_metadata *mdata; struct mlx5e_ipsec_sa_entry *sa_entry; @@ -245,7 +244,7 @@ struct sk_buff *mlx5e_ipsec_handle_tx_skb(struct net_device *netdev, struct sec_path *sp; if (!xo) - return skb; + return true; sp = skb_sec_path(skb); if (unlikely(sp->len != 1)) { @@ -276,16 +275,16 @@ struct sk_buff *mlx5e_ipsec_handle_tx_skb(struct net_device *netdev, atomic64_inc(&priv->ipsec->sw_stats.ipsec_tx_drop_metadata); goto drop; } - mlx5e_ipsec_set_swp(skb, &wqe->eth, x->props.mode, xo); + mlx5e_ipsec_set_swp(skb, eseg, x->props.mode, xo); sa_entry = (struct mlx5e_ipsec_sa_entry *)x->xso.offload_handle; sa_entry->set_iv_op(skb, x, xo); mlx5e_ipsec_set_metadata(skb, mdata, xo); - return skb; + return true; drop: kfree_skb(skb); - return NULL; + return false; } static inline struct xfrm_state * diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h index db84500b024f..ba02643586a5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h @@ -52,9 +52,9 @@ void mlx5e_ipsec_set_iv_esn(struct sk_buff *skb, struct xfrm_state *x, struct xfrm_offload *xo); void mlx5e_ipsec_set_iv(struct sk_buff *skb, struct xfrm_state *x, struct xfrm_offload *xo); -struct sk_buff *mlx5e_ipsec_handle_tx_skb(struct net_device *netdev, - struct mlx5e_tx_wqe *wqe, - struct sk_buff *skb); +bool mlx5e_ipsec_handle_tx_skb(struct mlx5e_priv *priv, + struct mlx5_wqe_eth_seg *eseg, + struct sk_buff *skb); #endif /* CONFIG_MLX5_EN_IPSEC */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_stats.c index 6fea59223dc4..6c5c54bcd9be 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_stats.c @@ -38,6 +38,7 @@ #include "accel/ipsec.h" #include "fpga/sdk.h" #include "en_accel/ipsec.h" +#include "fpga/ipsec.h" static const struct counter_desc mlx5e_ipsec_hw_stats_desc[] = { { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_stats, ipsec_dec_in_packets) }, @@ -73,61 +74,74 @@ static const struct counter_desc mlx5e_ipsec_sw_stats_desc[] = { #define NUM_IPSEC_HW_COUNTERS ARRAY_SIZE(mlx5e_ipsec_hw_stats_desc) #define NUM_IPSEC_SW_COUNTERS ARRAY_SIZE(mlx5e_ipsec_sw_stats_desc) -#define NUM_IPSEC_COUNTERS (NUM_IPSEC_HW_COUNTERS + NUM_IPSEC_SW_COUNTERS) - -int mlx5e_ipsec_get_count(struct mlx5e_priv *priv) +static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(ipsec_sw) { - if (!priv->ipsec) - return 0; - - return NUM_IPSEC_COUNTERS; + return NUM_IPSEC_SW_COUNTERS; } -int mlx5e_ipsec_get_strings(struct mlx5e_priv *priv, uint8_t *data) -{ - unsigned int i, idx = 0; +static inline MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(ipsec_sw) {} - if (!priv->ipsec) - return 0; +static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(ipsec_sw) +{ + unsigned int i; - for (i = 0; i < NUM_IPSEC_HW_COUNTERS; i++) - strcpy(data + (idx++) * ETH_GSTRING_LEN, - mlx5e_ipsec_hw_stats_desc[i].format); + if (priv->ipsec) + for (i = 0; i < NUM_IPSEC_SW_COUNTERS; i++) + strcpy(data + (idx++) * ETH_GSTRING_LEN, + mlx5e_ipsec_sw_stats_desc[i].format); + return idx; +} - for (i = 0; i < NUM_IPSEC_SW_COUNTERS; i++) - strcpy(data + (idx++) * ETH_GSTRING_LEN, - mlx5e_ipsec_sw_stats_desc[i].format); +static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(ipsec_sw) +{ + int i; - return NUM_IPSEC_COUNTERS; + if (priv->ipsec) + for (i = 0; i < NUM_IPSEC_SW_COUNTERS; i++) + data[idx++] = MLX5E_READ_CTR_ATOMIC64(&priv->ipsec->sw_stats, + mlx5e_ipsec_sw_stats_desc, i); + return idx; } -void mlx5e_ipsec_update_stats(struct mlx5e_priv *priv) +static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(ipsec_hw) { - int ret; + return (mlx5_fpga_ipsec_device_caps(priv->mdev)) ? NUM_IPSEC_HW_COUNTERS : 0; +} - if (!priv->ipsec) - return; +static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(ipsec_hw) +{ + int ret = 0; - ret = mlx5_accel_ipsec_counters_read(priv->mdev, (u64 *)&priv->ipsec->stats, - NUM_IPSEC_HW_COUNTERS); + if (priv->ipsec) + ret = mlx5_accel_ipsec_counters_read(priv->mdev, (u64 *)&priv->ipsec->stats, + NUM_IPSEC_HW_COUNTERS); if (ret) memset(&priv->ipsec->stats, 0, sizeof(priv->ipsec->stats)); } -int mlx5e_ipsec_get_stats(struct mlx5e_priv *priv, u64 *data) +static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(ipsec_hw) { - int i, idx = 0; - - if (!priv->ipsec) - return 0; + unsigned int i; - for (i = 0; i < NUM_IPSEC_HW_COUNTERS; i++) - data[idx++] = MLX5E_READ_CTR64_CPU(&priv->ipsec->stats, - mlx5e_ipsec_hw_stats_desc, i); + if (priv->ipsec && mlx5_fpga_ipsec_device_caps(priv->mdev)) + for (i = 0; i < NUM_IPSEC_HW_COUNTERS; i++) + strcpy(data + (idx++) * ETH_GSTRING_LEN, + mlx5e_ipsec_hw_stats_desc[i].format); - for (i = 0; i < NUM_IPSEC_SW_COUNTERS; i++) - data[idx++] = MLX5E_READ_CTR_ATOMIC64(&priv->ipsec->sw_stats, - mlx5e_ipsec_sw_stats_desc, i); + return idx; +} - return NUM_IPSEC_COUNTERS; +static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(ipsec_hw) +{ + int i; + + if (priv->ipsec && mlx5_fpga_ipsec_device_caps(priv->mdev)) + for (i = 0; i < NUM_IPSEC_HW_COUNTERS; i++) + data[idx++] = MLX5E_READ_CTR64_CPU(&priv->ipsec->stats, + mlx5e_ipsec_hw_stats_desc, + i); + return idx; } + +MLX5E_DEFINE_STATS_GRP(ipsec_sw, 0); +MLX5E_DEFINE_STATS_GRP(ipsec_hw, 0); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c index 46725cd743a3..452fcf59c36b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c @@ -4,6 +4,19 @@ #include "en.h" #include "en_accel/ktls.h" +u16 mlx5e_ktls_get_stop_room(struct mlx5e_txqsq *sq) +{ + u16 num_dumps, stop_room = 0; + + num_dumps = mlx5e_ktls_dumps_num_wqes(sq, MAX_SKB_FRAGS, TLS_MAX_PAYLOAD_SIZE); + + stop_room += mlx5e_stop_room_for_wqe(MLX5E_KTLS_STATIC_WQEBBS); + stop_room += mlx5e_stop_room_for_wqe(MLX5E_KTLS_PROGRESS_WQEBBS); + stop_room += num_dumps * mlx5e_stop_room_for_wqe(MLX5E_KTLS_DUMP_WQEBBS); + + return stop_room; +} + static int mlx5e_ktls_create_tis(struct mlx5_core_dev *mdev, u32 *tisn) { u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; @@ -69,8 +82,8 @@ static void mlx5e_ktls_del(struct net_device *netdev, struct mlx5e_ktls_offload_context_tx *tx_priv = mlx5e_get_ktls_tx_priv_ctx(tls_ctx); - mlx5_ktls_destroy_key(priv->mdev, tx_priv->key_id); mlx5e_destroy_tis(priv->mdev, tx_priv->tisn); + mlx5_ktls_destroy_key(priv->mdev, tx_priv->key_id); kvfree(tx_priv); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h index 63116be6b1d6..c6180892cfcb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h @@ -9,6 +9,7 @@ #ifdef CONFIG_MLX5_EN_TLS #include <net/tls.h> #include "accel/tls.h" +#include "en_accel/tls_rxtx.h" #define MLX5E_KTLS_STATIC_UMR_WQE_SZ \ (offsetof(struct mlx5e_umr_wqe, tls_static_params_ctx) + \ @@ -27,6 +28,14 @@ struct mlx5e_dump_wqe { struct mlx5_wqe_data_seg data; }; +#define MLX5E_TLS_FETCH_UMR_WQE(sq, pi) \ + ((struct mlx5e_umr_wqe *)mlx5e_fetch_wqe(&(sq)->wq, pi, MLX5E_KTLS_STATIC_UMR_WQE_SZ)) +#define MLX5E_TLS_FETCH_PROGRESS_WQE(sq, pi) \ + ((struct mlx5e_tx_wqe *)mlx5e_fetch_wqe(&(sq)->wq, pi, MLX5E_KTLS_PROGRESS_WQE_SZ)) +#define MLX5E_TLS_FETCH_DUMP_WQE(sq, pi) \ + ((struct mlx5e_dump_wqe *)mlx5e_fetch_wqe(&(sq)->wq, pi, \ + sizeof(struct mlx5e_dump_wqe))) + #define MLX5E_KTLS_DUMP_WQEBBS \ (DIV_ROUND_UP(sizeof(struct mlx5e_dump_wqe), MLX5_SEND_WQE_BB)) @@ -87,22 +96,22 @@ mlx5e_get_ktls_tx_priv_ctx(struct tls_context *tls_ctx) void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv); void mlx5e_ktls_tx_offload_set_pending(struct mlx5e_ktls_offload_context_tx *priv_tx); -struct sk_buff *mlx5e_ktls_handle_tx_skb(struct net_device *netdev, - struct mlx5e_txqsq *sq, - struct sk_buff *skb, - struct mlx5e_tx_wqe **wqe, u16 *pi); +bool mlx5e_ktls_handle_tx_skb(struct tls_context *tls_ctx, struct mlx5e_txqsq *sq, + struct sk_buff *skb, int datalen, + struct mlx5e_accel_tx_tls_state *state); void mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq, struct mlx5e_tx_wqe_info *wi, u32 *dma_fifo_cc); +u16 mlx5e_ktls_get_stop_room(struct mlx5e_txqsq *sq); + static inline u8 -mlx5e_ktls_dumps_num_wqebbs(struct mlx5e_txqsq *sq, unsigned int nfrags, - unsigned int sync_len) +mlx5e_ktls_dumps_num_wqes(struct mlx5e_txqsq *sq, unsigned int nfrags, + unsigned int sync_len) { /* Given the MTU and sync_len, calculates an upper bound for the - * number of WQEBBs needed for the TX resync DUMP WQEs of a record. + * number of DUMP WQEs needed for the TX resync of a record. */ - return MLX5E_KTLS_DUMP_WQEBBS * - (nfrags + DIV_ROUND_UP(sync_len, sq->hw_mtu)); + return nfrags + DIV_ROUND_UP(sync_len, sq->hw_mtu); } #else @@ -114,7 +123,6 @@ static inline void mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq, struct mlx5e_tx_wqe_info *wi, u32 *dma_fifo_cc) {} - #endif #endif /* __MLX5E_TLS_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c index 52a56622034a..3cd78d9503c1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c @@ -108,10 +108,11 @@ static void tx_fill_wi(struct mlx5e_txqsq *sq, { struct mlx5e_tx_wqe_info *wi = &sq->db.wqe_info[pi]; - memset(wi, 0, sizeof(*wi)); - wi->num_wqebbs = num_wqebbs; - wi->num_bytes = num_bytes; - wi->resync_dump_frag_page = page; + *wi = (struct mlx5e_tx_wqe_info) { + .num_wqebbs = num_wqebbs, + .num_bytes = num_bytes, + .resync_dump_frag_page = page, + }; } void mlx5e_ktls_tx_offload_set_pending(struct mlx5e_ktls_offload_context_tx *priv_tx) @@ -134,13 +135,14 @@ post_static_params(struct mlx5e_txqsq *sq, struct mlx5e_ktls_offload_context_tx *priv_tx, bool fence) { + u16 pi, num_wqebbs = MLX5E_KTLS_STATIC_WQEBBS; struct mlx5e_umr_wqe *umr_wqe; - u16 pi; - umr_wqe = mlx5e_sq_fetch_wqe(sq, MLX5E_KTLS_STATIC_UMR_WQE_SZ, &pi); + pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs); + umr_wqe = MLX5E_TLS_FETCH_UMR_WQE(sq, pi); build_static_params(umr_wqe, sq->pc, sq->sqn, priv_tx, fence); - tx_fill_wi(sq, pi, MLX5E_KTLS_STATIC_WQEBBS, 0, NULL); - sq->pc += MLX5E_KTLS_STATIC_WQEBBS; + tx_fill_wi(sq, pi, num_wqebbs, 0, NULL); + sq->pc += num_wqebbs; } static void @@ -148,13 +150,14 @@ post_progress_params(struct mlx5e_txqsq *sq, struct mlx5e_ktls_offload_context_tx *priv_tx, bool fence) { + u16 pi, num_wqebbs = MLX5E_KTLS_PROGRESS_WQEBBS; struct mlx5e_tx_wqe *wqe; - u16 pi; - wqe = mlx5e_sq_fetch_wqe(sq, MLX5E_KTLS_PROGRESS_WQE_SZ, &pi); + pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs); + wqe = MLX5E_TLS_FETCH_PROGRESS_WQE(sq, pi); build_progress_params(wqe, sq->pc, sq->sqn, priv_tx, fence); - tx_fill_wi(sq, pi, MLX5E_KTLS_PROGRESS_WQEBBS, 0, NULL); - sq->pc += MLX5E_KTLS_PROGRESS_WQEBBS; + tx_fill_wi(sq, pi, num_wqebbs, 0, NULL); + sq->pc += num_wqebbs; } static void @@ -163,14 +166,6 @@ mlx5e_ktls_tx_post_param_wqes(struct mlx5e_txqsq *sq, bool skip_static_post, bool fence_first_post) { bool progress_fence = skip_static_post || !fence_first_post; - struct mlx5_wq_cyc *wq = &sq->wq; - u16 contig_wqebbs_room, pi; - - pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); - contig_wqebbs_room = mlx5_wq_cyc_get_contig_wqebbs(wq, pi); - if (unlikely(contig_wqebbs_room < - MLX5E_KTLS_STATIC_WQEBBS + MLX5E_KTLS_PROGRESS_WQEBBS)) - mlx5e_fill_sq_frag_edge(sq, wq, pi, contig_wqebbs_room); if (!skip_static_post) post_static_params(sq, priv_tx, fence_first_post); @@ -278,7 +273,9 @@ tx_post_resync_dump(struct mlx5e_txqsq *sq, skb_frag_t *frag, u32 tisn, bool fir int fsz; u16 pi; - wqe = mlx5e_sq_fetch_wqe(sq, sizeof(*wqe), &pi); + BUILD_BUG_ON(MLX5E_KTLS_DUMP_WQEBBS != 1); + pi = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->pc); + wqe = MLX5E_TLS_FETCH_DUMP_WQE(sq, pi); ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS; @@ -343,11 +340,8 @@ mlx5e_ktls_tx_handle_ooo(struct mlx5e_ktls_offload_context_tx *priv_tx, u32 seq) { struct mlx5e_sq_stats *stats = sq->stats; - struct mlx5_wq_cyc *wq = &sq->wq; enum mlx5e_ktls_sync_retval ret; struct tx_sync_info info = {}; - u16 contig_wqebbs_room, pi; - u8 num_wqebbs; int i = 0; ret = tx_sync_info_get(priv_tx, seq, datalen, &info); @@ -376,13 +370,6 @@ mlx5e_ktls_tx_handle_ooo(struct mlx5e_ktls_offload_context_tx *priv_tx, return MLX5E_KTLS_SYNC_DONE; } - num_wqebbs = mlx5e_ktls_dumps_num_wqebbs(sq, info.nr_frags, info.sync_len); - pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); - contig_wqebbs_room = mlx5_wq_cyc_get_contig_wqebbs(wq, pi); - - if (unlikely(contig_wqebbs_room < num_wqebbs)) - mlx5e_fill_sq_frag_edge(sq, wq, pi, contig_wqebbs_room); - for (; i < info.nr_frags; i++) { unsigned int orig_fsz, frag_offset = 0, n = 0; skb_frag_t *f = &info.frags[i]; @@ -422,34 +409,18 @@ err_out: return MLX5E_KTLS_SYNC_FAIL; } -struct sk_buff *mlx5e_ktls_handle_tx_skb(struct net_device *netdev, - struct mlx5e_txqsq *sq, - struct sk_buff *skb, - struct mlx5e_tx_wqe **wqe, u16 *pi) +bool mlx5e_ktls_handle_tx_skb(struct tls_context *tls_ctx, struct mlx5e_txqsq *sq, + struct sk_buff *skb, int datalen, + struct mlx5e_accel_tx_tls_state *state) { struct mlx5e_ktls_offload_context_tx *priv_tx; struct mlx5e_sq_stats *stats = sq->stats; - struct mlx5_wqe_ctrl_seg *cseg; - struct tls_context *tls_ctx; - int datalen; u32 seq; - if (!skb->sk || !tls_is_sk_tx_device_offloaded(skb->sk)) - goto out; - - datalen = skb->len - (skb_transport_offset(skb) + tcp_hdrlen(skb)); - if (!datalen) - goto out; - - tls_ctx = tls_get_ctx(skb->sk); - if (WARN_ON_ONCE(tls_ctx->netdev != netdev)) - goto err_out; - priv_tx = mlx5e_get_ktls_tx_priv_ctx(tls_ctx); if (unlikely(mlx5e_ktls_tx_offload_test_and_clear_pending(priv_tx))) { mlx5e_ktls_tx_post_param_wqes(sq, priv_tx, false, false); - *wqe = mlx5e_sq_fetch_wqe(sq, sizeof(**wqe), pi); stats->tls_ctx++; } @@ -460,30 +431,28 @@ struct sk_buff *mlx5e_ktls_handle_tx_skb(struct net_device *netdev, switch (ret) { case MLX5E_KTLS_SYNC_DONE: - *wqe = mlx5e_sq_fetch_wqe(sq, sizeof(**wqe), pi); break; case MLX5E_KTLS_SYNC_SKIP_NO_DATA: if (likely(!skb->decrypted)) goto out; WARN_ON_ONCE(1); /* fall-through */ - default: /* MLX5E_KTLS_SYNC_FAIL */ + case MLX5E_KTLS_SYNC_FAIL: goto err_out; } } priv_tx->expected_seq = seq + datalen; - cseg = &(*wqe)->ctrl; - cseg->tisn = cpu_to_be32(priv_tx->tisn << 8); + state->tls_tisn = priv_tx->tisn; stats->tls_encrypted_packets += skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1; stats->tls_encrypted_bytes += datalen; out: - return skb; + return true; err_out: dev_kfree_skb_any(skb); - return NULL; + return false; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c index fba561ffe1d4..1fbb5a90cb38 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c @@ -167,7 +167,7 @@ static int mlx5e_tls_resync(struct net_device *netdev, struct sock *sk, struct tls_context *tls_ctx = tls_get_ctx(sk); struct mlx5e_priv *priv = netdev_priv(netdev); struct mlx5e_tls_offload_context_rx *rx_ctx; - u64 rcd_sn = *(u64 *)rcd_sn_data; + __be64 rcd_sn = *(__be64 *)rcd_sn_data; if (WARN_ON_ONCE(direction != TLS_OFFLOAD_CTX_DIR_RX)) return -EINVAL; @@ -240,3 +240,17 @@ void mlx5e_tls_cleanup(struct mlx5e_priv *priv) kfree(tls); priv->tls = NULL; } + +u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq) +{ + struct mlx5_core_dev *mdev = sq->channel->mdev; + + if (!mlx5_accel_is_tls_device(mdev)) + return 0; + + if (MLX5_CAP_GEN(mdev, tls_tx)) + return mlx5e_ktls_get_stop_room(sq); + + /* Resync SKB. */ + return mlx5e_stop_room_for_wqe(MLX5_SEND_WQE_MAX_WQEBBS); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h index 9015f3f7792d..9219bdb2786e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h @@ -94,6 +94,8 @@ int mlx5e_tls_get_count(struct mlx5e_priv *priv); int mlx5e_tls_get_strings(struct mlx5e_priv *priv, uint8_t *data); int mlx5e_tls_get_stats(struct mlx5e_priv *priv, u64 *data); +u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq); + #else static inline void mlx5e_tls_build_netdev(struct mlx5e_priv *priv) @@ -108,6 +110,11 @@ static inline int mlx5e_tls_get_count(struct mlx5e_priv *priv) { return 0; } static inline int mlx5e_tls_get_strings(struct mlx5e_priv *priv, uint8_t *data) { return 0; } static inline int mlx5e_tls_get_stats(struct mlx5e_priv *priv, u64 *data) { return 0; } +static inline u16 mlx5e_tls_get_stop_room(struct mlx5e_txqsq *sq) +{ + return 0; +} + #endif #endif /* __MLX5E_TLS_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c index ef1ed15a53b4..05454a843b28 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c @@ -184,18 +184,17 @@ static void mlx5e_tls_complete_sync_skb(struct sk_buff *skb, nskb->queue_mapping = skb->queue_mapping; } -static struct sk_buff * -mlx5e_tls_handle_ooo(struct mlx5e_tls_offload_context_tx *context, - struct mlx5e_txqsq *sq, struct sk_buff *skb, - struct mlx5e_tx_wqe **wqe, - u16 *pi, - struct mlx5e_tls *tls) +static bool mlx5e_tls_handle_ooo(struct mlx5e_tls_offload_context_tx *context, + struct mlx5e_txqsq *sq, struct sk_buff *skb, + struct mlx5e_tls *tls) { u32 tcp_seq = ntohl(tcp_hdr(skb)->seq); + struct mlx5e_tx_wqe *wqe; struct sync_info info; struct sk_buff *nskb; int linear_len = 0; int headln; + u16 pi; int i; sq->stats->tls_ooo++; @@ -217,7 +216,7 @@ mlx5e_tls_handle_ooo(struct mlx5e_tls_offload_context_tx *context, if (likely(payload <= -info.sync_len)) /* SKB payload doesn't require offload */ - return skb; + return true; atomic64_inc(&tls->sw_stats.tx_tls_drop_bypass_required); goto err_out; @@ -247,20 +246,19 @@ mlx5e_tls_handle_ooo(struct mlx5e_tls_offload_context_tx *context, sq->stats->tls_resync_bytes += nskb->len; mlx5e_tls_complete_sync_skb(skb, nskb, tcp_seq, headln, cpu_to_be64(info.rcd_sn)); - mlx5e_sq_xmit(sq, nskb, *wqe, *pi, true); - *wqe = mlx5e_sq_fetch_wqe(sq, sizeof(**wqe), pi); - return skb; + pi = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->pc); + wqe = MLX5E_TX_FETCH_WQE(sq, pi); + mlx5e_sq_xmit(sq, nskb, wqe, pi, true); + + return true; err_out: dev_kfree_skb_any(skb); - return NULL; + return false; } -struct sk_buff *mlx5e_tls_handle_tx_skb(struct net_device *netdev, - struct mlx5e_txqsq *sq, - struct sk_buff *skb, - struct mlx5e_tx_wqe **wqe, - u16 *pi) +bool mlx5e_tls_handle_tx_skb(struct net_device *netdev, struct mlx5e_txqsq *sq, + struct sk_buff *skb, struct mlx5e_accel_tx_tls_state *state) { struct mlx5e_priv *priv = netdev_priv(netdev); struct mlx5e_tls_offload_context_tx *context; @@ -269,41 +267,45 @@ struct sk_buff *mlx5e_tls_handle_tx_skb(struct net_device *netdev, int datalen; u32 skb_seq; - if (MLX5_CAP_GEN(sq->channel->mdev, tls_tx)) { - skb = mlx5e_ktls_handle_tx_skb(netdev, sq, skb, wqe, pi); - goto out; - } - if (!skb->sk || !tls_is_sk_tx_device_offloaded(skb->sk)) - goto out; + return true; datalen = skb->len - (skb_transport_offset(skb) + tcp_hdrlen(skb)); if (!datalen) - goto out; + return true; tls_ctx = tls_get_ctx(skb->sk); - if (unlikely(tls_ctx->netdev != netdev)) - goto out; + if (WARN_ON_ONCE(tls_ctx->netdev != netdev)) + goto err_out; + + if (MLX5_CAP_GEN(sq->channel->mdev, tls_tx)) + return mlx5e_ktls_handle_tx_skb(tls_ctx, sq, skb, datalen, state); skb_seq = ntohl(tcp_hdr(skb)->seq); context = mlx5e_get_tls_tx_context(tls_ctx); expected_seq = context->expected_seq; - if (unlikely(expected_seq != skb_seq)) { - skb = mlx5e_tls_handle_ooo(context, sq, skb, wqe, pi, priv->tls); - goto out; - } + if (unlikely(expected_seq != skb_seq)) + return mlx5e_tls_handle_ooo(context, sq, skb, priv->tls); if (unlikely(mlx5e_tls_add_metadata(skb, context->swid))) { atomic64_inc(&priv->tls->sw_stats.tx_tls_drop_metadata); dev_kfree_skb_any(skb); - skb = NULL; - goto out; + return false; } context->expected_seq = skb_seq + datalen; -out: - return skb; + return true; + +err_out: + dev_kfree_skb_any(skb); + return false; +} + +void mlx5e_tls_handle_tx_wqe(struct mlx5e_txqsq *sq, struct mlx5_wqe_ctrl_seg *cseg, + struct mlx5e_accel_tx_tls_state *state) +{ + cseg->tisn = cpu_to_be32(state->tls_tisn << 8); } static int tls_update_resync_sn(struct net_device *netdev, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h index 90bc1f2384c8..a50d0394df0a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.h @@ -40,11 +40,14 @@ #include "en.h" #include "en/txrx.h" -struct sk_buff *mlx5e_tls_handle_tx_skb(struct net_device *netdev, - struct mlx5e_txqsq *sq, - struct sk_buff *skb, - struct mlx5e_tx_wqe **wqe, - u16 *pi); +struct mlx5e_accel_tx_tls_state { + u32 tls_tisn; +}; + +bool mlx5e_tls_handle_tx_skb(struct net_device *netdev, struct mlx5e_txqsq *sq, + struct sk_buff *skb, struct mlx5e_accel_tx_tls_state *state); +void mlx5e_tls_handle_tx_wqe(struct mlx5e_txqsq *sq, struct mlx5_wqe_ctrl_seg *cseg, + struct mlx5e_accel_tx_tls_state *state); void mlx5e_tls_handle_rx_skb(struct net_device *netdev, struct sk_buff *skb, u32 *cqe_bcnt); |