diff options
Diffstat (limited to 'drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c')
-rw-r--r-- | drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c | 208 |
1 files changed, 55 insertions, 153 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index 7cab08a2f715..a715601865d3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -35,26 +35,14 @@ #include <crypto/aead.h> #include <linux/inetdevice.h> #include <linux/netdevice.h> -#include <linux/module.h> #include "en.h" -#include "en_accel/ipsec.h" -#include "en_accel/ipsec_rxtx.h" -#include "en_accel/ipsec_fs.h" +#include "ipsec.h" +#include "ipsec_rxtx.h" static struct mlx5e_ipsec_sa_entry *to_ipsec_sa_entry(struct xfrm_state *x) { - struct mlx5e_ipsec_sa_entry *sa; - - if (!x) - return NULL; - - sa = (struct mlx5e_ipsec_sa_entry *)x->xso.offload_handle; - if (!sa) - return NULL; - - WARN_ON(sa->x != x); - return sa; + return (struct mlx5e_ipsec_sa_entry *)x->xso.offload_handle; } struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *ipsec, @@ -75,9 +63,9 @@ struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *ipsec, return ret; } -static int mlx5e_ipsec_sadb_rx_add(struct mlx5e_ipsec_sa_entry *sa_entry, - unsigned int handle) +static int mlx5e_ipsec_sadb_rx_add(struct mlx5e_ipsec_sa_entry *sa_entry) { + unsigned int handle = sa_entry->ipsec_obj_id; struct mlx5e_ipsec *ipsec = sa_entry->ipsec; struct mlx5e_ipsec_sa_entry *_sa_entry; unsigned long flags; @@ -113,7 +101,6 @@ static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry) struct xfrm_replay_state_esn *replay_esn; u32 seq_bottom = 0; u8 overlap; - u32 *esn; if (!(sa_entry->x->props.flags & XFRM_STATE_ESN)) { sa_entry->esn_state.trigger = 0; @@ -128,11 +115,9 @@ static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry) sa_entry->esn_state.esn = xfrm_replay_seqhi(sa_entry->x, htonl(seq_bottom)); - esn = &sa_entry->esn_state.esn; sa_entry->esn_state.trigger = 1; if (unlikely(overlap && seq_bottom < MLX5E_IPSEC_ESN_SCOPE_MID)) { - ++(*esn); sa_entry->esn_state.overlap = 0; return true; } else if (unlikely(!overlap && @@ -149,7 +134,7 @@ mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, struct mlx5_accel_esp_xfrm_attrs *attrs) { struct xfrm_state *x = sa_entry->x; - struct aes_gcm_keymat *aes_gcm = &attrs->keymat.aes_gcm; + struct aes_gcm_keymat *aes_gcm = &attrs->aes_gcm; struct aead_geniv_ctx *geniv_ctx; struct crypto_aead *aead; unsigned int crypto_data_len, key_len; @@ -183,23 +168,17 @@ mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, attrs->flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; } - /* rx handle */ - attrs->sa_handle = sa_entry->handle; - - /* algo type */ - attrs->keymat_type = MLX5_ACCEL_ESP_KEYMAT_AES_GCM; - /* action */ - attrs->action = (!(x->xso.flags & XFRM_OFFLOAD_INBOUND)) ? - MLX5_ACCEL_ESP_ACTION_ENCRYPT : - MLX5_ACCEL_ESP_ACTION_DECRYPT; + attrs->action = (x->xso.dir == XFRM_DEV_OFFLOAD_OUT) ? + MLX5_ACCEL_ESP_ACTION_ENCRYPT : + MLX5_ACCEL_ESP_ACTION_DECRYPT; /* flags */ attrs->flags |= (x->props.mode == XFRM_MODE_TRANSPORT) ? MLX5_ACCEL_ESP_FLAGS_TRANSPORT : MLX5_ACCEL_ESP_FLAGS_TUNNEL; /* spi */ - attrs->spi = x->id.spi; + attrs->spi = be32_to_cpu(x->id.spi); /* source , destination ips */ memcpy(&attrs->saddr, x->props.saddr.a6, sizeof(attrs->saddr)); @@ -227,8 +206,7 @@ static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) return -EINVAL; } if (x->props.flags & XFRM_STATE_ESN && - !(mlx5_accel_ipsec_device_caps(priv->mdev) & - MLX5_ACCEL_IPSEC_CAP_ESN)) { + !(mlx5_ipsec_device_caps(priv->mdev) & MLX5_IPSEC_CAP_ESN)) { netdev_info(netdev, "Cannot offload ESN xfrm states\n"); return -EINVAL; } @@ -275,46 +253,29 @@ static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) netdev_info(netdev, "Cannot offload xfrm states with geniv other than seqiv\n"); return -EINVAL; } - if (x->props.family == AF_INET6 && - !(mlx5_accel_ipsec_device_caps(priv->mdev) & - MLX5_ACCEL_IPSEC_CAP_IPV6)) { - netdev_info(netdev, "IPv6 xfrm state offload is not supported by this device\n"); - return -EINVAL; - } return 0; } -static int mlx5e_xfrm_fs_add_rule(struct mlx5e_priv *priv, - struct mlx5e_ipsec_sa_entry *sa_entry) -{ - if (!mlx5_is_ipsec_device(priv->mdev)) - return 0; - - return mlx5e_accel_ipsec_fs_add_rule(priv, &sa_entry->xfrm->attrs, - sa_entry->ipsec_obj_id, - &sa_entry->ipsec_rule); -} - -static void mlx5e_xfrm_fs_del_rule(struct mlx5e_priv *priv, - struct mlx5e_ipsec_sa_entry *sa_entry) +static void _update_xfrm_state(struct work_struct *work) { - if (!mlx5_is_ipsec_device(priv->mdev)) - return; + struct mlx5e_ipsec_modify_state_work *modify_work = + container_of(work, struct mlx5e_ipsec_modify_state_work, work); + struct mlx5e_ipsec_sa_entry *sa_entry = container_of( + modify_work, struct mlx5e_ipsec_sa_entry, modify_work); - mlx5e_accel_ipsec_fs_del_rule(priv, &sa_entry->xfrm->attrs, - &sa_entry->ipsec_rule); + mlx5_accel_esp_modify_xfrm(sa_entry, &modify_work->attrs); } static int mlx5e_xfrm_add_state(struct xfrm_state *x) { struct mlx5e_ipsec_sa_entry *sa_entry = NULL; struct net_device *netdev = x->xso.real_dev; - struct mlx5_accel_esp_xfrm_attrs attrs; struct mlx5e_priv *priv; - unsigned int sa_handle; int err; priv = netdev_priv(netdev); + if (!priv->ipsec) + return -EOPNOTSUPP; err = mlx5e_xfrm_validate_state(x); if (err) @@ -332,33 +293,18 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) /* check esn */ mlx5e_ipsec_update_esn_state(sa_entry); - /* create xfrm */ - mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &attrs); - sa_entry->xfrm = - mlx5_accel_esp_create_xfrm(priv->mdev, &attrs, - MLX5_ACCEL_XFRM_FLAG_REQUIRE_METADATA); - if (IS_ERR(sa_entry->xfrm)) { - err = PTR_ERR(sa_entry->xfrm); - goto err_sa_entry; - } - + mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &sa_entry->attrs); /* create hw context */ - sa_entry->hw_context = - mlx5_accel_esp_create_hw_context(priv->mdev, - sa_entry->xfrm, - &sa_handle); - if (IS_ERR(sa_entry->hw_context)) { - err = PTR_ERR(sa_entry->hw_context); + err = mlx5_ipsec_create_sa_ctx(sa_entry); + if (err) goto err_xfrm; - } - sa_entry->ipsec_obj_id = sa_handle; - err = mlx5e_xfrm_fs_add_rule(priv, sa_entry); + err = mlx5e_accel_ipsec_fs_add_rule(priv, sa_entry); if (err) goto err_hw_ctx; - if (x->xso.flags & XFRM_OFFLOAD_INBOUND) { - err = mlx5e_ipsec_sadb_rx_add(sa_entry, sa_handle); + if (x->xso.dir == XFRM_DEV_OFFLOAD_IN) { + err = mlx5e_ipsec_sadb_rx_add(sa_entry); if (err) goto err_add_rule; } else { @@ -366,18 +312,16 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) mlx5e_ipsec_set_iv_esn : mlx5e_ipsec_set_iv; } + INIT_WORK(&sa_entry->modify_work.work, _update_xfrm_state); x->xso.offload_handle = (unsigned long)sa_entry; goto out; err_add_rule: - mlx5e_xfrm_fs_del_rule(priv, sa_entry); + mlx5e_accel_ipsec_fs_del_rule(priv, sa_entry); err_hw_ctx: - mlx5_accel_esp_free_hw_context(priv->mdev, sa_entry->hw_context); + mlx5_ipsec_free_sa_ctx(sa_entry); err_xfrm: - mlx5_accel_esp_destroy_xfrm(sa_entry->xfrm); -err_sa_entry: kfree(sa_entry); - out: return err; } @@ -386,10 +330,7 @@ static void mlx5e_xfrm_del_state(struct xfrm_state *x) { struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); - if (!sa_entry) - return; - - if (x->xso.flags & XFRM_OFFLOAD_INBOUND) + if (x->xso.dir == XFRM_DEV_OFFLOAD_IN) mlx5e_ipsec_sadb_rx_del(sa_entry); } @@ -398,24 +339,18 @@ static void mlx5e_xfrm_free_state(struct xfrm_state *x) struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); struct mlx5e_priv *priv = netdev_priv(x->xso.dev); - if (!sa_entry) - return; - - if (sa_entry->hw_context) { - flush_workqueue(sa_entry->ipsec->wq); - mlx5e_xfrm_fs_del_rule(priv, sa_entry); - mlx5_accel_esp_free_hw_context(sa_entry->xfrm->mdev, sa_entry->hw_context); - mlx5_accel_esp_destroy_xfrm(sa_entry->xfrm); - } - + cancel_work_sync(&sa_entry->modify_work.work); + mlx5e_accel_ipsec_fs_del_rule(priv, sa_entry); + mlx5_ipsec_free_sa_ctx(sa_entry); kfree(sa_entry); } int mlx5e_ipsec_init(struct mlx5e_priv *priv) { - struct mlx5e_ipsec *ipsec = NULL; + struct mlx5e_ipsec *ipsec; + int ret; - if (!MLX5_IPSEC_DEV(priv->mdev)) { + if (!mlx5_ipsec_device_caps(priv->mdev)) { netdev_dbg(priv->netdev, "Not an IPSec offload device\n"); return 0; } @@ -426,21 +361,27 @@ int mlx5e_ipsec_init(struct mlx5e_priv *priv) hash_init(ipsec->sadb_rx); spin_lock_init(&ipsec->sadb_rx_lock); - ida_init(&ipsec->halloc); - ipsec->en_priv = priv; - ipsec->no_trailer = !!(mlx5_accel_ipsec_device_caps(priv->mdev) & - MLX5_ACCEL_IPSEC_CAP_RX_NO_TRAILER); + ipsec->mdev = priv->mdev; ipsec->wq = alloc_ordered_workqueue("mlx5e_ipsec: %s", 0, priv->netdev->name); if (!ipsec->wq) { - kfree(ipsec); - return -ENOMEM; + ret = -ENOMEM; + goto err_wq; } + ret = mlx5e_accel_ipsec_fs_init(ipsec); + if (ret) + goto err_fs_init; + priv->ipsec = ipsec; - mlx5e_accel_ipsec_fs_init(priv); netdev_dbg(priv->netdev, "IPSec attached to netdevice\n"); return 0; + +err_fs_init: + destroy_workqueue(ipsec->wq); +err_wq: + kfree(ipsec); + return (ret != -EOPNOTSUPP) ? ret : 0; } void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv) @@ -450,10 +391,8 @@ void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv) if (!ipsec) return; - mlx5e_accel_ipsec_fs_cleanup(priv); + mlx5e_accel_ipsec_fs_cleanup(ipsec); destroy_workqueue(ipsec->wq); - - ida_destroy(&ipsec->halloc); kfree(ipsec); priv->ipsec = NULL; } @@ -473,50 +412,19 @@ static bool mlx5e_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x) return true; } -struct mlx5e_ipsec_modify_state_work { - struct work_struct work; - struct mlx5_accel_esp_xfrm_attrs attrs; - struct mlx5e_ipsec_sa_entry *sa_entry; -}; - -static void _update_xfrm_state(struct work_struct *work) -{ - int ret; - struct mlx5e_ipsec_modify_state_work *modify_work = - container_of(work, struct mlx5e_ipsec_modify_state_work, work); - struct mlx5e_ipsec_sa_entry *sa_entry = modify_work->sa_entry; - - ret = mlx5_accel_esp_modify_xfrm(sa_entry->xfrm, - &modify_work->attrs); - if (ret) - netdev_warn(sa_entry->ipsec->en_priv->netdev, - "Not an IPSec offload device\n"); - - kfree(modify_work); -} - static void mlx5e_xfrm_advance_esn_state(struct xfrm_state *x) { struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); - struct mlx5e_ipsec_modify_state_work *modify_work; + struct mlx5e_ipsec_modify_state_work *modify_work = + &sa_entry->modify_work; bool need_update; - if (!sa_entry) - return; - need_update = mlx5e_ipsec_update_esn_state(sa_entry); if (!need_update) return; - modify_work = kzalloc(sizeof(*modify_work), GFP_ATOMIC); - if (!modify_work) - return; - mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &modify_work->attrs); - modify_work->sa_entry = sa_entry; - - INIT_WORK(&modify_work->work, _update_xfrm_state); - WARN_ON(!queue_work(sa_entry->ipsec->wq, &modify_work->work)); + queue_work(sa_entry->ipsec->wq, &modify_work->work); } static const struct xfrmdev_ops mlx5e_ipsec_xfrmdev_ops = { @@ -532,11 +440,8 @@ void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv) struct mlx5_core_dev *mdev = priv->mdev; struct net_device *netdev = priv->netdev; - if (!(mlx5_accel_ipsec_device_caps(mdev) & MLX5_ACCEL_IPSEC_CAP_ESP) || - !MLX5_CAP_ETH(mdev, swp)) { - mlx5_core_dbg(mdev, "mlx5e: ESP and SWP offload not supported\n"); + if (!mlx5_ipsec_device_caps(mdev)) return; - } mlx5_core_info(mdev, "mlx5e: IPSec ESP acceleration enabled\n"); netdev->xfrmdev_ops = &mlx5e_ipsec_xfrmdev_ops; @@ -551,15 +456,12 @@ void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv) netdev->features |= NETIF_F_HW_ESP_TX_CSUM; netdev->hw_enc_features |= NETIF_F_HW_ESP_TX_CSUM; - if (!(mlx5_accel_ipsec_device_caps(mdev) & MLX5_ACCEL_IPSEC_CAP_LSO) || - !MLX5_CAP_ETH(mdev, swp_lso)) { + if (!MLX5_CAP_ETH(mdev, swp_lso)) { mlx5_core_dbg(mdev, "mlx5e: ESP LSO not supported\n"); return; } - if (mlx5_is_ipsec_device(mdev)) - netdev->gso_partial_features |= NETIF_F_GSO_ESP; - + netdev->gso_partial_features |= NETIF_F_GSO_ESP; mlx5_core_dbg(mdev, "mlx5e: ESP GSO capability turned on\n"); netdev->features |= NETIF_F_GSO_ESP; netdev->hw_features |= NETIF_F_GSO_ESP; |