diff options
Diffstat (limited to 'drivers/net/ethernet/netronome')
20 files changed, 1322 insertions, 666 deletions
diff --git a/drivers/net/ethernet/netronome/nfp/Makefile b/drivers/net/ethernet/netronome/nfp/Makefile index 4afb10375397..190e8b56a41f 100644 --- a/drivers/net/ethernet/netronome/nfp/Makefile +++ b/drivers/net/ethernet/netronome/nfp/Makefile @@ -57,6 +57,7 @@ endif ifeq ($(CONFIG_NFP_APP_ABM_NIC),y) nfp-objs += \ abm/ctrl.o \ + abm/qdisc.o \ abm/main.o endif diff --git a/drivers/net/ethernet/netronome/nfp/abm/ctrl.c b/drivers/net/ethernet/netronome/nfp/abm/ctrl.c index 3c661f422688..1629b07f727b 100644 --- a/drivers/net/ethernet/netronome/nfp/abm/ctrl.c +++ b/drivers/net/ethernet/netronome/nfp/abm/ctrl.c @@ -50,56 +50,37 @@ nfp_abm_ctrl_stat(struct nfp_abm_link *alink, const struct nfp_rtsym *sym, return 0; } -static int -nfp_abm_ctrl_stat_all(struct nfp_abm_link *alink, const struct nfp_rtsym *sym, - unsigned int stride, unsigned int offset, bool is_u64, - u64 *res) -{ - u64 val, sum = 0; - unsigned int i; - int err; - - for (i = 0; i < alink->vnic->max_rx_rings; i++) { - err = nfp_abm_ctrl_stat(alink, sym, stride, offset, i, - is_u64, &val); - if (err) - return err; - sum += val; - } - - *res = sum; - return 0; -} - -int nfp_abm_ctrl_set_q_lvl(struct nfp_abm_link *alink, unsigned int i, u32 val) +int __nfp_abm_ctrl_set_q_lvl(struct nfp_abm *abm, unsigned int id, u32 val) { - struct nfp_cpp *cpp = alink->abm->app->cpp; + struct nfp_cpp *cpp = abm->app->cpp; u64 sym_offset; int err; - sym_offset = (alink->queue_base + i) * NFP_QLVL_STRIDE + NFP_QLVL_THRS; - err = __nfp_rtsym_writel(cpp, alink->abm->q_lvls, 4, 0, - sym_offset, val); + __clear_bit(id, abm->threshold_undef); + if (abm->thresholds[id] == val) + return 0; + + sym_offset = id * NFP_QLVL_STRIDE + NFP_QLVL_THRS; + err = __nfp_rtsym_writel(cpp, abm->q_lvls, 4, 0, sym_offset, val); if (err) { - nfp_err(cpp, "RED offload setting level failed on vNIC %d queue %d\n", - alink->id, i); + nfp_err(cpp, + "RED offload setting level failed on subqueue %d\n", + id); return err; } + abm->thresholds[id] = val; return 0; } -int nfp_abm_ctrl_set_all_q_lvls(struct nfp_abm_link *alink, u32 val) +int nfp_abm_ctrl_set_q_lvl(struct nfp_abm_link *alink, unsigned int queue, + u32 val) { - int i, err; + unsigned int threshold; - for (i = 0; i < alink->vnic->max_rx_rings; i++) { - err = nfp_abm_ctrl_set_q_lvl(alink, i, val); - if (err) - return err; - } + threshold = alink->queue_base + queue; - return 0; + return __nfp_abm_ctrl_set_q_lvl(alink->abm, threshold, val); } u64 nfp_abm_ctrl_stat_non_sto(struct nfp_abm_link *alink, unsigned int i) @@ -153,42 +134,6 @@ int nfp_abm_ctrl_read_q_stats(struct nfp_abm_link *alink, unsigned int i, i, true, &stats->overlimits); } -int nfp_abm_ctrl_read_stats(struct nfp_abm_link *alink, - struct nfp_alink_stats *stats) -{ - u64 pkts = 0, bytes = 0; - int i, err; - - for (i = 0; i < alink->vnic->max_rx_rings; i++) { - pkts += nn_readq(alink->vnic, NFP_NET_CFG_RXR_STATS(i)); - bytes += nn_readq(alink->vnic, NFP_NET_CFG_RXR_STATS(i) + 8); - } - stats->tx_pkts = pkts; - stats->tx_bytes = bytes; - - err = nfp_abm_ctrl_stat_all(alink, alink->abm->q_lvls, - NFP_QLVL_STRIDE, NFP_QLVL_BLOG_BYTES, - false, &stats->backlog_bytes); - if (err) - return err; - - err = nfp_abm_ctrl_stat_all(alink, alink->abm->q_lvls, - NFP_QLVL_STRIDE, NFP_QLVL_BLOG_PKTS, - false, &stats->backlog_pkts); - if (err) - return err; - - err = nfp_abm_ctrl_stat_all(alink, alink->abm->qm_stats, - NFP_QMSTAT_STRIDE, NFP_QMSTAT_DROP, - true, &stats->drops); - if (err) - return err; - - return nfp_abm_ctrl_stat_all(alink, alink->abm->qm_stats, - NFP_QMSTAT_STRIDE, NFP_QMSTAT_ECN, - true, &stats->overlimits); -} - int nfp_abm_ctrl_read_q_xstats(struct nfp_abm_link *alink, unsigned int i, struct nfp_alink_xstats *xstats) { @@ -205,22 +150,6 @@ int nfp_abm_ctrl_read_q_xstats(struct nfp_abm_link *alink, unsigned int i, i, true, &xstats->ecn_marked); } -int nfp_abm_ctrl_read_xstats(struct nfp_abm_link *alink, - struct nfp_alink_xstats *xstats) -{ - int err; - - err = nfp_abm_ctrl_stat_all(alink, alink->abm->qm_stats, - NFP_QMSTAT_STRIDE, NFP_QMSTAT_DROP, - true, &xstats->pdrop); - if (err) - return err; - - return nfp_abm_ctrl_stat_all(alink, alink->abm->qm_stats, - NFP_QMSTAT_STRIDE, NFP_QMSTAT_ECN, - true, &xstats->ecn_marked); -} - int nfp_abm_ctrl_qm_enable(struct nfp_abm *abm) { return nfp_mbox_cmd(abm->app->pf, NFP_MBOX_PCIE_ABM_ENABLE, diff --git a/drivers/net/ethernet/netronome/nfp/abm/main.c b/drivers/net/ethernet/netronome/nfp/abm/main.c index c0830c0c2c3f..a5732d3bd1b7 100644 --- a/drivers/net/ethernet/netronome/nfp/abm/main.c +++ b/drivers/net/ethernet/netronome/nfp/abm/main.c @@ -2,14 +2,13 @@ /* Copyright (C) 2018 Netronome Systems, Inc. */ #include <linux/bitfield.h> +#include <linux/bitmap.h> #include <linux/etherdevice.h> #include <linux/lockdep.h> #include <linux/netdevice.h> #include <linux/rcupdate.h> +#include <linux/rtnetlink.h> #include <linux/slab.h> -#include <net/pkt_cls.h> -#include <net/pkt_sched.h> -#include <net/red.h> #include "../nfpcore/nfp.h" #include "../nfpcore/nfp_cpp.h" @@ -28,269 +27,6 @@ static u32 nfp_abm_portid(enum nfp_repr_type rtype, unsigned int id) } static int -__nfp_abm_reset_root(struct net_device *netdev, struct nfp_abm_link *alink, - u32 handle, unsigned int qs, u32 init_val) -{ - struct nfp_port *port = nfp_port_from_netdev(netdev); - int ret; - - ret = nfp_abm_ctrl_set_all_q_lvls(alink, init_val); - memset(alink->qdiscs, 0, sizeof(*alink->qdiscs) * alink->num_qdiscs); - - alink->parent = handle; - alink->num_qdiscs = qs; - port->tc_offload_cnt = qs; - - return ret; -} - -static void -nfp_abm_reset_root(struct net_device *netdev, struct nfp_abm_link *alink, - u32 handle, unsigned int qs) -{ - __nfp_abm_reset_root(netdev, alink, handle, qs, ~0); -} - -static int -nfp_abm_red_find(struct nfp_abm_link *alink, struct tc_red_qopt_offload *opt) -{ - unsigned int i = TC_H_MIN(opt->parent) - 1; - - if (opt->parent == TC_H_ROOT) - i = 0; - else if (TC_H_MAJ(alink->parent) == TC_H_MAJ(opt->parent)) - i = TC_H_MIN(opt->parent) - 1; - else - return -EOPNOTSUPP; - - if (i >= alink->num_qdiscs || opt->handle != alink->qdiscs[i].handle) - return -EOPNOTSUPP; - - return i; -} - -static void -nfp_abm_red_destroy(struct net_device *netdev, struct nfp_abm_link *alink, - u32 handle) -{ - unsigned int i; - - for (i = 0; i < alink->num_qdiscs; i++) - if (handle == alink->qdiscs[i].handle) - break; - if (i == alink->num_qdiscs) - return; - - if (alink->parent == TC_H_ROOT) { - nfp_abm_reset_root(netdev, alink, TC_H_ROOT, 0); - } else { - nfp_abm_ctrl_set_q_lvl(alink, i, ~0); - memset(&alink->qdiscs[i], 0, sizeof(*alink->qdiscs)); - } -} - -static int -nfp_abm_red_replace(struct net_device *netdev, struct nfp_abm_link *alink, - struct tc_red_qopt_offload *opt) -{ - bool existing; - int i, err; - - i = nfp_abm_red_find(alink, opt); - existing = i >= 0; - - if (opt->set.min != opt->set.max || !opt->set.is_ecn) { - nfp_warn(alink->abm->app->cpp, - "RED offload failed - unsupported parameters\n"); - err = -EINVAL; - goto err_destroy; - } - - if (existing) { - if (alink->parent == TC_H_ROOT) - err = nfp_abm_ctrl_set_all_q_lvls(alink, opt->set.min); - else - err = nfp_abm_ctrl_set_q_lvl(alink, i, opt->set.min); - if (err) - goto err_destroy; - return 0; - } - - if (opt->parent == TC_H_ROOT) { - i = 0; - err = __nfp_abm_reset_root(netdev, alink, TC_H_ROOT, 1, - opt->set.min); - } else if (TC_H_MAJ(alink->parent) == TC_H_MAJ(opt->parent)) { - i = TC_H_MIN(opt->parent) - 1; - err = nfp_abm_ctrl_set_q_lvl(alink, i, opt->set.min); - } else { - return -EINVAL; - } - /* Set the handle to try full clean up, in case IO failed */ - alink->qdiscs[i].handle = opt->handle; - if (err) - goto err_destroy; - - if (opt->parent == TC_H_ROOT) - err = nfp_abm_ctrl_read_stats(alink, &alink->qdiscs[i].stats); - else - err = nfp_abm_ctrl_read_q_stats(alink, i, - &alink->qdiscs[i].stats); - if (err) - goto err_destroy; - - if (opt->parent == TC_H_ROOT) - err = nfp_abm_ctrl_read_xstats(alink, - &alink->qdiscs[i].xstats); - else - err = nfp_abm_ctrl_read_q_xstats(alink, i, - &alink->qdiscs[i].xstats); - if (err) - goto err_destroy; - - alink->qdiscs[i].stats.backlog_pkts = 0; - alink->qdiscs[i].stats.backlog_bytes = 0; - - return 0; -err_destroy: - /* If the qdisc keeps on living, but we can't offload undo changes */ - if (existing) { - opt->set.qstats->qlen -= alink->qdiscs[i].stats.backlog_pkts; - opt->set.qstats->backlog -= - alink->qdiscs[i].stats.backlog_bytes; - } - nfp_abm_red_destroy(netdev, alink, opt->handle); - - return err; -} - -static void -nfp_abm_update_stats(struct nfp_alink_stats *new, struct nfp_alink_stats *old, - struct tc_qopt_offload_stats *stats) -{ - _bstats_update(stats->bstats, new->tx_bytes - old->tx_bytes, - new->tx_pkts - old->tx_pkts); - stats->qstats->qlen += new->backlog_pkts - old->backlog_pkts; - stats->qstats->backlog += new->backlog_bytes - old->backlog_bytes; - stats->qstats->overlimits += new->overlimits - old->overlimits; - stats->qstats->drops += new->drops - old->drops; -} - -static int -nfp_abm_red_stats(struct nfp_abm_link *alink, struct tc_red_qopt_offload *opt) -{ - struct nfp_alink_stats *prev_stats; - struct nfp_alink_stats stats; - int i, err; - - i = nfp_abm_red_find(alink, opt); - if (i < 0) - return i; - prev_stats = &alink->qdiscs[i].stats; - - if (alink->parent == TC_H_ROOT) - err = nfp_abm_ctrl_read_stats(alink, &stats); - else - err = nfp_abm_ctrl_read_q_stats(alink, i, &stats); - if (err) - return err; - - nfp_abm_update_stats(&stats, prev_stats, &opt->stats); - - *prev_stats = stats; - - return 0; -} - -static int -nfp_abm_red_xstats(struct nfp_abm_link *alink, struct tc_red_qopt_offload *opt) -{ - struct nfp_alink_xstats *prev_xstats; - struct nfp_alink_xstats xstats; - int i, err; - - i = nfp_abm_red_find(alink, opt); - if (i < 0) - return i; - prev_xstats = &alink->qdiscs[i].xstats; - - if (alink->parent == TC_H_ROOT) - err = nfp_abm_ctrl_read_xstats(alink, &xstats); - else - err = nfp_abm_ctrl_read_q_xstats(alink, i, &xstats); - if (err) - return err; - - opt->xstats->forced_mark += xstats.ecn_marked - prev_xstats->ecn_marked; - opt->xstats->pdrop += xstats.pdrop - prev_xstats->pdrop; - - *prev_xstats = xstats; - - return 0; -} - -static int -nfp_abm_setup_tc_red(struct net_device *netdev, struct nfp_abm_link *alink, - struct tc_red_qopt_offload *opt) -{ - switch (opt->command) { - case TC_RED_REPLACE: - return nfp_abm_red_replace(netdev, alink, opt); - case TC_RED_DESTROY: - nfp_abm_red_destroy(netdev, alink, opt->handle); - return 0; - case TC_RED_STATS: - return nfp_abm_red_stats(alink, opt); - case TC_RED_XSTATS: - return nfp_abm_red_xstats(alink, opt); - default: - return -EOPNOTSUPP; - } -} - -static int -nfp_abm_mq_stats(struct nfp_abm_link *alink, struct tc_mq_qopt_offload *opt) -{ - struct nfp_alink_stats stats; - unsigned int i; - int err; - - for (i = 0; i < alink->num_qdiscs; i++) { - if (alink->qdiscs[i].handle == TC_H_UNSPEC) - continue; - - err = nfp_abm_ctrl_read_q_stats(alink, i, &stats); - if (err) - return err; - - nfp_abm_update_stats(&stats, &alink->qdiscs[i].stats, - &opt->stats); - } - - return 0; -} - -static int -nfp_abm_setup_tc_mq(struct net_device *netdev, struct nfp_abm_link *alink, - struct tc_mq_qopt_offload *opt) -{ - switch (opt->command) { - case TC_MQ_CREATE: - nfp_abm_reset_root(netdev, alink, opt->handle, - alink->total_queues); - return 0; - case TC_MQ_DESTROY: - if (opt->handle == alink->parent) - nfp_abm_reset_root(netdev, alink, TC_H_ROOT, 0); - return 0; - case TC_MQ_STATS: - return nfp_abm_mq_stats(alink, opt); - default: - return -EOPNOTSUPP; - } -} - -static int nfp_abm_setup_tc(struct nfp_app *app, struct net_device *netdev, enum tc_setup_type type, void *type_data) { @@ -302,6 +38,8 @@ nfp_abm_setup_tc(struct nfp_app *app, struct net_device *netdev, return -EOPNOTSUPP; switch (type) { + case TC_SETUP_ROOT_QDISC: + return nfp_abm_setup_root(netdev, repr->app_priv, type_data); case TC_SETUP_QDISC_MQ: return nfp_abm_setup_tc_mq(netdev, repr->app_priv, type_data); case TC_SETUP_QDISC_RED: @@ -573,31 +311,23 @@ nfp_abm_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id) alink->abm = abm; alink->vnic = nn; alink->id = id; - alink->parent = TC_H_ROOT; alink->total_queues = alink->vnic->max_rx_rings; - alink->qdiscs = kvcalloc(alink->total_queues, sizeof(*alink->qdiscs), - GFP_KERNEL); - if (!alink->qdiscs) { - err = -ENOMEM; - goto err_free_alink; - } /* This is a multi-host app, make sure MAC/PHY is up, but don't * make the MAC/PHY state follow the state of any of the ports. */ err = nfp_eth_set_configured(app->cpp, eth_port->index, true); if (err < 0) - goto err_free_qdiscs; + goto err_free_alink; netif_keep_dst(nn->dp.netdev); nfp_abm_vnic_set_mac(app->pf, abm, nn, id); nfp_abm_ctrl_read_params(alink); + INIT_RADIX_TREE(&alink->qdiscs, GFP_KERNEL); return 0; -err_free_qdiscs: - kvfree(alink->qdiscs); err_free_alink: kfree(alink); return err; @@ -608,7 +338,7 @@ static void nfp_abm_vnic_free(struct nfp_app *app, struct nfp_net *nn) struct nfp_abm_link *alink = nn->app_priv; nfp_abm_kill_reprs(alink->abm, alink); - kvfree(alink->qdiscs); + WARN(!radix_tree_empty(&alink->qdiscs), "left over qdiscs\n"); kfree(alink); } @@ -664,6 +394,7 @@ static int nfp_abm_init(struct nfp_app *app) struct nfp_pf *pf = app->pf; struct nfp_reprs *reprs; struct nfp_abm *abm; + unsigned int i; int err; if (!pf->eth_tbl) { @@ -690,15 +421,28 @@ static int nfp_abm_init(struct nfp_app *app) if (err) goto err_free_abm; + err = -ENOMEM; + abm->num_thresholds = NFP_NET_MAX_RX_RINGS; + abm->threshold_undef = bitmap_zalloc(abm->num_thresholds, GFP_KERNEL); + if (!abm->threshold_undef) + goto err_free_abm; + + abm->thresholds = kvcalloc(abm->num_thresholds, + sizeof(*abm->thresholds), GFP_KERNEL); + if (!abm->thresholds) + goto err_free_thresh_umap; + for (i = 0; i < NFP_NET_MAX_RX_RINGS; i++) + __nfp_abm_ctrl_set_q_lvl(abm, i, NFP_ABM_LVL_INFINITY); + /* We start in legacy mode, make sure advanced queuing is disabled */ err = nfp_abm_ctrl_qm_disable(abm); if (err) - goto err_free_abm; + goto err_free_thresh; err = -ENOMEM; reprs = nfp_reprs_alloc(pf->max_data_vnics); if (!reprs) - goto err_free_abm; + goto err_free_thresh; RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PHYS_PORT], reprs); reprs = nfp_reprs_alloc(pf->max_data_vnics); @@ -710,6 +454,10 @@ static int nfp_abm_init(struct nfp_app *app) err_free_phys: nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT); +err_free_thresh: + kvfree(abm->thresholds); +err_free_thresh_umap: + bitmap_free(abm->threshold_undef); err_free_abm: kfree(abm); app->priv = NULL; @@ -723,6 +471,8 @@ static void nfp_abm_clean(struct nfp_app *app) nfp_abm_eswitch_clean_up(abm); nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PF); nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT); + bitmap_free(abm->threshold_undef); + kvfree(abm->thresholds); kfree(abm); app->priv = NULL; } diff --git a/drivers/net/ethernet/netronome/nfp/abm/main.h b/drivers/net/ethernet/netronome/nfp/abm/main.h index f907b7d98917..240e2c8683fe 100644 --- a/drivers/net/ethernet/netronome/nfp/abm/main.h +++ b/drivers/net/ethernet/netronome/nfp/abm/main.h @@ -4,7 +4,17 @@ #ifndef __NFP_ABM_H__ #define __NFP_ABM_H__ 1 +#include <linux/bits.h> +#include <linux/radix-tree.h> #include <net/devlink.h> +#include <net/pkt_cls.h> + +/* Dump of 64 PRIOs and 256 REDs seems to take 850us on Xeon v4 @ 2.20GHz; + * 2.5ms / 400Hz seems more than sufficient for stats resolution. + */ +#define NFP_ABM_STATS_REFRESH_IVAL (2500 * 1000) /* ns */ + +#define NFP_ABM_LVL_INFINITY S32_MAX struct nfp_app; struct nfp_net; @@ -16,6 +26,11 @@ struct nfp_net; * struct nfp_abm - ABM NIC app structure * @app: back pointer to nfp_app * @pf_id: ID of our PF link + * + * @thresholds: current threshold configuration + * @threshold_undef: bitmap of thresholds which have not been set + * @num_thresholds: number of @thresholds and bits in @threshold_undef + * * @eswitch_mode: devlink eswitch mode, advanced functions only visible * in switchdev mode * @q_lvls: queue level control area @@ -24,6 +39,11 @@ struct nfp_net; struct nfp_abm { struct nfp_app *app; unsigned int pf_id; + + u32 *thresholds; + unsigned long *threshold_undef; + size_t num_thresholds; + enum devlink_eswitch_mode eswitch_mode; const struct nfp_rtsym *q_lvls; const struct nfp_rtsym *qm_stats; @@ -57,16 +77,67 @@ struct nfp_alink_xstats { u64 pdrop; }; +enum nfp_qdisc_type { + NFP_QDISC_NONE = 0, + NFP_QDISC_MQ, + NFP_QDISC_RED, +}; + +#define NFP_QDISC_UNTRACKED ((struct nfp_qdisc *)1UL) + /** - * struct nfp_red_qdisc - representation of single RED Qdisc - * @handle: handle of currently offloaded RED Qdisc - * @stats: statistics from last refresh - * @xstats: base of extended statistics + * struct nfp_qdisc - tracked TC Qdisc + * @netdev: netdev on which Qdisc was created + * @type: Qdisc type + * @handle: handle of this Qdisc + * @parent_handle: handle of the parent (unreliable if Qdisc was grafted) + * @use_cnt: number of attachment points in the hierarchy + * @num_children: current size of the @children array + * @children: pointers to children + * + * @params_ok: parameters of this Qdisc are OK for offload + * @offload_mark: offload refresh state - selected for offload + * @offloaded: Qdisc is currently offloaded to the HW + * + * @mq: MQ Qdisc specific parameters and state + * @mq.stats: current stats of the MQ Qdisc + * @mq.prev_stats: previously reported @mq.stats + * + * @red: RED Qdisc specific parameters and state + * @red.threshold: ECN marking threshold + * @red.stats: current stats of the RED Qdisc + * @red.prev_stats: previously reported @red.stats + * @red.xstats: extended stats for RED - current + * @red.prev_xstats: extended stats for RED - previously reported */ -struct nfp_red_qdisc { +struct nfp_qdisc { + struct net_device *netdev; + enum nfp_qdisc_type type; u32 handle; - struct nfp_alink_stats stats; - struct nfp_alink_xstats xstats; + u32 parent_handle; + unsigned int use_cnt; + unsigned int num_children; + struct nfp_qdisc **children; + + bool params_ok; + bool offload_mark; + bool offloaded; + + union { + /* NFP_QDISC_MQ */ + struct { + struct nfp_alink_stats stats; + struct nfp_alink_stats prev_stats; + } mq; + /* TC_SETUP_QDISC_RED */ + struct { + u32 threshold; + struct nfp_alink_stats stats; + struct nfp_alink_stats prev_stats; + struct nfp_alink_xstats xstats; + struct nfp_alink_xstats prev_xstats; + } red; + }; }; /** @@ -76,9 +147,11 @@ struct nfp_red_qdisc { * @id: id of the data vNIC * @queue_base: id of base to host queue within PCIe (not QC idx) * @total_queues: number of PF queues - * @parent: handle of expected parent, i.e. handle of MQ, or TC_H_ROOT - * @num_qdiscs: number of currently used qdiscs - * @qdiscs: array of qdiscs + * + * @last_stats_update: ktime of last stats update + * + * @root_qdisc: pointer to the current root of the Qdisc hierarchy + * @qdiscs: all qdiscs recorded by major part of the handle */ struct nfp_abm_link { struct nfp_abm *abm; @@ -86,22 +159,28 @@ struct nfp_abm_link { unsigned int id; unsigned int queue_base; unsigned int total_queues; - u32 parent; - unsigned int num_qdiscs; - struct nfp_red_qdisc *qdiscs; + + u64 last_stats_update; + + struct nfp_qdisc *root_qdisc; + struct radix_tree_root qdiscs; }; +void nfp_abm_qdisc_offload_update(struct nfp_abm_link *alink); +int nfp_abm_setup_root(struct net_device *netdev, struct nfp_abm_link *alink, + struct tc_root_qopt_offload *opt); +int nfp_abm_setup_tc_red(struct net_device *netdev, struct nfp_abm_link *alink, + struct tc_red_qopt_offload *opt); +int nfp_abm_setup_tc_mq(struct net_device *netdev, struct nfp_abm_link *alink, + struct tc_mq_qopt_offload *opt); + void nfp_abm_ctrl_read_params(struct nfp_abm_link *alink); int nfp_abm_ctrl_find_addrs(struct nfp_abm *abm); -int nfp_abm_ctrl_set_all_q_lvls(struct nfp_abm_link *alink, u32 val); -int nfp_abm_ctrl_set_q_lvl(struct nfp_abm_link *alink, unsigned int i, +int __nfp_abm_ctrl_set_q_lvl(struct nfp_abm *abm, unsigned int id, u32 val); +int nfp_abm_ctrl_set_q_lvl(struct nfp_abm_link *alink, unsigned int queue, u32 val); -int nfp_abm_ctrl_read_stats(struct nfp_abm_link *alink, - struct nfp_alink_stats *stats); int nfp_abm_ctrl_read_q_stats(struct nfp_abm_link *alink, unsigned int i, struct nfp_alink_stats *stats); -int nfp_abm_ctrl_read_xstats(struct nfp_abm_link *alink, - struct nfp_alink_xstats *xstats); int nfp_abm_ctrl_read_q_xstats(struct nfp_abm_link *alink, unsigned int i, struct nfp_alink_xstats *xstats); u64 nfp_abm_ctrl_stat_non_sto(struct nfp_abm_link *alink, unsigned int i); diff --git a/drivers/net/ethernet/netronome/nfp/abm/qdisc.c b/drivers/net/ethernet/netronome/nfp/abm/qdisc.c new file mode 100644 index 000000000000..16c4afe3a37f --- /dev/null +++ b/drivers/net/ethernet/netronome/nfp/abm/qdisc.c @@ -0,0 +1,663 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* Copyright (C) 2018 Netronome Systems, Inc. */ + +#include <linux/rtnetlink.h> +#include <net/pkt_cls.h> +#include <net/pkt_sched.h> +#include <net/red.h> + +#include "../nfpcore/nfp_cpp.h" +#include "../nfp_app.h" +#include "../nfp_main.h" +#include "../nfp_net.h" +#include "../nfp_port.h" +#include "main.h" + +static bool nfp_abm_qdisc_is_red(struct nfp_qdisc *qdisc) +{ + return qdisc->type == NFP_QDISC_RED; +} + +static bool nfp_abm_qdisc_child_valid(struct nfp_qdisc *qdisc, unsigned int id) +{ + return qdisc->children[id] && + qdisc->children[id] != NFP_QDISC_UNTRACKED; +} + +static void *nfp_abm_qdisc_tree_deref_slot(void __rcu **slot) +{ + return rtnl_dereference(*slot); +} + +static void +nfp_abm_stats_propagate(struct nfp_alink_stats *parent, + struct nfp_alink_stats *child) +{ + parent->tx_pkts += child->tx_pkts; + parent->tx_bytes += child->tx_bytes; + parent->backlog_pkts += child->backlog_pkts; + parent->backlog_bytes += child->backlog_bytes; + parent->overlimits += child->overlimits; + parent->drops += child->drops; +} + +static void +nfp_abm_stats_update_red(struct nfp_abm_link *alink, struct nfp_qdisc *qdisc, + unsigned int queue) +{ + struct nfp_cpp *cpp = alink->abm->app->cpp; + int err; + + if (!qdisc->offloaded) + return; + + err = nfp_abm_ctrl_read_q_stats(alink, queue, &qdisc->red.stats); + if (err) + nfp_err(cpp, "RED stats (%d) read failed with error %d\n", + queue, err); + + err = nfp_abm_ctrl_read_q_xstats(alink, queue, &qdisc->red.xstats); + if (err) + nfp_err(cpp, "RED xstats (%d) read failed with error %d\n", + queue, err); +} + +static void +nfp_abm_stats_update_mq(struct nfp_abm_link *alink, struct nfp_qdisc *qdisc) +{ + unsigned int i; + + if (qdisc->type != NFP_QDISC_MQ) + return; + + for (i = 0; i < alink->total_queues; i++) + if (nfp_abm_qdisc_child_valid(qdisc, i)) + nfp_abm_stats_update_red(alink, qdisc->children[i], i); +} + +static void __nfp_abm_stats_update(struct nfp_abm_link *alink, u64 time_now) +{ + alink->last_stats_update = time_now; + if (alink->root_qdisc) + nfp_abm_stats_update_mq(alink, alink->root_qdisc); +} + +static void nfp_abm_stats_update(struct nfp_abm_link *alink) +{ + u64 now; + + /* Limit the frequency of updates - stats of non-leaf qdiscs are a sum + * of all their leafs, so we would read the same stat multiple times + * for every dump. + */ + now = ktime_get(); + if (now - alink->last_stats_update < NFP_ABM_STATS_REFRESH_IVAL) + return; + + __nfp_abm_stats_update(alink, now); +} + +static void +nfp_abm_qdisc_unlink_children(struct nfp_qdisc *qdisc, + unsigned int start, unsigned int end) +{ + unsigned int i; + + for (i = start; i < end; i++) + if (nfp_abm_qdisc_child_valid(qdisc, i)) { + qdisc->children[i]->use_cnt--; + qdisc->children[i] = NULL; + } +} + +static void +nfp_abm_qdisc_offload_stop(struct nfp_abm_link *alink, struct nfp_qdisc *qdisc) +{ + /* Don't complain when qdisc is getting unlinked */ + if (qdisc->use_cnt) + nfp_warn(alink->abm->app->cpp, "Offload of '%08x' stopped\n", + qdisc->handle); + + if (!nfp_abm_qdisc_is_red(qdisc)) + return; + + qdisc->red.stats.backlog_pkts = 0; + qdisc->red.stats.backlog_bytes = 0; +} + +static int +__nfp_abm_stats_init(struct nfp_abm_link *alink, + unsigned int queue, struct nfp_alink_stats *prev_stats, + struct nfp_alink_xstats *prev_xstats) +{ + u64 backlog_pkts, backlog_bytes; + int err; + + /* Don't touch the backlog, backlog can only be reset after it has + * been reported back to the tc qdisc stats. + */ + backlog_pkts = prev_stats->backlog_pkts; + backlog_bytes = prev_stats->backlog_bytes; + + err = nfp_abm_ctrl_read_q_stats(alink, queue, prev_stats); + if (err) { + nfp_err(alink->abm->app->cpp, + "RED stats init (%d) failed with error %d\n", + queue, err); + return err; + } + + err = nfp_abm_ctrl_read_q_xstats(alink, queue, prev_xstats); + if (err) { + nfp_err(alink->abm->app->cpp, + "RED xstats init (%d) failed with error %d\n", + queue, err); + return err; + } + + prev_stats->backlog_pkts = backlog_pkts; + prev_stats->backlog_bytes = backlog_bytes; + return 0; +} + +static int +nfp_abm_stats_init(struct nfp_abm_link *alink, struct nfp_qdisc *qdisc, + unsigned int queue) +{ + return __nfp_abm_stats_init(alink, queue, + &qdisc->red.prev_stats, + &qdisc->red.prev_xstats); +} + +static void +nfp_abm_offload_compile_red(struct nfp_abm_link *alink, struct nfp_qdisc *qdisc, + unsigned int queue) +{ + qdisc->offload_mark = qdisc->type == NFP_QDISC_RED && + qdisc->params_ok && + qdisc->use_cnt == 1 && + !qdisc->children[0]; + + /* If we are starting offload init prev_stats */ + if (qdisc->offload_mark && !qdisc->offloaded) + if (nfp_abm_stats_init(alink, qdisc, queue)) + qdisc->offload_mark = false; + + if (!qdisc->offload_mark) + return; + + nfp_abm_ctrl_set_q_lvl(alink, queue, qdisc->red.threshold); +} + +static void +nfp_abm_offload_compile_mq(struct nfp_abm_link *alink, struct nfp_qdisc *qdisc) +{ + unsigned int i; + + qdisc->offload_mark = qdisc->type == NFP_QDISC_MQ; + if (!qdisc->offload_mark) + return; + + for (i = 0; i < alink->total_queues; i++) { + struct nfp_qdisc *child = qdisc->children[i]; + + if (!nfp_abm_qdisc_child_valid(qdisc, i)) + continue; + + nfp_abm_offload_compile_red(alink, child, i); + } +} + +void nfp_abm_qdisc_offload_update(struct nfp_abm_link *alink) +{ + struct nfp_abm *abm = alink->abm; + struct radix_tree_iter iter; + struct nfp_qdisc *qdisc; + void __rcu **slot; + size_t i; + + /* Mark all thresholds as unconfigured */ + __bitmap_set(abm->threshold_undef, + alink->queue_base, alink->total_queues); + + /* Clear offload marks */ + radix_tree_for_each_slot(slot, &alink->qdiscs, &iter, 0) { + qdisc = nfp_abm_qdisc_tree_deref_slot(slot); + qdisc->offload_mark = false; + } + + if (alink->root_qdisc) + nfp_abm_offload_compile_mq(alink, alink->root_qdisc); + + /* Refresh offload status */ + radix_tree_for_each_slot(slot, &alink->qdiscs, &iter, 0) { + qdisc = nfp_abm_qdisc_tree_deref_slot(slot); + if (!qdisc->offload_mark && qdisc->offloaded) + nfp_abm_qdisc_offload_stop(alink, qdisc); + qdisc->offloaded = qdisc->offload_mark; + } + + /* Reset the unconfigured thresholds */ + for (i = 0; i < abm->num_thresholds; i++) + if (test_bit(i, abm->threshold_undef)) + __nfp_abm_ctrl_set_q_lvl(abm, i, NFP_ABM_LVL_INFINITY); + + __nfp_abm_stats_update(alink, ktime_get()); +} + +static void +nfp_abm_qdisc_clear_mq(struct net_device *netdev, struct nfp_abm_link *alink, + struct nfp_qdisc *qdisc) +{ + struct radix_tree_iter iter; + unsigned int mq_refs = 0; + void __rcu **slot; + + if (!qdisc->use_cnt) + return; + /* MQ doesn't notify well on destruction, we need special handling of + * MQ's children. + */ + if (qdisc->type == NFP_QDISC_MQ && + qdisc == alink->root_qdisc && + netdev->reg_state == NETREG_UNREGISTERING) + return; + + /* Count refs held by MQ instances and clear pointers */ + radix_tree_for_each_slot(slot, &alink->qdiscs, &iter, 0) { + struct nfp_qdisc *mq = nfp_abm_qdisc_tree_deref_slot(slot); + unsigned int i; + + if (mq->type != NFP_QDISC_MQ || mq->netdev != netdev) + continue; + for (i = 0; i < mq->num_children; i++) + if (mq->children[i] == qdisc) { + mq->children[i] = NULL; + mq_refs++; + } + } + + WARN(qdisc->use_cnt != mq_refs, "non-zero qdisc use count: %d (- %d)\n", + qdisc->use_cnt, mq_refs); +} + +static void +nfp_abm_qdisc_free(struct net_device *netdev, struct nfp_abm_link *alink, + struct nfp_qdisc *qdisc) +{ + struct nfp_port *port = nfp_port_from_netdev(netdev); + + if (!qdisc) + return; + nfp_abm_qdisc_clear_mq(netdev, alink, qdisc); + WARN_ON(radix_tree_delete(&alink->qdiscs, + TC_H_MAJ(qdisc->handle)) != qdisc); + + kfree(qdisc->children); + kfree(qdisc); + + port->tc_offload_cnt--; +} + +static struct nfp_qdisc * +nfp_abm_qdisc_alloc(struct net_device *netdev, struct nfp_abm_link *alink, + enum nfp_qdisc_type type, u32 parent_handle, u32 handle, + unsigned int children) +{ + struct nfp_port *port = nfp_port_from_netdev(netdev); + struct nfp_qdisc *qdisc; + int err; + + qdisc = kzalloc(sizeof(*qdisc), GFP_KERNEL); + if (!qdisc) + return NULL; + + qdisc->children = kcalloc(children, sizeof(void *), GFP_KERNEL); + if (!qdisc->children) + goto err_free_qdisc; + + qdisc->netdev = netdev; + qdisc->type = type; + qdisc->parent_handle = parent_handle; + qdisc->handle = handle; + qdisc->num_children = children; + + err = radix_tree_insert(&alink->qdiscs, TC_H_MAJ(qdisc->handle), qdisc); + if (err) { + nfp_err(alink->abm->app->cpp, + "Qdisc insertion into radix tree failed: %d\n", err); + goto err_free_child_tbl; + } + + port->tc_offload_cnt++; + return qdisc; + +err_free_child_tbl: + kfree(qdisc->children); +err_free_qdisc: + kfree(qdisc); + return NULL; +} + +static struct nfp_qdisc * +nfp_abm_qdisc_find(struct nfp_abm_link *alink, u32 handle) +{ + return radix_tree_lookup(&alink->qdiscs, TC_H_MAJ(handle)); +} + +static int +nfp_abm_qdisc_replace(struct net_device *netdev, struct nfp_abm_link *alink, + enum nfp_qdisc_type type, u32 parent_handle, u32 handle, + unsigned int children, struct nfp_qdisc **qdisc) +{ + *qdisc = nfp_abm_qdisc_find(alink, handle); + if (*qdisc) { + if (WARN_ON((*qdisc)->type != type)) + return -EINVAL; + return 1; + } + + *qdisc = nfp_abm_qdisc_alloc(netdev, alink, type, parent_handle, handle, + children); + return *qdisc ? 0 : -ENOMEM; +} + +static void +nfp_abm_qdisc_destroy(struct net_device *netdev, struct nfp_abm_link *alink, + u32 handle) +{ + struct nfp_qdisc *qdisc; + + qdisc = nfp_abm_qdisc_find(alink, handle); + if (!qdisc) + return; + + /* We don't get TC_SETUP_ROOT_QDISC w/ MQ when netdev is unregistered */ + if (alink->root_qdisc == qdisc) + qdisc->use_cnt--; + + nfp_abm_qdisc_unlink_children(qdisc, 0, qdisc->num_children); + nfp_abm_qdisc_free(netdev, alink, qdisc); + + if (alink->root_qdisc == qdisc) { + alink->root_qdisc = NULL; + /* Only root change matters, other changes are acted upon on + * the graft notification. + */ + nfp_abm_qdisc_offload_update(alink); + } +} + +static int +nfp_abm_qdisc_graft(struct nfp_abm_link *alink, u32 handle, u32 child_handle, + unsigned int id) +{ + struct nfp_qdisc *parent, *child; + + parent = nfp_abm_qdisc_find(alink, handle); + if (!parent) + return 0; + + if (WARN(id >= parent->num_children, + "graft child out of bound %d >= %d\n", + id, parent->num_children)) + return -EINVAL; + + nfp_abm_qdisc_unlink_children(parent, id, id + 1); + + child = nfp_abm_qdisc_find(alink, child_handle); + if (child) + child->use_cnt++; + else + child = NFP_QDISC_UNTRACKED; + parent->children[id] = child; + + nfp_abm_qdisc_offload_update(alink); + + return 0; +} + +static void +nfp_abm_stats_calculate(struct nfp_alink_stats *new, + struct nfp_alink_stats *old, + struct gnet_stats_basic_packed *bstats, + struct gnet_stats_queue *qstats) +{ + _bstats_update(bstats, new->tx_bytes - old->tx_bytes, + new->tx_pkts - old->tx_pkts); + qstats->qlen += new->backlog_pkts - old->backlog_pkts; + qstats->backlog += new->backlog_bytes - old->backlog_bytes; + qstats->overlimits += new->overlimits - old->overlimits; + qstats->drops += new->drops - old->drops; +} + +static void +nfp_abm_stats_red_calculate(struct nfp_alink_xstats *new, + struct nfp_alink_xstats *old, + struct red_stats *stats) +{ + stats->forced_mark += new->ecn_marked - old->ecn_marked; + stats->pdrop += new->pdrop - old->pdrop; +} + +static int +nfp_abm_red_xstats(struct nfp_abm_link *alink, struct tc_red_qopt_offload *opt) +{ + struct nfp_qdisc *qdisc; + + nfp_abm_stats_update(alink); + + qdisc = nfp_abm_qdisc_find(alink, opt->handle); + if (!qdisc || !qdisc->offloaded) + return -EOPNOTSUPP; + + nfp_abm_stats_red_calculate(&qdisc->red.xstats, + &qdisc->red.prev_xstats, + opt->xstats); + qdisc->red.prev_xstats = qdisc->red.xstats; + return 0; +} + +static int +nfp_abm_red_stats(struct nfp_abm_link *alink, u32 handle, + struct tc_qopt_offload_stats *stats) +{ + struct nfp_qdisc *qdisc; + + nfp_abm_stats_update(alink); + + qdisc = nfp_abm_qdisc_find(alink, handle); + if (!qdisc) + return -EOPNOTSUPP; + /* If the qdisc offload has stopped we may need to adjust the backlog + * counters back so carry on even if qdisc is not currently offloaded. + */ + + nfp_abm_stats_calculate(&qdisc->red.stats, + &qdisc->red.prev_stats, + stats->bstats, stats->qstats); + qdisc->red.prev_stats = qdisc->red.stats; + + return qdisc->offloaded ? 0 : -EOPNOTSUPP; +} + +static bool +nfp_abm_red_check_params(struct nfp_abm_link *alink, + struct tc_red_qopt_offload *opt) +{ + struct nfp_cpp *cpp = alink->abm->app->cpp; + + if (!opt->set.is_ecn) { + nfp_warn(cpp, "RED offload failed - drop is not supported (ECN option required) (p:%08x h:%08x)\n", + opt->parent, opt->handle); + return false; + } + if (opt->set.is_harddrop) { + nfp_warn(cpp, "RED offload failed - harddrop is not supported (p:%08x h:%08x)\n", + opt->parent, opt->handle); + return false; + } + if (opt->set.min != opt->set.max) { + nfp_warn(cpp, "RED offload failed - unsupported min/max parameters (p:%08x h:%08x)\n", + opt->parent, opt->handle); + return false; + } + if (opt->set.min > NFP_ABM_LVL_INFINITY) { + nfp_warn(cpp, "RED offload failed - threshold too large %d > %d (p:%08x h:%08x)\n", + opt->set.min, NFP_ABM_LVL_INFINITY, opt->parent, + opt->handle); + return false; + } + + return true; +} + +static int +nfp_abm_red_replace(struct net_device *netdev, struct nfp_abm_link *alink, + struct tc_red_qopt_offload *opt) +{ + struct nfp_qdisc *qdisc; + int ret; + + ret = nfp_abm_qdisc_replace(netdev, alink, NFP_QDISC_RED, opt->parent, + opt->handle, 1, &qdisc); + if (ret < 0) + return ret; + + /* If limit != 0 child gets reset */ + if (opt->set.limit) { + if (nfp_abm_qdisc_child_valid(qdisc, 0)) + qdisc->children[0]->use_cnt--; + qdisc->children[0] = NULL; + } else { + /* Qdisc was just allocated without a limit will use noop_qdisc, + * i.e. a block hole. + */ + if (!ret) + qdisc->children[0] = NFP_QDISC_UNTRACKED; + } + + qdisc->params_ok = nfp_abm_red_check_params(alink, opt); + if (qdisc->params_ok) + qdisc->red.threshold = opt->set.min; + + if (qdisc->use_cnt == 1) + nfp_abm_qdisc_offload_update(alink); + + return 0; +} + +int nfp_abm_setup_tc_red(struct net_device *netdev, struct nfp_abm_link *alink, + struct tc_red_qopt_offload *opt) +{ + switch (opt->command) { + case TC_RED_REPLACE: + return nfp_abm_red_replace(netdev, alink, opt); + case TC_RED_DESTROY: + nfp_abm_qdisc_destroy(netdev, alink, opt->handle); + return 0; + case TC_RED_STATS: + return nfp_abm_red_stats(alink, opt->handle, &opt->stats); + case TC_RED_XSTATS: + return nfp_abm_red_xstats(alink, opt); + case TC_RED_GRAFT: + return nfp_abm_qdisc_graft(alink, opt->handle, + opt->child_handle, 0); + default: + return -EOPNOTSUPP; + } +} + +static int +nfp_abm_mq_create(struct net_device *netdev, struct nfp_abm_link *alink, + struct tc_mq_qopt_offload *opt) +{ + struct nfp_qdisc *qdisc; + int ret; + + ret = nfp_abm_qdisc_replace(netdev, alink, NFP_QDISC_MQ, + TC_H_ROOT, opt->handle, alink->total_queues, + &qdisc); + if (ret < 0) + return ret; + + qdisc->params_ok = true; + qdisc->offloaded = true; + nfp_abm_qdisc_offload_update(alink); + return 0; +} + +static int +nfp_abm_mq_stats(struct nfp_abm_link *alink, u32 handle, + struct tc_qopt_offload_stats *stats) +{ + struct nfp_qdisc *qdisc, *red; + unsigned int i; + + qdisc = nfp_abm_qdisc_find(alink, handle); + if (!qdisc) + return -EOPNOTSUPP; + + nfp_abm_stats_update(alink); + + /* MQ stats are summed over the children in the core, so we need + * to add up the unreported child values. + */ + memset(&qdisc->mq.stats, 0, sizeof(qdisc->mq.stats)); + memset(&qdisc->mq.prev_stats, 0, sizeof(qdisc->mq.prev_stats)); + + for (i = 0; i < qdisc->num_children; i++) { + if (!nfp_abm_qdisc_child_valid(qdisc, i)) + continue; + + if (!nfp_abm_qdisc_is_red(qdisc->children[i])) + continue; + red = qdisc->children[i]; + + nfp_abm_stats_propagate(&qdisc->mq.stats, + &red->red.stats); + nfp_abm_stats_propagate(&qdisc->mq.prev_stats, + &red->red.prev_stats); + } + + nfp_abm_stats_calculate(&qdisc->mq.stats, &qdisc->mq.prev_stats, + stats->bstats, stats->qstats); + + return qdisc->offloaded ? 0 : -EOPNOTSUPP; +} + +int nfp_abm_setup_tc_mq(struct net_device *netdev, struct nfp_abm_link *alink, + struct tc_mq_qopt_offload *opt) +{ + switch (opt->command) { + case TC_MQ_CREATE: + return nfp_abm_mq_create(netdev, alink, opt); + case TC_MQ_DESTROY: + nfp_abm_qdisc_destroy(netdev, alink, opt->handle); + return 0; + case TC_MQ_STATS: + return nfp_abm_mq_stats(alink, opt->handle, &opt->stats); + case TC_MQ_GRAFT: + return nfp_abm_qdisc_graft(alink, opt->handle, + opt->graft_params.child_handle, + opt->graft_params.queue); + default: + return -EOPNOTSUPP; + } +} + +int nfp_abm_setup_root(struct net_device *netdev, struct nfp_abm_link *alink, + struct tc_root_qopt_offload *opt) +{ + if (opt->ingress) + return -EOPNOTSUPP; + if (alink->root_qdisc) + alink->root_qdisc->use_cnt--; + alink->root_qdisc = nfp_abm_qdisc_find(alink, opt->handle); + if (alink->root_qdisc) + alink->root_qdisc->use_cnt++; + + nfp_abm_qdisc_offload_update(alink); + + return 0; +} diff --git a/drivers/net/ethernet/netronome/nfp/flower/action.c b/drivers/net/ethernet/netronome/nfp/flower/action.c index 244dc261006e..8d54b36afee8 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/action.c +++ b/drivers/net/ethernet/netronome/nfp/flower/action.c @@ -2,7 +2,6 @@ /* Copyright (C) 2017-2018 Netronome Systems, Inc. */ #include <linux/bitfield.h> -#include <net/geneve.h> #include <net/pkt_cls.h> #include <net/switchdev.h> #include <net/tc_act/tc_csum.h> @@ -91,21 +90,6 @@ nfp_fl_pre_lag(struct nfp_app *app, const struct tc_action *action, return act_size; } -static bool nfp_fl_netdev_is_tunnel_type(struct net_device *out_dev, - enum nfp_flower_tun_type tun_type) -{ - if (!out_dev->rtnl_link_ops) - return false; - - if (!strcmp(out_dev->rtnl_link_ops->kind, "vxlan")) - return tun_type == NFP_FL_TUNNEL_VXLAN; - - if (!strcmp(out_dev->rtnl_link_ops->kind, "geneve")) - return tun_type == NFP_FL_TUNNEL_GENEVE; - - return false; -} - static int nfp_fl_output(struct nfp_app *app, struct nfp_fl_output *output, const struct tc_action *action, struct nfp_fl_payload *nfp_flow, @@ -151,11 +135,12 @@ nfp_fl_output(struct nfp_app *app, struct nfp_fl_output *output, /* Set action output parameters. */ output->flags = cpu_to_be16(tmp_flags); - /* Only offload if egress ports are on the same device as the - * ingress port. - */ - if (!switchdev_port_same_parent_id(in_dev, out_dev)) - return -EOPNOTSUPP; + if (nfp_netdev_is_nfp_repr(in_dev)) { + /* Confirm ingress and egress are on same device. */ + if (!switchdev_port_same_parent_id(in_dev, out_dev)) + return -EOPNOTSUPP; + } + if (!nfp_netdev_is_nfp_repr(out_dev)) return -EOPNOTSUPP; @@ -384,10 +369,21 @@ nfp_fl_set_eth(const struct tc_action *action, int idx, u32 off, return 0; } +struct ipv4_ttl_word { + __u8 ttl; + __u8 protocol; + __sum16 check; +}; + static int nfp_fl_set_ip4(const struct tc_action *action, int idx, u32 off, - struct nfp_fl_set_ip4_addrs *set_ip_addr) + struct nfp_fl_set_ip4_addrs *set_ip_addr, + struct nfp_fl_set_ip4_ttl_tos *set_ip_ttl_tos) { + struct ipv4_ttl_word *ttl_word_mask; + struct ipv4_ttl_word *ttl_word; + struct iphdr *tos_word_mask; + struct iphdr *tos_word; __be32 exact, mask; /* We are expecting tcf_pedit to return a big endian value */ @@ -402,20 +398,53 @@ nfp_fl_set_ip4(const struct tc_action *action, int idx, u32 off, set_ip_addr->ipv4_dst_mask |= mask; set_ip_addr->ipv4_dst &= ~mask; set_ip_addr->ipv4_dst |= exact & mask; + set_ip_addr->head.jump_id = NFP_FL_ACTION_OPCODE_SET_IPV4_ADDRS; + set_ip_addr->head.len_lw = sizeof(*set_ip_addr) >> + NFP_FL_LW_SIZ; break; case offsetof(struct iphdr, saddr): set_ip_addr->ipv4_src_mask |= mask; set_ip_addr->ipv4_src &= ~mask; set_ip_addr->ipv4_src |= exact & mask; + set_ip_addr->head.jump_id = NFP_FL_ACTION_OPCODE_SET_IPV4_ADDRS; + set_ip_addr->head.len_lw = sizeof(*set_ip_addr) >> + NFP_FL_LW_SIZ; + break; + case offsetof(struct iphdr, ttl): + ttl_word_mask = (struct ipv4_ttl_word *)&mask; + ttl_word = (struct ipv4_ttl_word *)&exact; + + if (ttl_word_mask->protocol || ttl_word_mask->check) + return -EOPNOTSUPP; + + set_ip_ttl_tos->ipv4_ttl_mask |= ttl_word_mask->ttl; + set_ip_ttl_tos->ipv4_ttl &= ~ttl_word_mask->ttl; + set_ip_ttl_tos->ipv4_ttl |= ttl_word->ttl & ttl_word_mask->ttl; + set_ip_ttl_tos->head.jump_id = + NFP_FL_ACTION_OPCODE_SET_IPV4_TTL_TOS; + set_ip_ttl_tos->head.len_lw = sizeof(*set_ip_ttl_tos) >> + NFP_FL_LW_SIZ; + break; + case round_down(offsetof(struct iphdr, tos), 4): + tos_word_mask = (struct iphdr *)&mask; + tos_word = (struct iphdr *)&exact; + + if (tos_word_mask->version || tos_word_mask->ihl || + tos_word_mask->tot_len) + return -EOPNOTSUPP; + + set_ip_ttl_tos->ipv4_tos_mask |= tos_word_mask->tos; + set_ip_ttl_tos->ipv4_tos &= ~tos_word_mask->tos; + set_ip_ttl_tos->ipv4_tos |= tos_word->tos & tos_word_mask->tos; + set_ip_ttl_tos->head.jump_id = + NFP_FL_ACTION_OPCODE_SET_IPV4_TTL_TOS; + set_ip_ttl_tos->head.len_lw = sizeof(*set_ip_ttl_tos) >> + NFP_FL_LW_SIZ; break; default: return -EOPNOTSUPP; } - set_ip_addr->reserved = cpu_to_be16(0); - set_ip_addr->head.jump_id = NFP_FL_ACTION_OPCODE_SET_IPV4_ADDRS; - set_ip_addr->head.len_lw = sizeof(*set_ip_addr) >> NFP_FL_LW_SIZ; - return 0; } @@ -432,12 +461,57 @@ nfp_fl_set_ip6_helper(int opcode_tag, u8 word, __be32 exact, __be32 mask, ip6->head.len_lw = sizeof(*ip6) >> NFP_FL_LW_SIZ; } +struct ipv6_hop_limit_word { + __be16 payload_len; + u8 nexthdr; + u8 hop_limit; +}; + +static int +nfp_fl_set_ip6_hop_limit_flow_label(u32 off, __be32 exact, __be32 mask, + struct nfp_fl_set_ipv6_tc_hl_fl *ip_hl_fl) +{ + struct ipv6_hop_limit_word *fl_hl_mask; + struct ipv6_hop_limit_word *fl_hl; + + switch (off) { + case offsetof(struct ipv6hdr, payload_len): + fl_hl_mask = (struct ipv6_hop_limit_word *)&mask; + fl_hl = (struct ipv6_hop_limit_word *)&exact; + + if (fl_hl_mask->nexthdr || fl_hl_mask->payload_len) + return -EOPNOTSUPP; + + ip_hl_fl->ipv6_hop_limit_mask |= fl_hl_mask->hop_limit; + ip_hl_fl->ipv6_hop_limit &= ~fl_hl_mask->hop_limit; + ip_hl_fl->ipv6_hop_limit |= fl_hl->hop_limit & + fl_hl_mask->hop_limit; + break; + case round_down(offsetof(struct ipv6hdr, flow_lbl), 4): + if (mask & ~IPV6_FLOW_LABEL_MASK || + exact & ~IPV6_FLOW_LABEL_MASK) + return -EOPNOTSUPP; + + ip_hl_fl->ipv6_label_mask |= mask; + ip_hl_fl->ipv6_label &= ~mask; + ip_hl_fl->ipv6_label |= exact & mask; + break; + } + + ip_hl_fl->head.jump_id = NFP_FL_ACTION_OPCODE_SET_IPV6_TC_HL_FL; + ip_hl_fl->head.len_lw = sizeof(*ip_hl_fl) >> NFP_FL_LW_SIZ; + + return 0; +} + static int nfp_fl_set_ip6(const struct tc_action *action, int idx, u32 off, struct nfp_fl_set_ipv6_addr *ip_dst, - struct nfp_fl_set_ipv6_addr *ip_src) + struct nfp_fl_set_ipv6_addr *ip_src, + struct nfp_fl_set_ipv6_tc_hl_fl *ip_hl_fl) { __be32 exact, mask; + int err = 0; u8 word; /* We are expecting tcf_pedit to return a big endian value */ @@ -448,7 +522,8 @@ nfp_fl_set_ip6(const struct tc_action *action, int idx, u32 off, return -EOPNOTSUPP; if (off < offsetof(struct ipv6hdr, saddr)) { - return -EOPNOTSUPP; + err = nfp_fl_set_ip6_hop_limit_flow_label(off, exact, mask, + ip_hl_fl); } else if (off < offsetof(struct ipv6hdr, daddr)) { word = (off - offsetof(struct ipv6hdr, saddr)) / sizeof(exact); nfp_fl_set_ip6_helper(NFP_FL_ACTION_OPCODE_SET_IPV6_SRC, word, @@ -462,7 +537,7 @@ nfp_fl_set_ip6(const struct tc_action *action, int idx, u32 off, return -EOPNOTSUPP; } - return 0; + return err; } static int @@ -513,6 +588,8 @@ nfp_fl_pedit(const struct tc_action *action, struct tc_cls_flower_offload *flow, char *nfp_action, int *a_len, u32 *csum_updated) { struct nfp_fl_set_ipv6_addr set_ip6_dst, set_ip6_src; + struct nfp_fl_set_ipv6_tc_hl_fl set_ip6_tc_hl_fl; + struct nfp_fl_set_ip4_ttl_tos set_ip_ttl_tos; struct nfp_fl_set_ip4_addrs set_ip_addr; struct nfp_fl_set_tport set_tport; struct nfp_fl_set_eth set_eth; @@ -522,6 +599,8 @@ nfp_fl_pedit(const struct tc_action *action, struct tc_cls_flower_offload *flow, u32 offset, cmd; u8 ip_proto = 0; + memset(&set_ip6_tc_hl_fl, 0, sizeof(set_ip6_tc_hl_fl)); + memset(&set_ip_ttl_tos, 0, sizeof(set_ip_ttl_tos)); memset(&set_ip6_dst, 0, sizeof(set_ip6_dst)); memset(&set_ip6_src, 0, sizeof(set_ip6_src)); memset(&set_ip_addr, 0, sizeof(set_ip_addr)); @@ -542,11 +621,12 @@ nfp_fl_pedit(const struct tc_action *action, struct tc_cls_flower_offload *flow, err = nfp_fl_set_eth(action, idx, offset, &set_eth); break; case TCA_PEDIT_KEY_EX_HDR_TYPE_IP4: - err = nfp_fl_set_ip4(action, idx, offset, &set_ip_addr); + err = nfp_fl_set_ip4(action, idx, offset, &set_ip_addr, + &set_ip_ttl_tos); break; case TCA_PEDIT_KEY_EX_HDR_TYPE_IP6: err = nfp_fl_set_ip6(action, idx, offset, &set_ip6_dst, - &set_ip6_src); + &set_ip6_src, &set_ip6_tc_hl_fl); break; case TCA_PEDIT_KEY_EX_HDR_TYPE_TCP: err = nfp_fl_set_tport(action, idx, offset, &set_tport, @@ -577,6 +657,16 @@ nfp_fl_pedit(const struct tc_action *action, struct tc_cls_flower_offload *flow, memcpy(nfp_action, &set_eth, act_size); *a_len += act_size; } + if (set_ip_ttl_tos.head.len_lw) { + nfp_action += act_size; + act_size = sizeof(set_ip_ttl_tos); + memcpy(nfp_action, &set_ip_ttl_tos, act_size); + *a_len += act_size; + + /* Hardware will automatically fix IPv4 and TCP/UDP checksum. */ + *csum_updated |= TCA_CSUM_UPDATE_FLAG_IPV4HDR | + nfp_fl_csum_l4_to_flag(ip_proto); + } if (set_ip_addr.head.len_lw) { nfp_action += act_size; act_size = sizeof(set_ip_addr); @@ -587,6 +677,15 @@ nfp_fl_pedit(const struct tc_action *action, struct tc_cls_flower_offload *flow, *csum_updated |= TCA_CSUM_UPDATE_FLAG_IPV4HDR | nfp_fl_csum_l4_to_flag(ip_proto); } + if (set_ip6_tc_hl_fl.head.len_lw) { + nfp_action += act_size; + act_size = sizeof(set_ip6_tc_hl_fl); + memcpy(nfp_action, &set_ip6_tc_hl_fl, act_size); + *a_len += act_size; + + /* Hardware will automatically fix TCP/UDP checksum. */ + *csum_updated |= nfp_fl_csum_l4_to_flag(ip_proto); + } if (set_ip6_dst.head.len_lw && set_ip6_src.head.len_lw) { /* TC compiles set src and dst IPv6 address as a single action, * the hardware requires this to be 2 separate actions. @@ -728,9 +827,8 @@ nfp_flower_loop_action(struct nfp_app *app, const struct tc_action *a, *a_len += sizeof(struct nfp_fl_push_vlan); } else if (is_tcf_tunnel_set(a)) { struct ip_tunnel_info *ip_tun = tcf_tunnel_info(a); - struct nfp_repr *repr = netdev_priv(netdev); - *tun_type = nfp_fl_get_tun_from_act_l4_port(repr->app, a); + *tun_type = nfp_fl_get_tun_from_act_l4_port(app, a); if (*tun_type == NFP_FL_TUNNEL_NONE) return -EOPNOTSUPP; diff --git a/drivers/net/ethernet/netronome/nfp/flower/cmsg.h b/drivers/net/ethernet/netronome/nfp/flower/cmsg.h index 29d673aa5277..15f41cfef9f1 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/cmsg.h +++ b/drivers/net/ethernet/netronome/nfp/flower/cmsg.h @@ -8,6 +8,7 @@ #include <linux/skbuff.h> #include <linux/types.h> #include <net/geneve.h> +#include <net/vxlan.h> #include "../nfp_app.h" #include "../nfpcore/nfp_cpp.h" @@ -65,8 +66,10 @@ #define NFP_FL_ACTION_OPCODE_SET_IPV4_TUNNEL 6 #define NFP_FL_ACTION_OPCODE_SET_ETHERNET 7 #define NFP_FL_ACTION_OPCODE_SET_IPV4_ADDRS 9 +#define NFP_FL_ACTION_OPCODE_SET_IPV4_TTL_TOS 10 #define NFP_FL_ACTION_OPCODE_SET_IPV6_SRC 11 #define NFP_FL_ACTION_OPCODE_SET_IPV6_DST 12 +#define NFP_FL_ACTION_OPCODE_SET_IPV6_TC_HL_FL 13 #define NFP_FL_ACTION_OPCODE_SET_UDP 14 #define NFP_FL_ACTION_OPCODE_SET_TCP 15 #define NFP_FL_ACTION_OPCODE_PRE_LAG 16 @@ -82,6 +85,8 @@ #define NFP_FL_PUSH_VLAN_CFI BIT(12) #define NFP_FL_PUSH_VLAN_VID GENMASK(11, 0) +#define IPV6_FLOW_LABEL_MASK cpu_to_be32(0x000fffff) + /* LAG ports */ #define NFP_FL_LAG_OUT 0xC0DE0000 @@ -125,6 +130,26 @@ struct nfp_fl_set_ip4_addrs { __be32 ipv4_dst; }; +struct nfp_fl_set_ip4_ttl_tos { + struct nfp_fl_act_head head; + u8 ipv4_ttl_mask; + u8 ipv4_tos_mask; + u8 ipv4_ttl; + u8 ipv4_tos; + __be16 reserved; +}; + +struct nfp_fl_set_ipv6_tc_hl_fl { + struct nfp_fl_act_head head; + u8 ipv6_tc_mask; + u8 ipv6_hop_limit_mask; + __be16 reserved; + u8 ipv6_tc; + u8 ipv6_hop_limit; + __be32 ipv6_label_mask; + __be32 ipv6_label; +}; + struct nfp_fl_set_ipv6_addr { struct nfp_fl_act_head head; __be16 reserved; @@ -475,6 +500,32 @@ static inline int nfp_flower_cmsg_get_data_len(struct sk_buff *skb) return skb->len - NFP_FLOWER_CMSG_HLEN; } +static inline bool +nfp_fl_netdev_is_tunnel_type(struct net_device *netdev, + enum nfp_flower_tun_type tun_type) +{ + if (netif_is_vxlan(netdev)) + return tun_type == NFP_FL_TUNNEL_VXLAN; + if (netif_is_geneve(netdev)) + return tun_type == NFP_FL_TUNNEL_GENEVE; + + return false; +} + +static inline bool nfp_fl_is_netdev_to_offload(struct net_device *netdev) +{ + if (!netdev->rtnl_link_ops) + return false; + if (!strcmp(netdev->rtnl_link_ops->kind, "openvswitch")) + return true; + if (netif_is_vxlan(netdev)) + return true; + if (netif_is_geneve(netdev)) + return true; + + return false; +} + struct sk_buff * nfp_flower_cmsg_mac_repr_start(struct nfp_app *app, unsigned int num_ports); void diff --git a/drivers/net/ethernet/netronome/nfp/flower/lag_conf.c b/drivers/net/ethernet/netronome/nfp/flower/lag_conf.c index 81dcf5b318ba..5db838f45694 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/lag_conf.c +++ b/drivers/net/ethernet/netronome/nfp/flower/lag_conf.c @@ -472,17 +472,25 @@ nfp_fl_lag_schedule_group_remove(struct nfp_fl_lag *lag, schedule_delayed_work(&lag->work, NFP_FL_LAG_DELAY); } -static int +static void nfp_fl_lag_schedule_group_delete(struct nfp_fl_lag *lag, struct net_device *master) { struct nfp_fl_lag_group *group; + struct nfp_flower_priv *priv; + + priv = container_of(lag, struct nfp_flower_priv, nfp_lag); + + if (!netif_is_bond_master(master)) + return; mutex_lock(&lag->lock); group = nfp_fl_lag_find_group_for_master_with_lag(lag, master); if (!group) { mutex_unlock(&lag->lock); - return -ENOENT; + nfp_warn(priv->app->cpp, "untracked bond got unregistered %s\n", + netdev_name(master)); + return; } group->to_remove = true; @@ -490,7 +498,6 @@ nfp_fl_lag_schedule_group_delete(struct nfp_fl_lag *lag, mutex_unlock(&lag->lock); schedule_delayed_work(&lag->work, NFP_FL_LAG_DELAY); - return 0; } static int @@ -575,7 +582,7 @@ nfp_fl_lag_changeupper_event(struct nfp_fl_lag *lag, return 0; } -static int +static void nfp_fl_lag_changels_event(struct nfp_fl_lag *lag, struct net_device *netdev, struct netdev_notifier_changelowerstate_info *info) { @@ -586,18 +593,18 @@ nfp_fl_lag_changels_event(struct nfp_fl_lag *lag, struct net_device *netdev, unsigned long *flags; if (!netif_is_lag_port(netdev) || !nfp_netdev_is_nfp_repr(netdev)) - return 0; + return; lag_lower_info = info->lower_state_info; if (!lag_lower_info) - return 0; + return; priv = container_of(lag, struct nfp_flower_priv, nfp_lag); repr = netdev_priv(netdev); /* Verify that the repr is associated with this app. */ if (repr->app != priv->app) - return 0; + return; repr_priv = repr->app_priv; flags = &repr_priv->lag_port_flags; @@ -617,20 +624,15 @@ nfp_fl_lag_changels_event(struct nfp_fl_lag *lag, struct net_device *netdev, mutex_unlock(&lag->lock); schedule_delayed_work(&lag->work, NFP_FL_LAG_DELAY); - return 0; } -static int -nfp_fl_lag_netdev_event(struct notifier_block *nb, unsigned long event, - void *ptr) +int nfp_flower_lag_netdev_event(struct nfp_flower_priv *priv, + struct net_device *netdev, + unsigned long event, void *ptr) { - struct net_device *netdev; - struct nfp_fl_lag *lag; + struct nfp_fl_lag *lag = &priv->nfp_lag; int err; - netdev = netdev_notifier_info_to_dev(ptr); - lag = container_of(nb, struct nfp_fl_lag, lag_nb); - switch (event) { case NETDEV_CHANGEUPPER: err = nfp_fl_lag_changeupper_event(lag, ptr); @@ -638,17 +640,11 @@ nfp_fl_lag_netdev_event(struct notifier_block *nb, unsigned long event, return NOTIFY_BAD; return NOTIFY_OK; case NETDEV_CHANGELOWERSTATE: - err = nfp_fl_lag_changels_event(lag, netdev, ptr); - if (err) - return NOTIFY_BAD; + nfp_fl_lag_changels_event(lag, netdev, ptr); return NOTIFY_OK; case NETDEV_UNREGISTER: - if (netif_is_bond_master(netdev)) { - err = nfp_fl_lag_schedule_group_delete(lag, netdev); - if (err) - return NOTIFY_BAD; - return NOTIFY_OK; - } + nfp_fl_lag_schedule_group_delete(lag, netdev); + return NOTIFY_OK; } return NOTIFY_DONE; @@ -673,8 +669,6 @@ void nfp_flower_lag_init(struct nfp_fl_lag *lag) /* 0 is a reserved batch version so increment to first valid value. */ nfp_fl_increment_version(lag); - - lag->lag_nb.notifier_call = nfp_fl_lag_netdev_event; } void nfp_flower_lag_cleanup(struct nfp_fl_lag *lag) diff --git a/drivers/net/ethernet/netronome/nfp/flower/main.c b/drivers/net/ethernet/netronome/nfp/flower/main.c index 3a54728d2ea6..5059110a1768 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/main.c +++ b/drivers/net/ethernet/netronome/nfp/flower/main.c @@ -146,23 +146,12 @@ nfp_flower_repr_netdev_stop(struct nfp_app *app, struct nfp_repr *repr) return nfp_flower_cmsg_portmod(repr, false, repr->netdev->mtu, false); } -static int -nfp_flower_repr_netdev_init(struct nfp_app *app, struct net_device *netdev) -{ - return tc_setup_cb_egdev_register(netdev, - nfp_flower_setup_tc_egress_cb, - netdev_priv(netdev)); -} - static void nfp_flower_repr_netdev_clean(struct nfp_app *app, struct net_device *netdev) { struct nfp_repr *repr = netdev_priv(netdev); kfree(repr->app_priv); - - tc_setup_cb_egdev_unregister(netdev, nfp_flower_setup_tc_egress_cb, - netdev_priv(netdev)); } static void @@ -568,6 +557,8 @@ static int nfp_flower_init(struct nfp_app *app) goto err_cleanup_metadata; } + INIT_LIST_HEAD(&app_priv->indr_block_cb_priv); + return 0; err_cleanup_metadata: @@ -661,10 +652,6 @@ static int nfp_flower_start(struct nfp_app *app) err = nfp_flower_lag_reset(&app_priv->nfp_lag); if (err) return err; - - err = register_netdevice_notifier(&app_priv->nfp_lag.lag_nb); - if (err) - return err; } return nfp_tunnel_config_start(app); @@ -672,12 +659,27 @@ static int nfp_flower_start(struct nfp_app *app) static void nfp_flower_stop(struct nfp_app *app) { + nfp_tunnel_config_stop(app); +} + +static int +nfp_flower_netdev_event(struct nfp_app *app, struct net_device *netdev, + unsigned long event, void *ptr) +{ struct nfp_flower_priv *app_priv = app->priv; + int ret; - if (app_priv->flower_ext_feats & NFP_FL_FEATS_LAG) - unregister_netdevice_notifier(&app_priv->nfp_lag.lag_nb); + if (app_priv->flower_ext_feats & NFP_FL_FEATS_LAG) { + ret = nfp_flower_lag_netdev_event(app_priv, netdev, event, ptr); + if (ret & NOTIFY_STOP_MASK) + return ret; + } - nfp_tunnel_config_stop(app); + ret = nfp_flower_reg_indir_block_handler(app, netdev, event); + if (ret & NOTIFY_STOP_MASK) + return ret; + + return nfp_tunnel_mac_event_handler(app, netdev, event, ptr); } const struct nfp_app_type app_flower = { @@ -698,7 +700,6 @@ const struct nfp_app_type app_flower = { .vnic_init = nfp_flower_vnic_init, .vnic_clean = nfp_flower_vnic_clean, - .repr_init = nfp_flower_repr_netdev_init, .repr_preclean = nfp_flower_repr_netdev_preclean, .repr_clean = nfp_flower_repr_netdev_clean, @@ -708,6 +709,8 @@ const struct nfp_app_type app_flower = { .start = nfp_flower_start, .stop = nfp_flower_stop, + .netdev_event = nfp_flower_netdev_event, + .ctrl_msg_rx = nfp_flower_cmsg_rx, .sriov_enable = nfp_flower_sriov_enable, diff --git a/drivers/net/ethernet/netronome/nfp/flower/main.h b/drivers/net/ethernet/netronome/nfp/flower/main.h index 90045bab95bf..b858bac47621 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/main.h +++ b/drivers/net/ethernet/netronome/nfp/flower/main.h @@ -20,7 +20,6 @@ struct nfp_fl_pre_lag; struct net_device; struct nfp_app; -#define NFP_FL_STATS_CTX_DONT_CARE cpu_to_be32(0xffffffff) #define NFP_FL_STATS_ELEM_RS FIELD_SIZEOF(struct nfp_fl_stats_id, \ init_unalloc) #define NFP_FLOWER_MASK_ENTRY_RS 256 @@ -72,7 +71,6 @@ struct nfp_mtu_conf { /** * struct nfp_fl_lag - Flower APP priv data for link aggregation - * @lag_nb: Notifier to track master/slave events * @work: Work queue for writing configs to the HW * @lock: Lock to protect lag_group_list * @group_list: List of all master/slave groups offloaded @@ -85,7 +83,6 @@ struct nfp_mtu_conf { * retransmission */ struct nfp_fl_lag { - struct notifier_block lag_nb; struct delayed_work work; struct mutex lock; struct list_head group_list; @@ -126,13 +123,13 @@ struct nfp_fl_lag { * @nfp_neigh_off_lock: Lock for the neighbour address list * @nfp_mac_off_ids: IDA to manage id assignment for offloaded macs * @nfp_mac_off_count: Number of MACs in address list - * @nfp_tun_mac_nb: Notifier to monitor link state * @nfp_tun_neigh_nb: Notifier to monitor neighbour state * @reify_replies: atomically stores the number of replies received * from firmware for repr reify * @reify_wait_queue: wait queue for repr reify response counting * @mtu_conf: Configuration of repr MTU value * @nfp_lag: Link aggregation data block + * @indr_block_cb_priv: List of priv data passed to indirect block cbs */ struct nfp_flower_priv { struct nfp_app *app; @@ -160,12 +157,12 @@ struct nfp_flower_priv { spinlock_t nfp_neigh_off_lock; struct ida nfp_mac_off_ids; int nfp_mac_off_count; - struct notifier_block nfp_tun_mac_nb; struct notifier_block nfp_tun_neigh_nb; atomic_t reify_replies; wait_queue_head_t reify_wait_queue; struct nfp_mtu_conf mtu_conf; struct nfp_fl_lag nfp_lag; + struct list_head indr_block_cb_priv; }; /** @@ -209,7 +206,6 @@ struct nfp_fl_payload { char *unmasked_data; char *mask_data; char *action_data; - bool ingress_offload; }; extern const struct rhashtable_params nfp_flower_table_params; @@ -226,7 +222,8 @@ void nfp_flower_metadata_cleanup(struct nfp_app *app); int nfp_flower_setup_tc(struct nfp_app *app, struct net_device *netdev, enum tc_setup_type type, void *type_data); -int nfp_flower_compile_flow_match(struct tc_cls_flower_offload *flow, +int nfp_flower_compile_flow_match(struct nfp_app *app, + struct tc_cls_flower_offload *flow, struct nfp_fl_key_ls *key_ls, struct net_device *netdev, struct nfp_fl_payload *nfp_flow, @@ -244,7 +241,7 @@ int nfp_modify_flow_metadata(struct nfp_app *app, struct nfp_fl_payload * nfp_flower_search_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie, - struct net_device *netdev, __be32 host_ctx); + struct net_device *netdev); struct nfp_fl_payload * nfp_flower_remove_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie); @@ -252,21 +249,28 @@ void nfp_flower_rx_flow_stats(struct nfp_app *app, struct sk_buff *skb); int nfp_tunnel_config_start(struct nfp_app *app); void nfp_tunnel_config_stop(struct nfp_app *app); +int nfp_tunnel_mac_event_handler(struct nfp_app *app, + struct net_device *netdev, + unsigned long event, void *ptr); void nfp_tunnel_write_macs(struct nfp_app *app); void nfp_tunnel_del_ipv4_off(struct nfp_app *app, __be32 ipv4); void nfp_tunnel_add_ipv4_off(struct nfp_app *app, __be32 ipv4); void nfp_tunnel_request_route(struct nfp_app *app, struct sk_buff *skb); void nfp_tunnel_keep_alive(struct nfp_app *app, struct sk_buff *skb); -int nfp_flower_setup_tc_egress_cb(enum tc_setup_type type, void *type_data, - void *cb_priv); void nfp_flower_lag_init(struct nfp_fl_lag *lag); void nfp_flower_lag_cleanup(struct nfp_fl_lag *lag); int nfp_flower_lag_reset(struct nfp_fl_lag *lag); +int nfp_flower_lag_netdev_event(struct nfp_flower_priv *priv, + struct net_device *netdev, + unsigned long event, void *ptr); bool nfp_flower_lag_unprocessed_msg(struct nfp_app *app, struct sk_buff *skb); int nfp_flower_lag_populate_pre_action(struct nfp_app *app, struct net_device *master, struct nfp_fl_pre_lag *pre_act); int nfp_flower_lag_get_output_id(struct nfp_app *app, struct net_device *master); +int nfp_flower_reg_indir_block_handler(struct nfp_app *app, + struct net_device *netdev, + unsigned long event); #endif diff --git a/drivers/net/ethernet/netronome/nfp/flower/match.c b/drivers/net/ethernet/netronome/nfp/flower/match.c index e54fb6034326..cdf75595f627 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/match.c +++ b/drivers/net/ethernet/netronome/nfp/flower/match.c @@ -52,10 +52,13 @@ nfp_flower_compile_port(struct nfp_flower_in_port *frame, u32 cmsg_port, return 0; } - if (tun_type) + if (tun_type) { frame->in_port = cpu_to_be32(NFP_FL_PORT_TYPE_TUN | tun_type); - else + } else { + if (!cmsg_port) + return -EOPNOTSUPP; frame->in_port = cpu_to_be32(cmsg_port); + } return 0; } @@ -289,17 +292,21 @@ nfp_flower_compile_ipv4_udp_tun(struct nfp_flower_ipv4_udp_tun *frame, } } -int nfp_flower_compile_flow_match(struct tc_cls_flower_offload *flow, +int nfp_flower_compile_flow_match(struct nfp_app *app, + struct tc_cls_flower_offload *flow, struct nfp_fl_key_ls *key_ls, struct net_device *netdev, struct nfp_fl_payload *nfp_flow, enum nfp_flower_tun_type tun_type) { - struct nfp_repr *netdev_repr; + u32 cmsg_port = 0; int err; u8 *ext; u8 *msk; + if (nfp_netdev_is_nfp_repr(netdev)) + cmsg_port = nfp_repr_get_port_id(netdev); + memset(nfp_flow->unmasked_data, 0, key_ls->key_size); memset(nfp_flow->mask_data, 0, key_ls->key_size); @@ -327,15 +334,13 @@ int nfp_flower_compile_flow_match(struct tc_cls_flower_offload *flow, /* Populate Exact Port data. */ err = nfp_flower_compile_port((struct nfp_flower_in_port *)ext, - nfp_repr_get_port_id(netdev), - false, tun_type); + cmsg_port, false, tun_type); if (err) return err; /* Populate Mask Port Data. */ err = nfp_flower_compile_port((struct nfp_flower_in_port *)msk, - nfp_repr_get_port_id(netdev), - true, tun_type); + cmsg_port, true, tun_type); if (err) return err; @@ -399,16 +404,13 @@ int nfp_flower_compile_flow_match(struct tc_cls_flower_offload *flow, msk += sizeof(struct nfp_flower_ipv4_udp_tun); /* Configure tunnel end point MAC. */ - if (nfp_netdev_is_nfp_repr(netdev)) { - netdev_repr = netdev_priv(netdev); - nfp_tunnel_write_macs(netdev_repr->app); - - /* Store the tunnel destination in the rule data. - * This must be present and be an exact match. - */ - nfp_flow->nfp_tun_ipv4_addr = tun_dst; - nfp_tunnel_add_ipv4_off(netdev_repr->app, tun_dst); - } + nfp_tunnel_write_macs(app); + + /* Store the tunnel destination in the rule data. + * This must be present and be an exact match. + */ + nfp_flow->nfp_tun_ipv4_addr = tun_dst; + nfp_tunnel_add_ipv4_off(app, tun_dst); if (key_ls->key_layer_two & NFP_FLOWER_LAYER2_GENEVE_OP) { err = nfp_flower_compile_geneve_opt(ext, flow, false); diff --git a/drivers/net/ethernet/netronome/nfp/flower/metadata.c b/drivers/net/ethernet/netronome/nfp/flower/metadata.c index 48729bf171e0..573a4400a26c 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/metadata.c +++ b/drivers/net/ethernet/netronome/nfp/flower/metadata.c @@ -21,7 +21,6 @@ struct nfp_mask_id_table { struct nfp_fl_flow_table_cmp_arg { struct net_device *netdev; unsigned long cookie; - __be32 host_ctx; }; static int nfp_release_stats_entry(struct nfp_app *app, u32 stats_context_id) @@ -76,14 +75,13 @@ static int nfp_get_stats_entry(struct nfp_app *app, u32 *stats_context_id) /* Must be called with either RTNL or rcu_read_lock */ struct nfp_fl_payload * nfp_flower_search_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie, - struct net_device *netdev, __be32 host_ctx) + struct net_device *netdev) { struct nfp_fl_flow_table_cmp_arg flower_cmp_arg; struct nfp_flower_priv *priv = app->priv; flower_cmp_arg.netdev = netdev; flower_cmp_arg.cookie = tc_flower_cookie; - flower_cmp_arg.host_ctx = host_ctx; return rhashtable_lookup_fast(&priv->flow_table, &flower_cmp_arg, nfp_flower_table_params); @@ -287,6 +285,7 @@ int nfp_compile_flow_metadata(struct nfp_app *app, nfp_flow->meta.host_ctx_id = cpu_to_be32(stats_cxt); nfp_flow->meta.host_cookie = cpu_to_be64(flow->cookie); + nfp_flow->ingress_dev = netdev; new_mask_id = 0; if (!nfp_check_mask_add(app, nfp_flow->mask_data, @@ -306,8 +305,7 @@ int nfp_compile_flow_metadata(struct nfp_app *app, priv->stats[stats_cxt].bytes = 0; priv->stats[stats_cxt].used = jiffies; - check_entry = nfp_flower_search_fl_table(app, flow->cookie, netdev, - NFP_FL_STATS_CTX_DONT_CARE); + check_entry = nfp_flower_search_fl_table(app, flow->cookie, netdev); if (check_entry) { if (nfp_release_stats_entry(app, stats_cxt)) return -EINVAL; @@ -352,9 +350,7 @@ static int nfp_fl_obj_cmpfn(struct rhashtable_compare_arg *arg, const struct nfp_fl_flow_table_cmp_arg *cmp_arg = arg->key; const struct nfp_fl_payload *flow_entry = obj; - if ((!cmp_arg->netdev || flow_entry->ingress_dev == cmp_arg->netdev) && - (cmp_arg->host_ctx == NFP_FL_STATS_CTX_DONT_CARE || - flow_entry->meta.host_ctx_id == cmp_arg->host_ctx)) + if (flow_entry->ingress_dev == cmp_arg->netdev) return flow_entry->tc_flower_cookie != cmp_arg->cookie; return 1; diff --git a/drivers/net/ethernet/netronome/nfp/flower/offload.c b/drivers/net/ethernet/netronome/nfp/flower/offload.c index 29c95423ab64..545d94168874 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/offload.c +++ b/drivers/net/ethernet/netronome/nfp/flower/offload.c @@ -56,11 +56,10 @@ BIT(FLOW_DISSECTOR_KEY_ENC_PORTS)) static int -nfp_flower_xmit_flow(struct net_device *netdev, - struct nfp_fl_payload *nfp_flow, u8 mtype) +nfp_flower_xmit_flow(struct nfp_app *app, struct nfp_fl_payload *nfp_flow, + u8 mtype) { u32 meta_len, key_len, mask_len, act_len, tot_len; - struct nfp_repr *priv = netdev_priv(netdev); struct sk_buff *skb; unsigned char *msg; @@ -78,7 +77,7 @@ nfp_flower_xmit_flow(struct net_device *netdev, nfp_flow->meta.mask_len >>= NFP_FL_LW_SIZ; nfp_flow->meta.act_len >>= NFP_FL_LW_SIZ; - skb = nfp_flower_cmsg_alloc(priv->app, tot_len, mtype, GFP_KERNEL); + skb = nfp_flower_cmsg_alloc(app, tot_len, mtype, GFP_KERNEL); if (!skb) return -ENOMEM; @@ -96,7 +95,7 @@ nfp_flower_xmit_flow(struct net_device *netdev, nfp_flow->meta.mask_len <<= NFP_FL_LW_SIZ; nfp_flow->meta.act_len <<= NFP_FL_LW_SIZ; - nfp_ctrl_tx(priv->app->ctrl, skb); + nfp_ctrl_tx(app->ctrl, skb); return 0; } @@ -129,9 +128,9 @@ nfp_flower_calc_opt_layer(struct flow_dissector_key_enc_opts *enc_opts, static int nfp_flower_calculate_key_layers(struct nfp_app *app, + struct net_device *netdev, struct nfp_fl_key_ls *ret_key_ls, struct tc_cls_flower_offload *flow, - bool egress, enum nfp_flower_tun_type *tun_type) { struct flow_dissector_key_basic *mask_basic = NULL; @@ -187,8 +186,6 @@ nfp_flower_calculate_key_layers(struct nfp_app *app, skb_flow_dissector_target(flow->dissector, FLOW_DISSECTOR_KEY_ENC_CONTROL, flow->key); - if (!egress) - return -EOPNOTSUPP; if (mask_enc_ctl->addr_type != 0xffff || enc_ctl->addr_type != FLOW_DISSECTOR_KEY_IPV4_ADDRS) @@ -251,9 +248,10 @@ nfp_flower_calculate_key_layers(struct nfp_app *app, default: return -EOPNOTSUPP; } - } else if (egress) { - /* Reject non tunnel matches offloaded to egress repr. */ - return -EOPNOTSUPP; + + /* Ensure the ingress netdev matches the expected tun type. */ + if (!nfp_fl_netdev_is_tunnel_type(netdev, *tun_type)) + return -EOPNOTSUPP; } if (dissector_uses_key(flow->dissector, FLOW_DISSECTOR_KEY_BASIC)) { @@ -374,7 +372,7 @@ nfp_flower_calculate_key_layers(struct nfp_app *app, } static struct nfp_fl_payload * -nfp_flower_allocate_new(struct nfp_fl_key_ls *key_layer, bool egress) +nfp_flower_allocate_new(struct nfp_fl_key_ls *key_layer) { struct nfp_fl_payload *flow_pay; @@ -398,7 +396,6 @@ nfp_flower_allocate_new(struct nfp_fl_key_ls *key_layer, bool egress) flow_pay->nfp_tun_ipv4_addr = 0; flow_pay->meta.flags = 0; - flow_pay->ingress_offload = !egress; return flow_pay; @@ -416,7 +413,6 @@ err_free_flow: * @app: Pointer to the APP handle * @netdev: netdev structure. * @flow: TC flower classifier offload structure. - * @egress: NFP netdev is the egress. * * Adds a new flow to the repeated hash structure and action payload. * @@ -424,46 +420,35 @@ err_free_flow: */ static int nfp_flower_add_offload(struct nfp_app *app, struct net_device *netdev, - struct tc_cls_flower_offload *flow, bool egress) + struct tc_cls_flower_offload *flow) { enum nfp_flower_tun_type tun_type = NFP_FL_TUNNEL_NONE; - struct nfp_port *port = nfp_port_from_netdev(netdev); struct nfp_flower_priv *priv = app->priv; struct nfp_fl_payload *flow_pay; struct nfp_fl_key_ls *key_layer; - struct net_device *ingr_dev; + struct nfp_port *port = NULL; int err; - ingr_dev = egress ? NULL : netdev; - flow_pay = nfp_flower_search_fl_table(app, flow->cookie, ingr_dev, - NFP_FL_STATS_CTX_DONT_CARE); - if (flow_pay) { - /* Ignore as duplicate if it has been added by different cb. */ - if (flow_pay->ingress_offload && egress) - return 0; - else - return -EOPNOTSUPP; - } + if (nfp_netdev_is_nfp_repr(netdev)) + port = nfp_port_from_netdev(netdev); key_layer = kmalloc(sizeof(*key_layer), GFP_KERNEL); if (!key_layer) return -ENOMEM; - err = nfp_flower_calculate_key_layers(app, key_layer, flow, egress, + err = nfp_flower_calculate_key_layers(app, netdev, key_layer, flow, &tun_type); if (err) goto err_free_key_ls; - flow_pay = nfp_flower_allocate_new(key_layer, egress); + flow_pay = nfp_flower_allocate_new(key_layer); if (!flow_pay) { err = -ENOMEM; goto err_free_key_ls; } - flow_pay->ingress_dev = egress ? NULL : netdev; - - err = nfp_flower_compile_flow_match(flow, key_layer, netdev, flow_pay, - tun_type); + err = nfp_flower_compile_flow_match(app, flow, key_layer, netdev, + flow_pay, tun_type); if (err) goto err_destroy_flow; @@ -471,12 +456,11 @@ nfp_flower_add_offload(struct nfp_app *app, struct net_device *netdev, if (err) goto err_destroy_flow; - err = nfp_compile_flow_metadata(app, flow, flow_pay, - flow_pay->ingress_dev); + err = nfp_compile_flow_metadata(app, flow, flow_pay, netdev); if (err) goto err_destroy_flow; - err = nfp_flower_xmit_flow(netdev, flow_pay, + err = nfp_flower_xmit_flow(app, flow_pay, NFP_FLOWER_CMSG_TYPE_FLOW_ADD); if (err) goto err_destroy_flow; @@ -487,7 +471,8 @@ nfp_flower_add_offload(struct nfp_app *app, struct net_device *netdev, if (err) goto err_destroy_flow; - port->tc_offload_cnt++; + if (port) + port->tc_offload_cnt++; /* Deallocate flow payload when flower rule has been destroyed. */ kfree(key_layer); @@ -509,7 +494,6 @@ err_free_key_ls: * @app: Pointer to the APP handle * @netdev: netdev structure. * @flow: TC flower classifier offload structure - * @egress: Netdev is the egress dev. * * Removes a flow from the repeated hash structure and clears the * action payload. @@ -518,19 +502,19 @@ err_free_key_ls: */ static int nfp_flower_del_offload(struct nfp_app *app, struct net_device *netdev, - struct tc_cls_flower_offload *flow, bool egress) + struct tc_cls_flower_offload *flow) { - struct nfp_port *port = nfp_port_from_netdev(netdev); struct nfp_flower_priv *priv = app->priv; struct nfp_fl_payload *nfp_flow; - struct net_device *ingr_dev; + struct nfp_port *port = NULL; int err; - ingr_dev = egress ? NULL : netdev; - nfp_flow = nfp_flower_search_fl_table(app, flow->cookie, ingr_dev, - NFP_FL_STATS_CTX_DONT_CARE); + if (nfp_netdev_is_nfp_repr(netdev)) + port = nfp_port_from_netdev(netdev); + + nfp_flow = nfp_flower_search_fl_table(app, flow->cookie, netdev); if (!nfp_flow) - return egress ? 0 : -ENOENT; + return -ENOENT; err = nfp_modify_flow_metadata(app, nfp_flow); if (err) @@ -539,13 +523,14 @@ nfp_flower_del_offload(struct nfp_app *app, struct net_device *netdev, if (nfp_flow->nfp_tun_ipv4_addr) nfp_tunnel_del_ipv4_off(app, nfp_flow->nfp_tun_ipv4_addr); - err = nfp_flower_xmit_flow(netdev, nfp_flow, + err = nfp_flower_xmit_flow(app, nfp_flow, NFP_FLOWER_CMSG_TYPE_FLOW_DEL); if (err) goto err_free_flow; err_free_flow: - port->tc_offload_cnt--; + if (port) + port->tc_offload_cnt--; kfree(nfp_flow->action_data); kfree(nfp_flow->mask_data); kfree(nfp_flow->unmasked_data); @@ -561,7 +546,6 @@ err_free_flow: * @app: Pointer to the APP handle * @netdev: Netdev structure. * @flow: TC flower classifier offload structure - * @egress: Netdev is the egress dev. * * Populates a flow statistics structure which which corresponds to a * specific flow. @@ -570,22 +554,16 @@ err_free_flow: */ static int nfp_flower_get_stats(struct nfp_app *app, struct net_device *netdev, - struct tc_cls_flower_offload *flow, bool egress) + struct tc_cls_flower_offload *flow) { struct nfp_flower_priv *priv = app->priv; struct nfp_fl_payload *nfp_flow; - struct net_device *ingr_dev; u32 ctx_id; - ingr_dev = egress ? NULL : netdev; - nfp_flow = nfp_flower_search_fl_table(app, flow->cookie, ingr_dev, - NFP_FL_STATS_CTX_DONT_CARE); + nfp_flow = nfp_flower_search_fl_table(app, flow->cookie, netdev); if (!nfp_flow) return -EINVAL; - if (nfp_flow->ingress_offload && egress) - return 0; - ctx_id = be32_to_cpu(nfp_flow->meta.host_ctx_id); spin_lock_bh(&priv->stats_lock); @@ -602,35 +580,18 @@ nfp_flower_get_stats(struct nfp_app *app, struct net_device *netdev, static int nfp_flower_repr_offload(struct nfp_app *app, struct net_device *netdev, - struct tc_cls_flower_offload *flower, bool egress) + struct tc_cls_flower_offload *flower) { if (!eth_proto_is_802_3(flower->common.protocol)) return -EOPNOTSUPP; switch (flower->command) { case TC_CLSFLOWER_REPLACE: - return nfp_flower_add_offload(app, netdev, flower, egress); + return nfp_flower_add_offload(app, netdev, flower); case TC_CLSFLOWER_DESTROY: - return nfp_flower_del_offload(app, netdev, flower, egress); + return nfp_flower_del_offload(app, netdev, flower); case TC_CLSFLOWER_STATS: - return nfp_flower_get_stats(app, netdev, flower, egress); - default: - return -EOPNOTSUPP; - } -} - -int nfp_flower_setup_tc_egress_cb(enum tc_setup_type type, void *type_data, - void *cb_priv) -{ - struct nfp_repr *repr = cb_priv; - - if (!tc_cls_can_offload_and_chain0(repr->netdev, type_data)) - return -EOPNOTSUPP; - - switch (type) { - case TC_SETUP_CLSFLOWER: - return nfp_flower_repr_offload(repr->app, repr->netdev, - type_data, true); + return nfp_flower_get_stats(app, netdev, flower); default: return -EOPNOTSUPP; } @@ -647,7 +608,7 @@ static int nfp_flower_setup_tc_block_cb(enum tc_setup_type type, switch (type) { case TC_SETUP_CLSFLOWER: return nfp_flower_repr_offload(repr->app, repr->netdev, - type_data, false); + type_data); default: return -EOPNOTSUPP; } @@ -686,3 +647,129 @@ int nfp_flower_setup_tc(struct nfp_app *app, struct net_device *netdev, return -EOPNOTSUPP; } } + +struct nfp_flower_indr_block_cb_priv { + struct net_device *netdev; + struct nfp_app *app; + struct list_head list; +}; + +static struct nfp_flower_indr_block_cb_priv * +nfp_flower_indr_block_cb_priv_lookup(struct nfp_app *app, + struct net_device *netdev) +{ + struct nfp_flower_indr_block_cb_priv *cb_priv; + struct nfp_flower_priv *priv = app->priv; + + /* All callback list access should be protected by RTNL. */ + ASSERT_RTNL(); + + list_for_each_entry(cb_priv, &priv->indr_block_cb_priv, list) + if (cb_priv->netdev == netdev) + return cb_priv; + + return NULL; +} + +static int nfp_flower_setup_indr_block_cb(enum tc_setup_type type, + void *type_data, void *cb_priv) +{ + struct nfp_flower_indr_block_cb_priv *priv = cb_priv; + struct tc_cls_flower_offload *flower = type_data; + + if (flower->common.chain_index) + return -EOPNOTSUPP; + + switch (type) { + case TC_SETUP_CLSFLOWER: + return nfp_flower_repr_offload(priv->app, priv->netdev, + type_data); + default: + return -EOPNOTSUPP; + } +} + +static int +nfp_flower_setup_indr_tc_block(struct net_device *netdev, struct nfp_app *app, + struct tc_block_offload *f) +{ + struct nfp_flower_indr_block_cb_priv *cb_priv; + struct nfp_flower_priv *priv = app->priv; + int err; + + if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) + return -EOPNOTSUPP; + + switch (f->command) { + case TC_BLOCK_BIND: + cb_priv = kmalloc(sizeof(*cb_priv), GFP_KERNEL); + if (!cb_priv) + return -ENOMEM; + + cb_priv->netdev = netdev; + cb_priv->app = app; + list_add(&cb_priv->list, &priv->indr_block_cb_priv); + + err = tcf_block_cb_register(f->block, + nfp_flower_setup_indr_block_cb, + netdev, cb_priv, f->extack); + if (err) { + list_del(&cb_priv->list); + kfree(cb_priv); + } + + return err; + case TC_BLOCK_UNBIND: + tcf_block_cb_unregister(f->block, + nfp_flower_setup_indr_block_cb, netdev); + cb_priv = nfp_flower_indr_block_cb_priv_lookup(app, netdev); + if (cb_priv) { + list_del(&cb_priv->list); + kfree(cb_priv); + } + + return 0; + default: + return -EOPNOTSUPP; + } + return 0; +} + +static int +nfp_flower_indr_setup_tc_cb(struct net_device *netdev, void *cb_priv, + enum tc_setup_type type, void *type_data) +{ + switch (type) { + case TC_SETUP_BLOCK: + return nfp_flower_setup_indr_tc_block(netdev, cb_priv, + type_data); + default: + return -EOPNOTSUPP; + } +} + +int nfp_flower_reg_indir_block_handler(struct nfp_app *app, + struct net_device *netdev, + unsigned long event) +{ + int err; + + if (!nfp_fl_is_netdev_to_offload(netdev)) + return NOTIFY_OK; + + if (event == NETDEV_REGISTER) { + err = __tc_indr_block_cb_register(netdev, app, + nfp_flower_indr_setup_tc_cb, + netdev); + if (err) + nfp_flower_cmsg_warn(app, + "Indirect block reg failed - %s\n", + netdev->name); + } else if (event == NETDEV_UNREGISTER) { + __tc_indr_block_cb_unregister(netdev, + nfp_flower_indr_setup_tc_cb, + netdev); + } + + return NOTIFY_OK; +} diff --git a/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c b/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c index 8e5bec04d1f9..2d9f26a725c2 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c +++ b/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c @@ -4,7 +4,6 @@ #include <linux/etherdevice.h> #include <linux/inetdevice.h> #include <net/netevent.h> -#include <net/vxlan.h> #include <linux/idr.h> #include <net/dst_metadata.h> #include <net/arp.h> @@ -182,18 +181,6 @@ void nfp_tunnel_keep_alive(struct nfp_app *app, struct sk_buff *skb) } } -static bool nfp_tun_is_netdev_to_offload(struct net_device *netdev) -{ - if (!netdev->rtnl_link_ops) - return false; - if (!strcmp(netdev->rtnl_link_ops->kind, "openvswitch")) - return true; - if (netif_is_vxlan(netdev)) - return true; - - return false; -} - static int nfp_flower_xmit_tun_conf(struct nfp_app *app, u8 mtype, u16 plen, void *pdata, gfp_t flag) @@ -615,7 +602,7 @@ static void nfp_tun_add_to_mac_offload_list(struct net_device *netdev, if (nfp_netdev_is_nfp_repr(netdev)) port = nfp_repr_get_port_id(netdev); - else if (!nfp_tun_is_netdev_to_offload(netdev)) + else if (!nfp_fl_is_netdev_to_offload(netdev)) return; entry = kmalloc(sizeof(*entry), GFP_KERNEL); @@ -652,29 +639,16 @@ static void nfp_tun_add_to_mac_offload_list(struct net_device *netdev, mutex_unlock(&priv->nfp_mac_off_lock); } -static int nfp_tun_mac_event_handler(struct notifier_block *nb, - unsigned long event, void *ptr) +int nfp_tunnel_mac_event_handler(struct nfp_app *app, + struct net_device *netdev, + unsigned long event, void *ptr) { - struct nfp_flower_priv *app_priv; - struct net_device *netdev; - struct nfp_app *app; - if (event == NETDEV_DOWN || event == NETDEV_UNREGISTER) { - app_priv = container_of(nb, struct nfp_flower_priv, - nfp_tun_mac_nb); - app = app_priv->app; - netdev = netdev_notifier_info_to_dev(ptr); - /* If non-nfp netdev then free its offload index. */ - if (nfp_tun_is_netdev_to_offload(netdev)) + if (nfp_fl_is_netdev_to_offload(netdev)) nfp_tun_del_mac_idx(app, netdev->ifindex); } else if (event == NETDEV_UP || event == NETDEV_CHANGEADDR || event == NETDEV_REGISTER) { - app_priv = container_of(nb, struct nfp_flower_priv, - nfp_tun_mac_nb); - app = app_priv->app; - netdev = netdev_notifier_info_to_dev(ptr); - nfp_tun_add_to_mac_offload_list(netdev, app); /* Force a list write to keep NFP up to date. */ @@ -686,14 +660,11 @@ static int nfp_tun_mac_event_handler(struct notifier_block *nb, int nfp_tunnel_config_start(struct nfp_app *app) { struct nfp_flower_priv *priv = app->priv; - struct net_device *netdev; - int err; /* Initialise priv data for MAC offloading. */ priv->nfp_mac_off_count = 0; mutex_init(&priv->nfp_mac_off_lock); INIT_LIST_HEAD(&priv->nfp_mac_off_list); - priv->nfp_tun_mac_nb.notifier_call = nfp_tun_mac_event_handler; mutex_init(&priv->nfp_mac_index_lock); INIT_LIST_HEAD(&priv->nfp_mac_index_list); ida_init(&priv->nfp_mac_off_ids); @@ -707,27 +678,7 @@ int nfp_tunnel_config_start(struct nfp_app *app) INIT_LIST_HEAD(&priv->nfp_neigh_off_list); priv->nfp_tun_neigh_nb.notifier_call = nfp_tun_neigh_event_handler; - err = register_netdevice_notifier(&priv->nfp_tun_mac_nb); - if (err) - goto err_free_mac_ida; - - err = register_netevent_notifier(&priv->nfp_tun_neigh_nb); - if (err) - goto err_unreg_mac_nb; - - /* Parse netdevs already registered for MACs that need offloaded. */ - rtnl_lock(); - for_each_netdev(&init_net, netdev) - nfp_tun_add_to_mac_offload_list(netdev, app); - rtnl_unlock(); - - return 0; - -err_unreg_mac_nb: - unregister_netdevice_notifier(&priv->nfp_tun_mac_nb); -err_free_mac_ida: - ida_destroy(&priv->nfp_mac_off_ids); - return err; + return register_netevent_notifier(&priv->nfp_tun_neigh_nb); } void nfp_tunnel_config_stop(struct nfp_app *app) @@ -739,7 +690,6 @@ void nfp_tunnel_config_stop(struct nfp_app *app) struct nfp_ipv4_addr_entry *ip_entry; struct list_head *ptr, *storage; - unregister_netdevice_notifier(&priv->nfp_tun_mac_nb); unregister_netevent_notifier(&priv->nfp_tun_neigh_nb); /* Free any memory that may be occupied by MAC list. */ diff --git a/drivers/net/ethernet/netronome/nfp/nfp_app.c b/drivers/net/ethernet/netronome/nfp/nfp_app.c index 68a0991aac22..4a1b8f79e731 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_app.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_app.c @@ -136,6 +136,53 @@ nfp_app_reprs_set(struct nfp_app *app, enum nfp_repr_type type, return old; } +static int +nfp_app_netdev_event(struct notifier_block *nb, unsigned long event, void *ptr) +{ + struct net_device *netdev; + struct nfp_app *app; + + netdev = netdev_notifier_info_to_dev(ptr); + app = container_of(nb, struct nfp_app, netdev_nb); + + if (app->type->netdev_event) + return app->type->netdev_event(app, netdev, event, ptr); + return NOTIFY_DONE; +} + +int nfp_app_start(struct nfp_app *app, struct nfp_net *ctrl) +{ + int err; + + app->ctrl = ctrl; + + if (app->type->start) { + err = app->type->start(app); + if (err) + return err; + } + + app->netdev_nb.notifier_call = nfp_app_netdev_event; + err = register_netdevice_notifier(&app->netdev_nb); + if (err) + goto err_app_stop; + + return 0; + +err_app_stop: + if (app->type->stop) + app->type->stop(app); + return err; +} + +void nfp_app_stop(struct nfp_app *app) +{ + unregister_netdevice_notifier(&app->netdev_nb); + + if (app->type->stop) + app->type->stop(app); +} + struct nfp_app *nfp_app_alloc(struct nfp_pf *pf, enum nfp_app_id id) { struct nfp_app *app; diff --git a/drivers/net/ethernet/netronome/nfp/nfp_app.h b/drivers/net/ethernet/netronome/nfp/nfp_app.h index 4d6ecf99b1cc..d578d856a009 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_app.h +++ b/drivers/net/ethernet/netronome/nfp/nfp_app.h @@ -69,6 +69,7 @@ extern const struct nfp_app_type app_abm; * @port_get_stats_strings: get strings for extra statistics * @start: start application logic * @stop: stop application logic + * @netdev_event: Netdevice notifier event * @ctrl_msg_rx: control message handler * @ctrl_msg_rx_raw: handler for control messages from data queues * @setup_tc: setup TC ndo @@ -122,6 +123,9 @@ struct nfp_app_type { int (*start)(struct nfp_app *app); void (*stop)(struct nfp_app *app); + int (*netdev_event)(struct nfp_app *app, struct net_device *netdev, + unsigned long event, void *ptr); + void (*ctrl_msg_rx)(struct nfp_app *app, struct sk_buff *skb); void (*ctrl_msg_rx_raw)(struct nfp_app *app, const void *data, unsigned int len); @@ -151,6 +155,7 @@ struct nfp_app_type { * @reprs: array of pointers to representors * @type: pointer to const application ops and info * @ctrl_mtu: MTU to set on the control vNIC (set in .init()) + * @netdev_nb: Netdevice notifier block * @priv: app-specific priv data */ struct nfp_app { @@ -163,6 +168,9 @@ struct nfp_app { const struct nfp_app_type *type; unsigned int ctrl_mtu; + + struct notifier_block netdev_nb; + void *priv; }; @@ -264,21 +272,6 @@ nfp_app_repr_change_mtu(struct nfp_app *app, struct net_device *netdev, return app->type->repr_change_mtu(app, netdev, new_mtu); } -static inline int nfp_app_start(struct nfp_app *app, struct nfp_net *ctrl) -{ - app->ctrl = ctrl; - if (!app->type->start) - return 0; - return app->type->start(app); -} - -static inline void nfp_app_stop(struct nfp_app *app) -{ - if (!app->type->stop) - return; - app->type->stop(app); -} - static inline const char *nfp_app_name(struct nfp_app *app) { if (!app) @@ -430,6 +423,8 @@ nfp_app_ctrl_msg_alloc(struct nfp_app *app, unsigned int size, gfp_t priority); struct nfp_app *nfp_app_alloc(struct nfp_pf *pf, enum nfp_app_id id); void nfp_app_free(struct nfp_app *app); +int nfp_app_start(struct nfp_app *app, struct nfp_net *ctrl); +void nfp_app_stop(struct nfp_app *app); /* Callbacks shared between apps */ diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net.h b/drivers/net/ethernet/netronome/nfp/nfp_net.h index 6f0c37d09256..dda02fefc806 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net.h +++ b/drivers/net/ethernet/netronome/nfp/nfp_net.h @@ -851,7 +851,7 @@ void nfp_net_get_fw_version(struct nfp_net_fw_version *fw_ver, void __iomem *ctrl_bar); struct nfp_net * -nfp_net_alloc(struct pci_dev *pdev, bool needs_netdev, +nfp_net_alloc(struct pci_dev *pdev, void __iomem *ctrl_bar, bool needs_netdev, unsigned int max_tx_rings, unsigned int max_rx_rings); void nfp_net_free(struct nfp_net *nn); diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c index 6bddfcfdec34..a0343f25068a 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c @@ -890,8 +890,6 @@ static int nfp_net_tx(struct sk_buff *skb, struct net_device *netdev) u64_stats_update_end(&r_vec->tx_sync); } - netdev_tx_sent_queue(nd_q, txbuf->real_len); - skb_tx_timestamp(skb); tx_ring->wr_p += nr_frags + 1; @@ -899,7 +897,7 @@ static int nfp_net_tx(struct sk_buff *skb, struct net_device *netdev) nfp_net_tx_ring_stop(nd_q, tx_ring); tx_ring->wr_ptr_add += nr_frags + 1; - if (!skb->xmit_more || netif_xmit_stopped(nd_q)) + if (__netdev_tx_sent_queue(nd_q, txbuf->real_len, skb->xmit_more)) nfp_net_tx_xmit_more_flush(tx_ring); return NETDEV_TX_OK; @@ -3560,6 +3558,7 @@ void nfp_net_info(struct nfp_net *nn) /** * nfp_net_alloc() - Allocate netdev and related structure * @pdev: PCI device + * @ctrl_bar: PCI IOMEM with vNIC config memory * @needs_netdev: Whether to allocate a netdev for this vNIC * @max_tx_rings: Maximum number of TX rings supported by device * @max_rx_rings: Maximum number of RX rings supported by device @@ -3570,11 +3569,12 @@ void nfp_net_info(struct nfp_net *nn) * * Return: NFP Net device structure, or ERR_PTR on error. */ -struct nfp_net *nfp_net_alloc(struct pci_dev *pdev, bool needs_netdev, - unsigned int max_tx_rings, - unsigned int max_rx_rings) +struct nfp_net * +nfp_net_alloc(struct pci_dev *pdev, void __iomem *ctrl_bar, bool needs_netdev, + unsigned int max_tx_rings, unsigned int max_rx_rings) { struct nfp_net *nn; + int err; if (needs_netdev) { struct net_device *netdev; @@ -3594,6 +3594,7 @@ struct nfp_net *nfp_net_alloc(struct pci_dev *pdev, bool needs_netdev, } nn->dp.dev = &pdev->dev; + nn->dp.ctrl_bar = ctrl_bar; nn->pdev = pdev; nn->max_tx_rings = max_tx_rings; @@ -3616,7 +3617,19 @@ struct nfp_net *nfp_net_alloc(struct pci_dev *pdev, bool needs_netdev, timer_setup(&nn->reconfig_timer, nfp_net_reconfig_timer, 0); + err = nfp_net_tlv_caps_parse(&nn->pdev->dev, nn->dp.ctrl_bar, + &nn->tlv_caps); + if (err) + goto err_free_nn; + return nn; + +err_free_nn: + if (nn->dp.netdev) + free_netdev(nn->dp.netdev); + else + vfree(nn); + return ERR_PTR(err); } /** @@ -3889,11 +3902,6 @@ int nfp_net_init(struct nfp_net *nn) nn->dp.ctrl |= NFP_NET_CFG_CTRL_IRQMOD; } - err = nfp_net_tlv_caps_parse(&nn->pdev->dev, nn->dp.ctrl_bar, - &nn->tlv_caps); - if (err) - return err; - if (nn->dp.netdev) nfp_net_netdev_init(nn); diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_main.c b/drivers/net/ethernet/netronome/nfp/nfp_net_main.c index 1e7d20468a34..08f5fdbd8e41 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_main.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_main.c @@ -116,13 +116,13 @@ nfp_net_pf_alloc_vnic(struct nfp_pf *pf, bool needs_netdev, n_rx_rings = readl(ctrl_bar + NFP_NET_CFG_MAX_RXRINGS); /* Allocate and initialise the vNIC */ - nn = nfp_net_alloc(pf->pdev, needs_netdev, n_tx_rings, n_rx_rings); + nn = nfp_net_alloc(pf->pdev, ctrl_bar, needs_netdev, + n_tx_rings, n_rx_rings); if (IS_ERR(nn)) return nn; nn->app = pf->app; nfp_net_get_fw_version(&nn->fw_ver, ctrl_bar); - nn->dp.ctrl_bar = ctrl_bar; nn->tx_bar = qc_bar + tx_base * NFP_QCP_QUEUE_ADDR_SZ; nn->rx_bar = qc_bar + rx_base * NFP_QCP_QUEUE_ADDR_SZ; nn->dp.is_vf = 0; diff --git a/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c b/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c index d2c1e9ea5668..1145849ca7ba 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c @@ -172,7 +172,7 @@ static int nfp_netvf_pci_probe(struct pci_dev *pdev, rx_bar_off = NFP_PCIE_QUEUE(startq); /* Allocate and initialise the netdev */ - nn = nfp_net_alloc(pdev, true, max_tx_rings, max_rx_rings); + nn = nfp_net_alloc(pdev, ctrl_bar, true, max_tx_rings, max_rx_rings); if (IS_ERR(nn)) { err = PTR_ERR(nn); goto err_ctrl_unmap; @@ -180,7 +180,6 @@ static int nfp_netvf_pci_probe(struct pci_dev *pdev, vf->nn = nn; nn->fw_ver = fw_ver; - nn->dp.ctrl_bar = ctrl_bar; nn->dp.is_vf = 1; nn->stride_tx = stride; nn->stride_rx = stride; |