diff options
Diffstat (limited to 'drivers/net')
21 files changed, 269 insertions, 504 deletions
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 4e3d569bf32e..96416f5d97f3 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -7775,7 +7775,7 @@ static const struct net_device_ops bnxt_netdev_ops = { #endif .ndo_udp_tunnel_add = bnxt_udp_tunnel_add, .ndo_udp_tunnel_del = bnxt_udp_tunnel_del, - .ndo_xdp = bnxt_xdp, + .ndo_bpf = bnxt_xdp, .ndo_bridge_getlink = bnxt_bridge_getlink, .ndo_bridge_setlink = bnxt_bridge_setlink, .ndo_get_phys_port_name = bnxt_get_phys_port_name diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c index 06ce63c00821..261e5847557a 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c @@ -208,7 +208,7 @@ static int bnxt_xdp_set(struct bnxt *bp, struct bpf_prog *prog) return 0; } -int bnxt_xdp(struct net_device *dev, struct netdev_xdp *xdp) +int bnxt_xdp(struct net_device *dev, struct netdev_bpf *xdp) { struct bnxt *bp = netdev_priv(dev); int rc; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h index 12a5ad66b564..414b748038ca 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h @@ -16,6 +16,6 @@ void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts); bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, struct page *page, u8 **data_ptr, unsigned int *len, u8 *event); -int bnxt_xdp(struct net_device *dev, struct netdev_xdp *xdp); +int bnxt_xdp(struct net_device *dev, struct netdev_bpf *xdp); #endif diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_main.c b/drivers/net/ethernet/cavium/thunder/nicvf_main.c index 71989e180289..a063c36c4c58 100644 --- a/drivers/net/ethernet/cavium/thunder/nicvf_main.c +++ b/drivers/net/ethernet/cavium/thunder/nicvf_main.c @@ -1741,7 +1741,7 @@ static int nicvf_xdp_setup(struct nicvf *nic, struct bpf_prog *prog) return 0; } -static int nicvf_xdp(struct net_device *netdev, struct netdev_xdp *xdp) +static int nicvf_xdp(struct net_device *netdev, struct netdev_bpf *xdp) { struct nicvf *nic = netdev_priv(netdev); @@ -1774,7 +1774,7 @@ static const struct net_device_ops nicvf_netdev_ops = { .ndo_tx_timeout = nicvf_tx_timeout, .ndo_fix_features = nicvf_fix_features, .ndo_set_features = nicvf_set_features, - .ndo_xdp = nicvf_xdp, + .ndo_bpf = nicvf_xdp, }; static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index dfecaeda0654..05b94d87a6c3 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -11648,12 +11648,12 @@ static int i40e_xdp_setup(struct i40e_vsi *vsi, } /** - * i40e_xdp - implements ndo_xdp for i40e + * i40e_xdp - implements ndo_bpf for i40e * @dev: netdevice * @xdp: XDP command **/ static int i40e_xdp(struct net_device *dev, - struct netdev_xdp *xdp) + struct netdev_bpf *xdp) { struct i40e_netdev_priv *np = netdev_priv(dev); struct i40e_vsi *vsi = np->vsi; @@ -11705,7 +11705,7 @@ static const struct net_device_ops i40e_netdev_ops = { .ndo_features_check = i40e_features_check, .ndo_bridge_getlink = i40e_ndo_bridge_getlink, .ndo_bridge_setlink = i40e_ndo_bridge_setlink, - .ndo_xdp = i40e_xdp, + .ndo_bpf = i40e_xdp, }; /** diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 507977994a03..e5dcb25be398 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -10004,7 +10004,7 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) return 0; } -static int ixgbe_xdp(struct net_device *dev, struct netdev_xdp *xdp) +static int ixgbe_xdp(struct net_device *dev, struct netdev_bpf *xdp) { struct ixgbe_adapter *adapter = netdev_priv(dev); @@ -10113,7 +10113,7 @@ static const struct net_device_ops ixgbe_netdev_ops = { .ndo_udp_tunnel_add = ixgbe_add_udp_tunnel_port, .ndo_udp_tunnel_del = ixgbe_del_udp_tunnel_port, .ndo_features_check = ixgbe_features_check, - .ndo_xdp = ixgbe_xdp, + .ndo_bpf = ixgbe_xdp, .ndo_xdp_xmit = ixgbe_xdp_xmit, .ndo_xdp_flush = ixgbe_xdp_flush, }; diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c index d611df2f274d..736a6ccaf05e 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c @@ -2916,7 +2916,7 @@ static u32 mlx4_xdp_query(struct net_device *dev) return prog_id; } -static int mlx4_xdp(struct net_device *dev, struct netdev_xdp *xdp) +static int mlx4_xdp(struct net_device *dev, struct netdev_bpf *xdp) { switch (xdp->command) { case XDP_SETUP_PROG: @@ -2958,7 +2958,7 @@ static const struct net_device_ops mlx4_netdev_ops = { .ndo_udp_tunnel_del = mlx4_en_del_vxlan_port, .ndo_features_check = mlx4_en_features_check, .ndo_set_tx_maxrate = mlx4_en_set_tx_maxrate, - .ndo_xdp = mlx4_xdp, + .ndo_bpf = mlx4_xdp, }; static const struct net_device_ops mlx4_netdev_ops_master = { @@ -2995,7 +2995,7 @@ static const struct net_device_ops mlx4_netdev_ops_master = { .ndo_udp_tunnel_del = mlx4_en_del_vxlan_port, .ndo_features_check = mlx4_en_features_check, .ndo_set_tx_maxrate = mlx4_en_set_tx_maxrate, - .ndo_xdp = mlx4_xdp, + .ndo_bpf = mlx4_xdp, }; struct mlx4_en_bond { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 28ae00b3eb88..3b7b7bb84eb0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -3831,7 +3831,7 @@ static u32 mlx5e_xdp_query(struct net_device *dev) return prog_id; } -static int mlx5e_xdp(struct net_device *dev, struct netdev_xdp *xdp) +static int mlx5e_xdp(struct net_device *dev, struct netdev_bpf *xdp) { switch (xdp->command) { case XDP_SETUP_PROG: @@ -3883,7 +3883,7 @@ static const struct net_device_ops mlx5e_netdev_ops = { .ndo_rx_flow_steer = mlx5e_rx_flow_steer, #endif .ndo_tx_timeout = mlx5e_tx_timeout, - .ndo_xdp = mlx5e_xdp, + .ndo_bpf = mlx5e_xdp, #ifdef CONFIG_NET_POLL_CONTROLLER .ndo_poll_controller = mlx5e_netpoll, #endif diff --git a/drivers/net/ethernet/netronome/nfp/bpf/jit.c b/drivers/net/ethernet/netronome/nfp/bpf/jit.c index 2609a2487100..995e95410b11 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/jit.c +++ b/drivers/net/ethernet/netronome/nfp/bpf/jit.c @@ -77,17 +77,6 @@ nfp_meta_has_prev(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) return meta->l.prev != &nfp_prog->insns; } -static void nfp_prog_free(struct nfp_prog *nfp_prog) -{ - struct nfp_insn_meta *meta, *tmp; - - list_for_each_entry_safe(meta, tmp, &nfp_prog->insns, l) { - list_del(&meta->l); - kfree(meta); - } - kfree(nfp_prog); -} - static void nfp_prog_push(struct nfp_prog *nfp_prog, u64 insn) { if (nfp_prog->__prog_alloc_len == nfp_prog->prog_len) { @@ -202,47 +191,6 @@ emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer) } static void -__emit_br_byte(struct nfp_prog *nfp_prog, u8 areg, u8 breg, bool imm8, - u8 byte, bool equal, u16 addr, u8 defer, bool src_lmextn) -{ - u16 addr_lo, addr_hi; - u64 insn; - - addr_lo = addr & (OP_BB_ADDR_LO >> __bf_shf(OP_BB_ADDR_LO)); - addr_hi = addr != addr_lo; - - insn = OP_BBYTE_BASE | - FIELD_PREP(OP_BB_A_SRC, areg) | - FIELD_PREP(OP_BB_BYTE, byte) | - FIELD_PREP(OP_BB_B_SRC, breg) | - FIELD_PREP(OP_BB_I8, imm8) | - FIELD_PREP(OP_BB_EQ, equal) | - FIELD_PREP(OP_BB_DEFBR, defer) | - FIELD_PREP(OP_BB_ADDR_LO, addr_lo) | - FIELD_PREP(OP_BB_ADDR_HI, addr_hi) | - FIELD_PREP(OP_BB_SRC_LMEXTN, src_lmextn); - - nfp_prog_push(nfp_prog, insn); -} - -static void -emit_br_byte_neq(struct nfp_prog *nfp_prog, - swreg src, u8 imm, u8 byte, u16 addr, u8 defer) -{ - struct nfp_insn_re_regs reg; - int err; - - err = swreg_to_restricted(reg_none(), src, reg_imm(imm), ®, true); - if (err) { - nfp_prog->error = err; - return; - } - - __emit_br_byte(nfp_prog, reg.areg, reg.breg, reg.i8, byte, false, addr, - defer, reg.src_lmextn); -} - -static void __emit_immed(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 imm_hi, enum immed_width width, bool invert, enum immed_shift shift, bool wr_both, @@ -1479,19 +1427,18 @@ static int mem_ldx_skb(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, swreg dst = reg_both(meta->insn.dst_reg * 2); switch (meta->insn.off) { - case offsetof(struct sk_buff, len): - if (size != FIELD_SIZEOF(struct sk_buff, len)) + case offsetof(struct __sk_buff, len): + if (size != FIELD_SIZEOF(struct __sk_buff, len)) return -EOPNOTSUPP; wrp_mov(nfp_prog, dst, plen_reg(nfp_prog)); break; - case offsetof(struct sk_buff, data): - if (size != sizeof(void *)) + case offsetof(struct __sk_buff, data): + if (size != FIELD_SIZEOF(struct __sk_buff, data)) return -EOPNOTSUPP; wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog)); break; - case offsetof(struct sk_buff, cb) + - offsetof(struct bpf_skb_data_end, data_end): - if (size != sizeof(void *)) + case offsetof(struct __sk_buff, data_end): + if (size != FIELD_SIZEOF(struct __sk_buff, data_end)) return -EOPNOTSUPP; emit_alu(nfp_prog, dst, plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog)); @@ -1510,14 +1457,15 @@ static int mem_ldx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, { swreg dst = reg_both(meta->insn.dst_reg * 2); - if (size != sizeof(void *)) - return -EINVAL; - switch (meta->insn.off) { - case offsetof(struct xdp_buff, data): + case offsetof(struct xdp_md, data): + if (size != FIELD_SIZEOF(struct xdp_md, data)) + return -EOPNOTSUPP; wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog)); break; - case offsetof(struct xdp_buff, data_end): + case offsetof(struct xdp_md, data_end): + if (size != FIELD_SIZEOF(struct xdp_md, data_end)) + return -EOPNOTSUPP; emit_alu(nfp_prog, dst, plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog)); break; @@ -1547,7 +1495,7 @@ mem_ldx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, unsigned int size) { if (meta->ptr.type == PTR_TO_CTX) { - if (nfp_prog->act == NN_ACT_XDP) + if (nfp_prog->type == BPF_PROG_TYPE_XDP) return mem_ldx_xdp(nfp_prog, meta, size); else return mem_ldx_skb(nfp_prog, meta, size); @@ -2022,34 +1970,6 @@ static void nfp_intro(struct nfp_prog *nfp_prog) plen_reg(nfp_prog), ALU_OP_AND, pv_len(nfp_prog)); } -static void nfp_outro_tc_legacy(struct nfp_prog *nfp_prog) -{ - const u8 act2code[] = { - [NN_ACT_TC_DROP] = 0x22, - [NN_ACT_TC_REDIR] = 0x24 - }; - /* Target for aborts */ - nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog); - wrp_immed(nfp_prog, reg_both(0), 0); - - /* Target for normal exits */ - nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog); - /* Legacy TC mode: - * 0 0x11 -> pass, count as stat0 - * -1 drop 0x22 -> drop, count as stat1 - * redir 0x24 -> redir, count as stat1 - * ife mark 0x21 -> pass, count as stat1 - * ife + tx 0x24 -> redir, count as stat1 - */ - emit_br_byte_neq(nfp_prog, reg_b(0), 0xff, 0, nfp_prog->tgt_done, 2); - wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS); - emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x11), SHF_SC_L_SHF, 16); - - emit_br(nfp_prog, BR_UNC, nfp_prog->tgt_done, 1); - emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(act2code[nfp_prog->act]), - SHF_SC_L_SHF, 16); -} - static void nfp_outro_tc_da(struct nfp_prog *nfp_prog) { /* TC direct-action mode: @@ -2142,17 +2062,15 @@ static void nfp_outro_xdp(struct nfp_prog *nfp_prog) static void nfp_outro(struct nfp_prog *nfp_prog) { - switch (nfp_prog->act) { - case NN_ACT_DIRECT: + switch (nfp_prog->type) { + case BPF_PROG_TYPE_SCHED_CLS: nfp_outro_tc_da(nfp_prog); break; - case NN_ACT_TC_DROP: - case NN_ACT_TC_REDIR: - nfp_outro_tc_legacy(nfp_prog); - break; - case NN_ACT_XDP: + case BPF_PROG_TYPE_XDP: nfp_outro_xdp(nfp_prog); break; + default: + WARN_ON(1); } } @@ -2198,28 +2116,6 @@ static int nfp_translate(struct nfp_prog *nfp_prog) return nfp_fixup_branches(nfp_prog); } -static int -nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog, - unsigned int cnt) -{ - unsigned int i; - - for (i = 0; i < cnt; i++) { - struct nfp_insn_meta *meta; - - meta = kzalloc(sizeof(*meta), GFP_KERNEL); - if (!meta) - return -ENOMEM; - - meta->insn = prog[i]; - meta->n = i; - - list_add_tail(&meta->l, &nfp_prog->insns); - } - - return 0; -} - /* --- Optimizations --- */ static void nfp_bpf_opt_reg_init(struct nfp_prog *nfp_prog) { @@ -2347,66 +2243,20 @@ static int nfp_bpf_ustore_calc(struct nfp_prog *nfp_prog, __le64 *ustore) return 0; } -/** - * nfp_bpf_jit() - translate BPF code into NFP assembly - * @filter: kernel BPF filter struct - * @prog_mem: memory to store assembler instructions - * @act: action attached to this eBPF program - * @prog_start: offset of the first instruction when loaded - * @prog_done: where to jump on exit - * @prog_sz: size of @prog_mem in instructions - * @res: achieved parameters of translation results - */ -int -nfp_bpf_jit(struct bpf_prog *filter, void *prog_mem, - enum nfp_bpf_action_type act, - unsigned int prog_start, unsigned int prog_done, - unsigned int prog_sz, struct nfp_bpf_result *res) +int nfp_bpf_jit(struct nfp_prog *nfp_prog) { - struct nfp_prog *nfp_prog; int ret; - nfp_prog = kzalloc(sizeof(*nfp_prog), GFP_KERNEL); - if (!nfp_prog) - return -ENOMEM; - - INIT_LIST_HEAD(&nfp_prog->insns); - nfp_prog->act = act; - nfp_prog->start_off = prog_start; - nfp_prog->tgt_done = prog_done; - - ret = nfp_prog_prepare(nfp_prog, filter->insnsi, filter->len); - if (ret) - goto out; - - ret = nfp_prog_verify(nfp_prog, filter); - if (ret) - goto out; - ret = nfp_bpf_optimize(nfp_prog); if (ret) - goto out; - - nfp_prog->num_regs = MAX_BPF_REG; - nfp_prog->regs_per_thread = 32; - - nfp_prog->prog = prog_mem; - nfp_prog->__prog_alloc_len = prog_sz; + return ret; ret = nfp_translate(nfp_prog); if (ret) { pr_err("Translation failed with error %d (translated: %u)\n", ret, nfp_prog->n_translated); - ret = -EINVAL; - goto out; + return -EINVAL; } - ret = nfp_bpf_ustore_calc(nfp_prog, (__force __le64 *)prog_mem); - - res->n_instr = nfp_prog->prog_len; - res->dense_mode = false; -out: - nfp_prog_free(nfp_prog); - - return ret; + return nfp_bpf_ustore_calc(nfp_prog, (__force __le64 *)nfp_prog->prog); } diff --git a/drivers/net/ethernet/netronome/nfp/bpf/main.c b/drivers/net/ethernet/netronome/nfp/bpf/main.c index 8e3e89cace8d..e379b78e86ef 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/main.c +++ b/drivers/net/ethernet/netronome/nfp/bpf/main.c @@ -54,28 +54,25 @@ static int nfp_bpf_xdp_offload(struct nfp_app *app, struct nfp_net *nn, struct bpf_prog *prog) { - struct tc_cls_bpf_offload cmd = { - .prog = prog, - }; + bool running, xdp_running; int ret; if (!nfp_net_ebpf_capable(nn)) return -EINVAL; - if (nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF) { - if (!nn->dp.bpf_offload_xdp) - return prog ? -EBUSY : 0; - cmd.command = prog ? TC_CLSBPF_REPLACE : TC_CLSBPF_DESTROY; - } else { - if (!prog) - return 0; - cmd.command = TC_CLSBPF_ADD; - } + running = nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF; + xdp_running = running && nn->dp.bpf_offload_xdp; + + if (!prog && !xdp_running) + return 0; + if (prog && running && !xdp_running) + return -EBUSY; - ret = nfp_net_bpf_offload(nn, &cmd); + ret = nfp_net_bpf_offload(nn, prog, running); /* Stop offload if replace not possible */ - if (ret && cmd.command == TC_CLSBPF_REPLACE) + if (ret && prog) nfp_bpf_xdp_offload(app, nn, NULL); + nn->dp.bpf_offload_xdp = prog && !ret; return ret; } @@ -85,34 +82,10 @@ static const char *nfp_bpf_extra_cap(struct nfp_app *app, struct nfp_net *nn) return nfp_net_ebpf_capable(nn) ? "BPF" : ""; } -static int -nfp_bpf_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id) -{ - struct nfp_net_bpf_priv *priv; - int ret; - - priv = kmalloc(sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; - - nn->app_priv = priv; - spin_lock_init(&priv->rx_filter_lock); - priv->nn = nn; - timer_setup(&priv->rx_filter_stats_timer, - nfp_net_filter_stats_timer, 0); - - ret = nfp_app_nic_vnic_alloc(app, nn, id); - if (ret) - kfree(priv); - - return ret; -} - static void nfp_bpf_vnic_free(struct nfp_app *app, struct nfp_net *nn) { if (nn->dp.bpf_offload_xdp) nfp_bpf_xdp_offload(app, nn, NULL); - kfree(nn->app_priv); } static int nfp_bpf_setup_tc_block_cb(enum tc_setup_type type, @@ -121,19 +94,29 @@ static int nfp_bpf_setup_tc_block_cb(enum tc_setup_type type, struct tc_cls_bpf_offload *cls_bpf = type_data; struct nfp_net *nn = cb_priv; - if (!tc_can_offload(nn->dp.netdev)) + if (type != TC_SETUP_CLSBPF || + !tc_can_offload(nn->dp.netdev) || + !nfp_net_ebpf_capable(nn) || + cls_bpf->common.protocol != htons(ETH_P_ALL) || + cls_bpf->common.chain_index) + return -EOPNOTSUPP; + if (nn->dp.bpf_offload_xdp) + return -EBUSY; + + /* Only support TC direct action */ + if (!cls_bpf->exts_integrated || + tcf_exts_has_actions(cls_bpf->exts)) { + nn_err(nn, "only direct action with no legacy actions supported\n"); return -EOPNOTSUPP; + } - switch (type) { - case TC_SETUP_CLSBPF: - if (!nfp_net_ebpf_capable(nn) || - cls_bpf->common.protocol != htons(ETH_P_ALL) || - cls_bpf->common.chain_index) - return -EOPNOTSUPP; - if (nn->dp.bpf_offload_xdp) - return -EBUSY; - - return nfp_net_bpf_offload(nn, cls_bpf); + switch (cls_bpf->command) { + case TC_CLSBPF_REPLACE: + return nfp_net_bpf_offload(nn, cls_bpf->prog, true); + case TC_CLSBPF_ADD: + return nfp_net_bpf_offload(nn, cls_bpf->prog, false); + case TC_CLSBPF_DESTROY: + return nfp_net_bpf_offload(nn, NULL, true); default: return -EOPNOTSUPP; } @@ -184,10 +167,14 @@ const struct nfp_app_type app_bpf = { .extra_cap = nfp_bpf_extra_cap, - .vnic_alloc = nfp_bpf_vnic_alloc, + .vnic_alloc = nfp_app_nic_vnic_alloc, .vnic_free = nfp_bpf_vnic_free, .setup_tc = nfp_bpf_setup_tc, .tc_busy = nfp_bpf_tc_busy, .xdp_offload = nfp_bpf_xdp_offload, + + .bpf_verifier_prep = nfp_bpf_verifier_prep, + .bpf_translate = nfp_bpf_translate, + .bpf_destroy = nfp_bpf_destroy, }; diff --git a/drivers/net/ethernet/netronome/nfp/bpf/main.h b/drivers/net/ethernet/netronome/nfp/bpf/main.h index bc604030ff6c..082a15f6dfb5 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/main.h +++ b/drivers/net/ethernet/netronome/nfp/bpf/main.h @@ -41,7 +41,6 @@ #include <linux/types.h> #include "../nfp_asm.h" -#include "../nfp_net.h" /* For branch fixup logic use up-most byte of branch instruction as scratch * area. Remember to clear this before sending instructions to HW! @@ -65,13 +64,6 @@ enum pkt_vec { PKT_VEC_PKT_PTR = 2, }; -enum nfp_bpf_action_type { - NN_ACT_TC_DROP, - NN_ACT_TC_REDIR, - NN_ACT_DIRECT, - NN_ACT_XDP, -}; - #define pv_len(np) reg_lm(1, PKT_VEC_PKT_LEN) #define pv_ctm_ptr(np) reg_lm(1, PKT_VEC_PKT_PTR) @@ -147,9 +139,8 @@ static inline u8 mbpf_mode(const struct nfp_insn_meta *meta) * @prog: machine code * @prog_len: number of valid instructions in @prog array * @__prog_alloc_len: alloc size of @prog array - * @act: BPF program/action type (TC DA, TC with action, XDP etc.) - * @num_regs: number of registers used by this program - * @regs_per_thread: number of basic registers allocated per thread + * @verifier_meta: temporary storage for verifier's insn meta + * @type: BPF program type * @start_off: address of the first instruction in the memory * @tgt_out: jump target for normal exit * @tgt_abort: jump target for abort (e.g. access outside of packet buffer) @@ -164,10 +155,9 @@ struct nfp_prog { unsigned int prog_len; unsigned int __prog_alloc_len; - enum nfp_bpf_action_type act; + struct nfp_insn_meta *verifier_meta; - unsigned int num_regs; - unsigned int regs_per_thread; + enum bpf_prog_type type; unsigned int start_off; unsigned int tgt_out; @@ -182,38 +172,21 @@ struct nfp_prog { struct list_head insns; }; -struct nfp_bpf_result { - unsigned int n_instr; - bool dense_mode; -}; - -int -nfp_bpf_jit(struct bpf_prog *filter, void *prog, enum nfp_bpf_action_type act, - unsigned int prog_start, unsigned int prog_done, - unsigned int prog_sz, struct nfp_bpf_result *res); +int nfp_bpf_jit(struct nfp_prog *prog); -int nfp_prog_verify(struct nfp_prog *nfp_prog, struct bpf_prog *prog); +extern const struct bpf_ext_analyzer_ops nfp_bpf_analyzer_ops; +struct netdev_bpf; +struct nfp_app; struct nfp_net; -struct tc_cls_bpf_offload; - -/** - * struct nfp_net_bpf_priv - per-vNIC BPF private data - * @rx_filter: Filter offload statistics - dropped packets/bytes - * @rx_filter_prev: Filter offload statistics - values from previous update - * @rx_filter_change: Jiffies when statistics last changed - * @rx_filter_stats_timer: Timer for polling filter offload statistics - * @rx_filter_lock: Lock protecting timer state changes (teardown) - */ -struct nfp_net_bpf_priv { - struct nfp_stat_pair rx_filter, rx_filter_prev; - unsigned long rx_filter_change; - struct timer_list rx_filter_stats_timer; - struct nfp_net *nn; - spinlock_t rx_filter_lock; -}; -int nfp_net_bpf_offload(struct nfp_net *nn, struct tc_cls_bpf_offload *cls_bpf); -void nfp_net_filter_stats_timer(struct timer_list *t); +int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog, + bool old_prog); +int nfp_bpf_verifier_prep(struct nfp_app *app, struct nfp_net *nn, + struct netdev_bpf *bpf); +int nfp_bpf_translate(struct nfp_app *app, struct nfp_net *nn, + struct bpf_prog *prog); +int nfp_bpf_destroy(struct nfp_app *app, struct nfp_net *nn, + struct bpf_prog *prog); #endif diff --git a/drivers/net/ethernet/netronome/nfp/bpf/offload.c b/drivers/net/ethernet/netronome/nfp/bpf/offload.c index 6d576f631392..b6cee71f49d3 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/offload.c +++ b/drivers/net/ethernet/netronome/nfp/bpf/offload.c @@ -51,109 +51,114 @@ #include "../nfp_net_ctrl.h" #include "../nfp_net.h" -void nfp_net_filter_stats_timer(struct timer_list *t) +static int +nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog, + unsigned int cnt) { - struct nfp_net_bpf_priv *priv = from_timer(priv, t, - rx_filter_stats_timer); - struct nfp_net *nn = priv->nn; - struct nfp_stat_pair latest; - - spin_lock_bh(&priv->rx_filter_lock); + unsigned int i; - if (nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF) - mod_timer(&priv->rx_filter_stats_timer, - jiffies + NFP_NET_STAT_POLL_IVL); + for (i = 0; i < cnt; i++) { + struct nfp_insn_meta *meta; - spin_unlock_bh(&priv->rx_filter_lock); + meta = kzalloc(sizeof(*meta), GFP_KERNEL); + if (!meta) + return -ENOMEM; - latest.pkts = nn_readq(nn, NFP_NET_CFG_STATS_APP1_FRAMES); - latest.bytes = nn_readq(nn, NFP_NET_CFG_STATS_APP1_BYTES); + meta->insn = prog[i]; + meta->n = i; - if (latest.pkts != priv->rx_filter.pkts) - priv->rx_filter_change = jiffies; + list_add_tail(&meta->l, &nfp_prog->insns); + } - priv->rx_filter = latest; + return 0; } -static void nfp_net_bpf_stats_reset(struct nfp_net *nn) +static void nfp_prog_free(struct nfp_prog *nfp_prog) { - struct nfp_net_bpf_priv *priv = nn->app_priv; + struct nfp_insn_meta *meta, *tmp; - priv->rx_filter.pkts = nn_readq(nn, NFP_NET_CFG_STATS_APP1_FRAMES); - priv->rx_filter.bytes = nn_readq(nn, NFP_NET_CFG_STATS_APP1_BYTES); - priv->rx_filter_prev = priv->rx_filter; - priv->rx_filter_change = jiffies; + list_for_each_entry_safe(meta, tmp, &nfp_prog->insns, l) { + list_del(&meta->l); + kfree(meta); + } + kfree(nfp_prog); } -static int -nfp_net_bpf_stats_update(struct nfp_net *nn, struct tc_cls_bpf_offload *cls_bpf) +int nfp_bpf_verifier_prep(struct nfp_app *app, struct nfp_net *nn, + struct netdev_bpf *bpf) { - struct nfp_net_bpf_priv *priv = nn->app_priv; - u64 bytes, pkts; + struct bpf_prog *prog = bpf->verifier.prog; + struct nfp_prog *nfp_prog; + int ret; - pkts = priv->rx_filter.pkts - priv->rx_filter_prev.pkts; - bytes = priv->rx_filter.bytes - priv->rx_filter_prev.bytes; - bytes -= pkts * ETH_HLEN; + nfp_prog = kzalloc(sizeof(*nfp_prog), GFP_KERNEL); + if (!nfp_prog) + return -ENOMEM; + prog->aux->offload->dev_priv = nfp_prog; - priv->rx_filter_prev = priv->rx_filter; + INIT_LIST_HEAD(&nfp_prog->insns); + nfp_prog->type = prog->type; + + ret = nfp_prog_prepare(nfp_prog, prog->insnsi, prog->len); + if (ret) + goto err_free; - tcf_exts_stats_update(cls_bpf->exts, - bytes, pkts, priv->rx_filter_change); + nfp_prog->verifier_meta = nfp_prog_first_meta(nfp_prog); + bpf->verifier.ops = &nfp_bpf_analyzer_ops; return 0; -} -static int -nfp_net_bpf_get_act(struct nfp_net *nn, struct tc_cls_bpf_offload *cls_bpf) -{ - const struct tc_action *a; - LIST_HEAD(actions); +err_free: + nfp_prog_free(nfp_prog); - if (!cls_bpf->exts) - return NN_ACT_XDP; + return ret; +} - /* TC direct action */ - if (cls_bpf->exts_integrated) { - if (!tcf_exts_has_actions(cls_bpf->exts)) - return NN_ACT_DIRECT; +int nfp_bpf_translate(struct nfp_app *app, struct nfp_net *nn, + struct bpf_prog *prog) +{ + struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv; + unsigned int stack_size; + unsigned int max_instr; + stack_size = nn_readb(nn, NFP_NET_CFG_BPF_STACK_SZ) * 64; + if (prog->aux->stack_depth > stack_size) { + nn_info(nn, "stack too large: program %dB > FW stack %dB\n", + prog->aux->stack_depth, stack_size); return -EOPNOTSUPP; } - /* TC legacy mode */ - if (!tcf_exts_has_one_action(cls_bpf->exts)) - return -EOPNOTSUPP; + nfp_prog->stack_depth = prog->aux->stack_depth; + nfp_prog->start_off = nn_readw(nn, NFP_NET_CFG_BPF_START); + nfp_prog->tgt_done = nn_readw(nn, NFP_NET_CFG_BPF_DONE); - tcf_exts_to_list(cls_bpf->exts, &actions); - list_for_each_entry(a, &actions, list) { - if (is_tcf_gact_shot(a)) - return NN_ACT_TC_DROP; + max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN); + nfp_prog->__prog_alloc_len = max_instr * sizeof(u64); - if (is_tcf_mirred_egress_redirect(a) && - tcf_mirred_ifindex(a) == nn->dp.netdev->ifindex) - return NN_ACT_TC_REDIR; - } + nfp_prog->prog = kmalloc(nfp_prog->__prog_alloc_len, GFP_KERNEL); + if (!nfp_prog->prog) + return -ENOMEM; - return -EOPNOTSUPP; + return nfp_bpf_jit(nfp_prog); } -static int -nfp_net_bpf_offload_prepare(struct nfp_net *nn, - struct tc_cls_bpf_offload *cls_bpf, - struct nfp_bpf_result *res, - void **code, dma_addr_t *dma_addr, u16 max_instr) +int nfp_bpf_destroy(struct nfp_app *app, struct nfp_net *nn, + struct bpf_prog *prog) { - unsigned int code_sz = max_instr * sizeof(u64); - enum nfp_bpf_action_type act; - unsigned int stack_size; - u16 start_off, done_off; - unsigned int max_mtu; - int ret; + struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv; - ret = nfp_net_bpf_get_act(nn, cls_bpf); - if (ret < 0) - return ret; - act = ret; + kfree(nfp_prog->prog); + nfp_prog_free(nfp_prog); + + return 0; +} + +static int nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog) +{ + struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv; + unsigned int max_mtu; + dma_addr_t dma_addr; + int err; max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32; if (max_mtu < nn->dp.netdev->mtu) { @@ -161,141 +166,80 @@ nfp_net_bpf_offload_prepare(struct nfp_net *nn, return -EOPNOTSUPP; } - start_off = nn_readw(nn, NFP_NET_CFG_BPF_START); - done_off = nn_readw(nn, NFP_NET_CFG_BPF_DONE); - - stack_size = nn_readb(nn, NFP_NET_CFG_BPF_STACK_SZ) * 64; - if (cls_bpf->prog->aux->stack_depth > stack_size) { - nn_info(nn, "stack too large: program %dB > FW stack %dB\n", - cls_bpf->prog->aux->stack_depth, stack_size); - return -EOPNOTSUPP; - } - - *code = dma_zalloc_coherent(nn->dp.dev, code_sz, dma_addr, GFP_KERNEL); - if (!*code) + dma_addr = dma_map_single(nn->dp.dev, nfp_prog->prog, + nfp_prog->prog_len * sizeof(u64), + DMA_TO_DEVICE); + if (dma_mapping_error(nn->dp.dev, dma_addr)) return -ENOMEM; - ret = nfp_bpf_jit(cls_bpf->prog, *code, act, start_off, done_off, - max_instr, res); - if (ret) - goto out; + nn_writew(nn, NFP_NET_CFG_BPF_SIZE, nfp_prog->prog_len); + nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, dma_addr); - return 0; + /* Load up the JITed code */ + err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_BPF); + if (err) + nn_err(nn, "FW command error while loading BPF: %d\n", err); -out: - dma_free_coherent(nn->dp.dev, code_sz, *code, *dma_addr); - return ret; + dma_unmap_single(nn->dp.dev, dma_addr, nfp_prog->prog_len * sizeof(u64), + DMA_TO_DEVICE); + + return err; } -static void -nfp_net_bpf_load_and_start(struct nfp_net *nn, u32 tc_flags, - void *code, dma_addr_t dma_addr, - unsigned int code_sz, unsigned int n_instr, - bool dense_mode) +static void nfp_net_bpf_start(struct nfp_net *nn) { - struct nfp_net_bpf_priv *priv = nn->app_priv; - u64 bpf_addr = dma_addr; int err; - nn->dp.bpf_offload_skip_sw = !!(tc_flags & TCA_CLS_FLAGS_SKIP_SW); - - if (dense_mode) - bpf_addr |= NFP_NET_CFG_BPF_CFG_8CTX; - - nn_writew(nn, NFP_NET_CFG_BPF_SIZE, n_instr); - nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, bpf_addr); - - /* Load up the JITed code */ - err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_BPF); - if (err) - nn_err(nn, "FW command error while loading BPF: %d\n", err); - /* Enable passing packets through BPF function */ nn->dp.ctrl |= NFP_NET_CFG_CTRL_BPF; nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl); err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN); if (err) nn_err(nn, "FW command error while enabling BPF: %d\n", err); - - dma_free_coherent(nn->dp.dev, code_sz, code, dma_addr); - - nfp_net_bpf_stats_reset(nn); - mod_timer(&priv->rx_filter_stats_timer, - jiffies + NFP_NET_STAT_POLL_IVL); } static int nfp_net_bpf_stop(struct nfp_net *nn) { - struct nfp_net_bpf_priv *priv = nn->app_priv; - if (!(nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)) return 0; - spin_lock_bh(&priv->rx_filter_lock); nn->dp.ctrl &= ~NFP_NET_CFG_CTRL_BPF; - spin_unlock_bh(&priv->rx_filter_lock); nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl); - del_timer_sync(&priv->rx_filter_stats_timer); - nn->dp.bpf_offload_skip_sw = 0; - return nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN); } -int nfp_net_bpf_offload(struct nfp_net *nn, struct tc_cls_bpf_offload *cls_bpf) +int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog, + bool old_prog) { - struct nfp_bpf_result res; - dma_addr_t dma_addr; - u16 max_instr; - void *code; int err; - max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN); - - switch (cls_bpf->command) { - case TC_CLSBPF_REPLACE: - /* There is nothing stopping us from implementing seamless - * replace but the simple method of loading I adopted in - * the firmware does not handle atomic replace (i.e. we have to - * stop the BPF offload and re-enable it). Leaking-in a few - * frames which didn't have BPF applied in the hardware should - * be fine if software fallback is available, though. - */ - if (nn->dp.bpf_offload_skip_sw) - return -EBUSY; - - err = nfp_net_bpf_offload_prepare(nn, cls_bpf, &res, &code, - &dma_addr, max_instr); - if (err) - return err; + if (prog && !prog->aux->offload) + return -EINVAL; - nfp_net_bpf_stop(nn); - nfp_net_bpf_load_and_start(nn, cls_bpf->gen_flags, code, - dma_addr, max_instr * sizeof(u64), - res.n_instr, res.dense_mode); - return 0; + if (prog && old_prog) { + u8 cap; - case TC_CLSBPF_ADD: - if (nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF) + cap = nn_readb(nn, NFP_NET_CFG_BPF_CAP); + if (!(cap & NFP_NET_BPF_CAP_RELO)) { + nn_err(nn, "FW does not support live reload\n"); return -EBUSY; + } + } - err = nfp_net_bpf_offload_prepare(nn, cls_bpf, &res, &code, - &dma_addr, max_instr); - if (err) - return err; + /* Something else is loaded, different program type? */ + if (!old_prog && nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF) + return -EBUSY; - nfp_net_bpf_load_and_start(nn, cls_bpf->gen_flags, code, - dma_addr, max_instr * sizeof(u64), - res.n_instr, res.dense_mode); - return 0; - - case TC_CLSBPF_DESTROY: + if (old_prog && !prog) return nfp_net_bpf_stop(nn); - case TC_CLSBPF_STATS: - return nfp_net_bpf_stats_update(nn, cls_bpf); + err = nfp_net_bpf_load(nn, prog); + if (err) + return err; - default: - return -EOPNOTSUPP; - } + if (!old_prog) + nfp_net_bpf_start(nn); + + return 0; } diff --git a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c index a8c7615546a9..8d43491ddd6b 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c +++ b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c @@ -40,12 +40,6 @@ #include "main.h" -/* Analyzer/verifier definitions */ -struct nfp_bpf_analyzer_priv { - struct nfp_prog *prog; - struct nfp_insn_meta *meta; -}; - static struct nfp_insn_meta * nfp_bpf_goto_meta(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, unsigned int insn_idx, unsigned int n_insns) @@ -81,7 +75,7 @@ nfp_bpf_check_exit(struct nfp_prog *nfp_prog, const struct bpf_reg_state *reg0 = cur_regs(env) + BPF_REG_0; u64 imm; - if (nfp_prog->act == NN_ACT_XDP) + if (nfp_prog->type == BPF_PROG_TYPE_XDP) return 0; if (!(reg0->type == SCALAR_VALUE && tnum_is_const(reg0->var_off))) { @@ -94,13 +88,8 @@ nfp_bpf_check_exit(struct nfp_prog *nfp_prog, } imm = reg0->var_off.value; - if (nfp_prog->act != NN_ACT_DIRECT && imm != 0 && (imm & ~0U) != ~0U) { - pr_info("unsupported exit state: %d, imm: %llx\n", - reg0->type, imm); - return -EINVAL; - } - - if (nfp_prog->act == NN_ACT_DIRECT && imm <= TC_ACT_REDIRECT && + if (nfp_prog->type == BPF_PROG_TYPE_SCHED_CLS && + imm <= TC_ACT_REDIRECT && imm != TC_ACT_SHOT && imm != TC_ACT_STOLEN && imm != TC_ACT_QUEUED) { pr_info("unsupported exit state: %d, imm: %llx\n", @@ -176,11 +165,11 @@ nfp_bpf_check_ptr(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, static int nfp_verify_insn(struct bpf_verifier_env *env, int insn_idx, int prev_insn_idx) { - struct nfp_bpf_analyzer_priv *priv = env->analyzer_priv; - struct nfp_insn_meta *meta = priv->meta; + struct nfp_prog *nfp_prog = env->prog->aux->offload->dev_priv; + struct nfp_insn_meta *meta = nfp_prog->verifier_meta; - meta = nfp_bpf_goto_meta(priv->prog, meta, insn_idx, env->prog->len); - priv->meta = meta; + meta = nfp_bpf_goto_meta(nfp_prog, meta, insn_idx, env->prog->len); + nfp_prog->verifier_meta = meta; if (meta->insn.src_reg >= MAX_BPF_REG || meta->insn.dst_reg >= MAX_BPF_REG) { @@ -189,39 +178,18 @@ nfp_verify_insn(struct bpf_verifier_env *env, int insn_idx, int prev_insn_idx) } if (meta->insn.code == (BPF_JMP | BPF_EXIT)) - return nfp_bpf_check_exit(priv->prog, env); + return nfp_bpf_check_exit(nfp_prog, env); if ((meta->insn.code & ~BPF_SIZE_MASK) == (BPF_LDX | BPF_MEM)) - return nfp_bpf_check_ptr(priv->prog, meta, env, + return nfp_bpf_check_ptr(nfp_prog, meta, env, meta->insn.src_reg); if ((meta->insn.code & ~BPF_SIZE_MASK) == (BPF_STX | BPF_MEM)) - return nfp_bpf_check_ptr(priv->prog, meta, env, + return nfp_bpf_check_ptr(nfp_prog, meta, env, meta->insn.dst_reg); return 0; } -static const struct bpf_ext_analyzer_ops nfp_bpf_analyzer_ops = { +const struct bpf_ext_analyzer_ops nfp_bpf_analyzer_ops = { .insn_hook = nfp_verify_insn, }; - -int nfp_prog_verify(struct nfp_prog *nfp_prog, struct bpf_prog *prog) -{ - struct nfp_bpf_analyzer_priv *priv; - int ret; - - nfp_prog->stack_depth = prog->aux->stack_depth; - - priv = kzalloc(sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; - - priv->prog = nfp_prog; - priv->meta = nfp_prog_first_meta(nfp_prog); - - ret = bpf_analyzer(prog, &nfp_bpf_analyzer_ops, priv); - - kfree(priv); - - return ret; -} diff --git a/drivers/net/ethernet/netronome/nfp/nfp_app.h b/drivers/net/ethernet/netronome/nfp/nfp_app.h index 857bb33020ba..54b67c9b8d5b 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_app.h +++ b/drivers/net/ethernet/netronome/nfp/nfp_app.h @@ -42,6 +42,7 @@ struct bpf_prog; struct net_device; +struct netdev_bpf; struct pci_dev; struct sk_buff; struct sk_buff; @@ -83,6 +84,9 @@ extern const struct nfp_app_type app_flower; * @setup_tc: setup TC ndo * @tc_busy: TC HW offload busy (rules loaded) * @xdp_offload: offload an XDP program + * @bpf_verifier_prep: verifier prep for dev-specific BPF programs + * @bpf_translate: translate call for dev-specific BPF programs + * @bpf_destroy: destroy for dev-specific BPF programs * @eswitch_mode_get: get SR-IOV eswitch mode * @sriov_enable: app-specific sriov initialisation * @sriov_disable: app-specific sriov clean-up @@ -118,6 +122,12 @@ struct nfp_app_type { bool (*tc_busy)(struct nfp_app *app, struct nfp_net *nn); int (*xdp_offload)(struct nfp_app *app, struct nfp_net *nn, struct bpf_prog *prog); + int (*bpf_verifier_prep)(struct nfp_app *app, struct nfp_net *nn, + struct netdev_bpf *bpf); + int (*bpf_translate)(struct nfp_app *app, struct nfp_net *nn, + struct bpf_prog *prog); + int (*bpf_destroy)(struct nfp_app *app, struct nfp_net *nn, + struct bpf_prog *prog); int (*sriov_enable)(struct nfp_app *app, int num_vfs); void (*sriov_disable)(struct nfp_app *app); @@ -271,6 +281,33 @@ static inline int nfp_app_xdp_offload(struct nfp_app *app, struct nfp_net *nn, return app->type->xdp_offload(app, nn, prog); } +static inline int +nfp_app_bpf_verifier_prep(struct nfp_app *app, struct nfp_net *nn, + struct netdev_bpf *bpf) +{ + if (!app || !app->type->bpf_verifier_prep) + return -EOPNOTSUPP; + return app->type->bpf_verifier_prep(app, nn, bpf); +} + +static inline int +nfp_app_bpf_translate(struct nfp_app *app, struct nfp_net *nn, + struct bpf_prog *prog) +{ + if (!app || !app->type->bpf_translate) + return -EOPNOTSUPP; + return app->type->bpf_translate(app, nn, prog); +} + +static inline int +nfp_app_bpf_destroy(struct nfp_app *app, struct nfp_net *nn, + struct bpf_prog *prog) +{ + if (!app || !app->type->bpf_destroy) + return -EOPNOTSUPP; + return app->type->bpf_destroy(app, nn, prog); +} + static inline bool nfp_app_ctrl_tx(struct nfp_app *app, struct sk_buff *skb) { trace_devlink_hwmsg(priv_to_devlink(app->pf), false, 0, diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net.h b/drivers/net/ethernet/netronome/nfp/nfp_net.h index 3d411f0d15b6..7f9857c276b1 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net.h +++ b/drivers/net/ethernet/netronome/nfp/nfp_net.h @@ -476,7 +476,6 @@ struct nfp_stat_pair { * @dev: Backpointer to struct device * @netdev: Backpointer to net_device structure * @is_vf: Is the driver attached to a VF? - * @bpf_offload_skip_sw: Offloaded BPF program will not be rerun by cls_bpf * @bpf_offload_xdp: Offloaded BPF program is XDP * @chained_metadata_format: Firemware will use new metadata format * @rx_dma_dir: Mapping direction for RX buffers @@ -502,7 +501,6 @@ struct nfp_net_dp { struct net_device *netdev; u8 is_vf:1; - u8 bpf_offload_skip_sw:1; u8 bpf_offload_xdp:1; u8 chained_metadata_format:1; diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c index 185a3dd35a3f..232044b1b7aa 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c @@ -3378,7 +3378,7 @@ nfp_net_xdp_setup(struct nfp_net *nn, struct bpf_prog *prog, u32 flags, return 0; } -static int nfp_net_xdp(struct net_device *netdev, struct netdev_xdp *xdp) +static int nfp_net_xdp(struct net_device *netdev, struct netdev_bpf *xdp) { struct nfp_net *nn = netdev_priv(netdev); @@ -3393,6 +3393,14 @@ static int nfp_net_xdp(struct net_device *netdev, struct netdev_xdp *xdp) xdp->prog_attached = XDP_ATTACHED_HW; xdp->prog_id = nn->xdp_prog ? nn->xdp_prog->aux->id : 0; return 0; + case BPF_OFFLOAD_VERIFIER_PREP: + return nfp_app_bpf_verifier_prep(nn->app, nn, xdp); + case BPF_OFFLOAD_TRANSLATE: + return nfp_app_bpf_translate(nn->app, nn, + xdp->offload.prog); + case BPF_OFFLOAD_DESTROY: + return nfp_app_bpf_destroy(nn->app, nn, + xdp->offload.prog); default: return -EINVAL; } @@ -3441,7 +3449,7 @@ const struct net_device_ops nfp_net_netdev_ops = { .ndo_get_phys_port_name = nfp_port_get_phys_port_name, .ndo_udp_tunnel_add = nfp_net_add_vxlan_port, .ndo_udp_tunnel_del = nfp_net_del_vxlan_port, - .ndo_xdp = nfp_net_xdp, + .ndo_bpf = nfp_net_xdp, }; /** diff --git a/drivers/net/ethernet/qlogic/qede/qede.h b/drivers/net/ethernet/qlogic/qede/qede.h index adb700512baa..a3a70ade411f 100644 --- a/drivers/net/ethernet/qlogic/qede/qede.h +++ b/drivers/net/ethernet/qlogic/qede/qede.h @@ -503,7 +503,7 @@ void qede_fill_rss_params(struct qede_dev *edev, void qede_udp_tunnel_add(struct net_device *dev, struct udp_tunnel_info *ti); void qede_udp_tunnel_del(struct net_device *dev, struct udp_tunnel_info *ti); -int qede_xdp(struct net_device *dev, struct netdev_xdp *xdp); +int qede_xdp(struct net_device *dev, struct netdev_bpf *xdp); #ifdef CONFIG_DCB void qede_set_dcbnl_ops(struct net_device *ndev); diff --git a/drivers/net/ethernet/qlogic/qede/qede_filter.c b/drivers/net/ethernet/qlogic/qede/qede_filter.c index f79e36e4060a..c1a0708a7d7c 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_filter.c +++ b/drivers/net/ethernet/qlogic/qede/qede_filter.c @@ -1065,7 +1065,7 @@ static int qede_xdp_set(struct qede_dev *edev, struct bpf_prog *prog) return 0; } -int qede_xdp(struct net_device *dev, struct netdev_xdp *xdp) +int qede_xdp(struct net_device *dev, struct netdev_bpf *xdp) { struct qede_dev *edev = netdev_priv(dev); diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c index e5ee9f274a71..8f9b3eb82137 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_main.c +++ b/drivers/net/ethernet/qlogic/qede/qede_main.c @@ -556,7 +556,7 @@ static const struct net_device_ops qede_netdev_ops = { .ndo_udp_tunnel_add = qede_udp_tunnel_add, .ndo_udp_tunnel_del = qede_udp_tunnel_del, .ndo_features_check = qede_features_check, - .ndo_xdp = qede_xdp, + .ndo_bpf = qede_xdp, #ifdef CONFIG_RFS_ACCEL .ndo_rx_flow_steer = qede_rx_flow_steer, #endif @@ -594,7 +594,7 @@ static const struct net_device_ops qede_netdev_vf_xdp_ops = { .ndo_udp_tunnel_add = qede_udp_tunnel_add, .ndo_udp_tunnel_del = qede_udp_tunnel_del, .ndo_features_check = qede_features_check, - .ndo_xdp = qede_xdp, + .ndo_bpf = qede_xdp, }; /* ------------------------------------------------------------------------- diff --git a/drivers/net/tun.c b/drivers/net/tun.c index 8125956f62a1..1a326b697221 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -1141,7 +1141,7 @@ static u32 tun_xdp_query(struct net_device *dev) return 0; } -static int tun_xdp(struct net_device *dev, struct netdev_xdp *xdp) +static int tun_xdp(struct net_device *dev, struct netdev_bpf *xdp) { switch (xdp->command) { case XDP_SETUP_PROG: @@ -1185,7 +1185,7 @@ static const struct net_device_ops tap_netdev_ops = { .ndo_features_check = passthru_features_check, .ndo_set_rx_headroom = tun_set_headroom, .ndo_get_stats64 = tun_net_get_stats64, - .ndo_xdp = tun_xdp, + .ndo_bpf = tun_xdp, }; static void tun_flow_init(struct tun_struct *tun) diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index fc059f193e7d..edf984406ba0 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -2088,7 +2088,7 @@ static u32 virtnet_xdp_query(struct net_device *dev) return 0; } -static int virtnet_xdp(struct net_device *dev, struct netdev_xdp *xdp) +static int virtnet_xdp(struct net_device *dev, struct netdev_bpf *xdp) { switch (xdp->command) { case XDP_SETUP_PROG: @@ -2115,7 +2115,7 @@ static const struct net_device_ops virtnet_netdev = { #ifdef CONFIG_NET_POLL_CONTROLLER .ndo_poll_controller = virtnet_netpoll, #endif - .ndo_xdp = virtnet_xdp, + .ndo_bpf = virtnet_xdp, .ndo_xdp_xmit = virtnet_xdp_xmit, .ndo_xdp_flush = virtnet_xdp_flush, .ndo_features_check = passthru_features_check, |