diff options
Diffstat (limited to 'drivers/net/ethernet/netronome/nfp/flower')
| -rw-r--r-- | drivers/net/ethernet/netronome/nfp/flower/action.c | 203 | ||||
| -rw-r--r-- | drivers/net/ethernet/netronome/nfp/flower/cmsg.c | 58 | ||||
| -rw-r--r-- | drivers/net/ethernet/netronome/nfp/flower/cmsg.h | 22 | ||||
| -rw-r--r-- | drivers/net/ethernet/netronome/nfp/flower/main.c | 155 | ||||
| -rw-r--r-- | drivers/net/ethernet/netronome/nfp/flower/main.h | 103 | ||||
| -rw-r--r-- | drivers/net/ethernet/netronome/nfp/flower/match.c | 9 | ||||
| -rw-r--r-- | drivers/net/ethernet/netronome/nfp/flower/metadata.c | 117 | ||||
| -rw-r--r-- | drivers/net/ethernet/netronome/nfp/flower/offload.c | 618 | ||||
| -rw-r--r-- | drivers/net/ethernet/netronome/nfp/flower/qos_conf.c | 366 | ||||
| -rw-r--r-- | drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c | 11 |
10 files changed, 1535 insertions, 127 deletions
diff --git a/drivers/net/ethernet/netronome/nfp/flower/action.c b/drivers/net/ethernet/netronome/nfp/flower/action.c index e336f6ee94f5..c56e31d9f8a4 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/action.c +++ b/drivers/net/ethernet/netronome/nfp/flower/action.c @@ -160,9 +160,9 @@ nfp_fl_get_tun_from_act_l4_port(struct nfp_app *app, struct nfp_flower_priv *priv = app->priv; switch (tun->key.tp_dst) { - case htons(NFP_FL_VXLAN_PORT): + case htons(IANA_VXLAN_UDP_PORT): return NFP_FL_TUNNEL_VXLAN; - case htons(NFP_FL_GENEVE_PORT): + case htons(GENEVE_UDP_PORT): if (priv->flower_ext_feats & NFP_FL_FEATS_GENEVE) return NFP_FL_TUNNEL_GENEVE; /* FALLTHROUGH */ @@ -582,60 +582,23 @@ static u32 nfp_fl_csum_l4_to_flag(u8 ip_proto) } } -static int -nfp_fl_pedit(const struct flow_action_entry *act, - struct tc_cls_flower_offload *flow, - char *nfp_action, int *a_len, u32 *csum_updated) -{ - struct flow_rule *rule = tc_cls_flower_offload_flow_rule(flow); +struct nfp_flower_pedit_acts { struct nfp_fl_set_ipv6_addr set_ip6_dst, set_ip6_src; struct nfp_fl_set_ipv6_tc_hl_fl set_ip6_tc_hl_fl; struct nfp_fl_set_ip4_ttl_tos set_ip_ttl_tos; struct nfp_fl_set_ip4_addrs set_ip_addr; - enum flow_action_mangle_base htype; struct nfp_fl_set_tport set_tport; struct nfp_fl_set_eth set_eth; +}; + +static int +nfp_fl_commit_mangle(struct tc_cls_flower_offload *flow, char *nfp_action, + int *a_len, struct nfp_flower_pedit_acts *set_act, + u32 *csum_updated) +{ + struct flow_rule *rule = tc_cls_flower_offload_flow_rule(flow); size_t act_size = 0; u8 ip_proto = 0; - u32 offset; - int err; - - memset(&set_ip6_tc_hl_fl, 0, sizeof(set_ip6_tc_hl_fl)); - memset(&set_ip_ttl_tos, 0, sizeof(set_ip_ttl_tos)); - memset(&set_ip6_dst, 0, sizeof(set_ip6_dst)); - memset(&set_ip6_src, 0, sizeof(set_ip6_src)); - memset(&set_ip_addr, 0, sizeof(set_ip_addr)); - memset(&set_tport, 0, sizeof(set_tport)); - memset(&set_eth, 0, sizeof(set_eth)); - - htype = act->mangle.htype; - offset = act->mangle.offset; - - switch (htype) { - case TCA_PEDIT_KEY_EX_HDR_TYPE_ETH: - err = nfp_fl_set_eth(act, offset, &set_eth); - break; - case TCA_PEDIT_KEY_EX_HDR_TYPE_IP4: - err = nfp_fl_set_ip4(act, offset, &set_ip_addr, - &set_ip_ttl_tos); - break; - case TCA_PEDIT_KEY_EX_HDR_TYPE_IP6: - err = nfp_fl_set_ip6(act, offset, &set_ip6_dst, - &set_ip6_src, &set_ip6_tc_hl_fl); - break; - case TCA_PEDIT_KEY_EX_HDR_TYPE_TCP: - err = nfp_fl_set_tport(act, offset, &set_tport, - NFP_FL_ACTION_OPCODE_SET_TCP); - break; - case TCA_PEDIT_KEY_EX_HDR_TYPE_UDP: - err = nfp_fl_set_tport(act, offset, &set_tport, - NFP_FL_ACTION_OPCODE_SET_UDP); - break; - default: - return -EOPNOTSUPP; - } - if (err) - return err; if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { struct flow_match_basic match; @@ -644,77 +607,82 @@ nfp_fl_pedit(const struct flow_action_entry *act, ip_proto = match.key->ip_proto; } - if (set_eth.head.len_lw) { - act_size = sizeof(set_eth); - memcpy(nfp_action, &set_eth, act_size); + if (set_act->set_eth.head.len_lw) { + act_size = sizeof(set_act->set_eth); + memcpy(nfp_action, &set_act->set_eth, act_size); *a_len += act_size; } - if (set_ip_ttl_tos.head.len_lw) { + + if (set_act->set_ip_ttl_tos.head.len_lw) { nfp_action += act_size; - act_size = sizeof(set_ip_ttl_tos); - memcpy(nfp_action, &set_ip_ttl_tos, act_size); + act_size = sizeof(set_act->set_ip_ttl_tos); + memcpy(nfp_action, &set_act->set_ip_ttl_tos, act_size); *a_len += act_size; /* Hardware will automatically fix IPv4 and TCP/UDP checksum. */ *csum_updated |= TCA_CSUM_UPDATE_FLAG_IPV4HDR | nfp_fl_csum_l4_to_flag(ip_proto); } - if (set_ip_addr.head.len_lw) { + + if (set_act->set_ip_addr.head.len_lw) { nfp_action += act_size; - act_size = sizeof(set_ip_addr); - memcpy(nfp_action, &set_ip_addr, act_size); + act_size = sizeof(set_act->set_ip_addr); + memcpy(nfp_action, &set_act->set_ip_addr, act_size); *a_len += act_size; /* Hardware will automatically fix IPv4 and TCP/UDP checksum. */ *csum_updated |= TCA_CSUM_UPDATE_FLAG_IPV4HDR | nfp_fl_csum_l4_to_flag(ip_proto); } - if (set_ip6_tc_hl_fl.head.len_lw) { + + if (set_act->set_ip6_tc_hl_fl.head.len_lw) { nfp_action += act_size; - act_size = sizeof(set_ip6_tc_hl_fl); - memcpy(nfp_action, &set_ip6_tc_hl_fl, act_size); + act_size = sizeof(set_act->set_ip6_tc_hl_fl); + memcpy(nfp_action, &set_act->set_ip6_tc_hl_fl, act_size); *a_len += act_size; /* Hardware will automatically fix TCP/UDP checksum. */ *csum_updated |= nfp_fl_csum_l4_to_flag(ip_proto); } - if (set_ip6_dst.head.len_lw && set_ip6_src.head.len_lw) { + + if (set_act->set_ip6_dst.head.len_lw && + set_act->set_ip6_src.head.len_lw) { /* TC compiles set src and dst IPv6 address as a single action, * the hardware requires this to be 2 separate actions. */ nfp_action += act_size; - act_size = sizeof(set_ip6_src); - memcpy(nfp_action, &set_ip6_src, act_size); + act_size = sizeof(set_act->set_ip6_src); + memcpy(nfp_action, &set_act->set_ip6_src, act_size); *a_len += act_size; - act_size = sizeof(set_ip6_dst); - memcpy(&nfp_action[sizeof(set_ip6_src)], &set_ip6_dst, - act_size); + act_size = sizeof(set_act->set_ip6_dst); + memcpy(&nfp_action[sizeof(set_act->set_ip6_src)], + &set_act->set_ip6_dst, act_size); *a_len += act_size; /* Hardware will automatically fix TCP/UDP checksum. */ *csum_updated |= nfp_fl_csum_l4_to_flag(ip_proto); - } else if (set_ip6_dst.head.len_lw) { + } else if (set_act->set_ip6_dst.head.len_lw) { nfp_action += act_size; - act_size = sizeof(set_ip6_dst); - memcpy(nfp_action, &set_ip6_dst, act_size); + act_size = sizeof(set_act->set_ip6_dst); + memcpy(nfp_action, &set_act->set_ip6_dst, act_size); *a_len += act_size; /* Hardware will automatically fix TCP/UDP checksum. */ *csum_updated |= nfp_fl_csum_l4_to_flag(ip_proto); - } else if (set_ip6_src.head.len_lw) { + } else if (set_act->set_ip6_src.head.len_lw) { nfp_action += act_size; - act_size = sizeof(set_ip6_src); - memcpy(nfp_action, &set_ip6_src, act_size); + act_size = sizeof(set_act->set_ip6_src); + memcpy(nfp_action, &set_act->set_ip6_src, act_size); *a_len += act_size; /* Hardware will automatically fix TCP/UDP checksum. */ *csum_updated |= nfp_fl_csum_l4_to_flag(ip_proto); } - if (set_tport.head.len_lw) { + if (set_act->set_tport.head.len_lw) { nfp_action += act_size; - act_size = sizeof(set_tport); - memcpy(nfp_action, &set_tport, act_size); + act_size = sizeof(set_act->set_tport); + memcpy(nfp_action, &set_act->set_tport, act_size); *a_len += act_size; /* Hardware will automatically fix TCP/UDP checksum. */ @@ -725,7 +693,40 @@ nfp_fl_pedit(const struct flow_action_entry *act, } static int -nfp_flower_output_action(struct nfp_app *app, const struct flow_action_entry *act, +nfp_fl_pedit(const struct flow_action_entry *act, + struct tc_cls_flower_offload *flow, char *nfp_action, int *a_len, + u32 *csum_updated, struct nfp_flower_pedit_acts *set_act) +{ + enum flow_action_mangle_base htype; + u32 offset; + + htype = act->mangle.htype; + offset = act->mangle.offset; + + switch (htype) { + case TCA_PEDIT_KEY_EX_HDR_TYPE_ETH: + return nfp_fl_set_eth(act, offset, &set_act->set_eth); + case TCA_PEDIT_KEY_EX_HDR_TYPE_IP4: + return nfp_fl_set_ip4(act, offset, &set_act->set_ip_addr, + &set_act->set_ip_ttl_tos); + case TCA_PEDIT_KEY_EX_HDR_TYPE_IP6: + return nfp_fl_set_ip6(act, offset, &set_act->set_ip6_dst, + &set_act->set_ip6_src, + &set_act->set_ip6_tc_hl_fl); + case TCA_PEDIT_KEY_EX_HDR_TYPE_TCP: + return nfp_fl_set_tport(act, offset, &set_act->set_tport, + NFP_FL_ACTION_OPCODE_SET_TCP); + case TCA_PEDIT_KEY_EX_HDR_TYPE_UDP: + return nfp_fl_set_tport(act, offset, &set_act->set_tport, + NFP_FL_ACTION_OPCODE_SET_UDP); + default: + return -EOPNOTSUPP; + } +} + +static int +nfp_flower_output_action(struct nfp_app *app, + const struct flow_action_entry *act, struct nfp_fl_payload *nfp_fl, int *a_len, struct net_device *netdev, bool last, enum nfp_flower_tun_type *tun_type, int *tun_out_cnt, @@ -775,7 +776,8 @@ nfp_flower_loop_action(struct nfp_app *app, const struct flow_action_entry *act, struct nfp_fl_payload *nfp_fl, int *a_len, struct net_device *netdev, enum nfp_flower_tun_type *tun_type, int *tun_out_cnt, - int *out_cnt, u32 *csum_updated) + int *out_cnt, u32 *csum_updated, + struct nfp_flower_pedit_acts *set_act) { struct nfp_fl_set_ipv4_udp_tun *set_tun; struct nfp_fl_pre_tunnel *pre_tun; @@ -860,7 +862,7 @@ nfp_flower_loop_action(struct nfp_app *app, const struct flow_action_entry *act, return 0; case FLOW_ACTION_MANGLE: if (nfp_fl_pedit(act, flow, &nfp_fl->action_data[*a_len], - a_len, csum_updated)) + a_len, csum_updated, set_act)) return -EOPNOTSUPP; break; case FLOW_ACTION_CSUM: @@ -880,12 +882,49 @@ nfp_flower_loop_action(struct nfp_app *app, const struct flow_action_entry *act, return 0; } +static bool nfp_fl_check_mangle_start(struct flow_action *flow_act, + int current_act_idx) +{ + struct flow_action_entry current_act; + struct flow_action_entry prev_act; + + current_act = flow_act->entries[current_act_idx]; + if (current_act.id != FLOW_ACTION_MANGLE) + return false; + + if (current_act_idx == 0) + return true; + + prev_act = flow_act->entries[current_act_idx - 1]; + + return prev_act.id != FLOW_ACTION_MANGLE; +} + +static bool nfp_fl_check_mangle_end(struct flow_action *flow_act, + int current_act_idx) +{ + struct flow_action_entry current_act; + struct flow_action_entry next_act; + + current_act = flow_act->entries[current_act_idx]; + if (current_act.id != FLOW_ACTION_MANGLE) + return false; + + if (current_act_idx == flow_act->num_entries) + return true; + + next_act = flow_act->entries[current_act_idx + 1]; + + return next_act.id != FLOW_ACTION_MANGLE; +} + int nfp_flower_compile_action(struct nfp_app *app, struct tc_cls_flower_offload *flow, struct net_device *netdev, struct nfp_fl_payload *nfp_flow) { int act_len, act_cnt, err, tun_out_cnt, out_cnt, i; + struct nfp_flower_pedit_acts set_act; enum nfp_flower_tun_type tun_type; struct flow_action_entry *act; u32 csum_updated = 0; @@ -899,12 +938,18 @@ int nfp_flower_compile_action(struct nfp_app *app, out_cnt = 0; flow_action_for_each(i, act, &flow->rule->action) { + if (nfp_fl_check_mangle_start(&flow->rule->action, i)) + memset(&set_act, 0, sizeof(set_act)); err = nfp_flower_loop_action(app, act, flow, nfp_flow, &act_len, netdev, &tun_type, &tun_out_cnt, - &out_cnt, &csum_updated); + &out_cnt, &csum_updated, &set_act); if (err) return err; act_cnt++; + if (nfp_fl_check_mangle_end(&flow->rule->action, i)) + nfp_fl_commit_mangle(flow, + &nfp_flow->action_data[act_len], + &act_len, &set_act, &csum_updated); } /* We optimise when the action list is small, this can unfortunately diff --git a/drivers/net/ethernet/netronome/nfp/flower/cmsg.c b/drivers/net/ethernet/netronome/nfp/flower/cmsg.c index cf9e1118ee8f..d5bbe3d6048b 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/cmsg.c +++ b/drivers/net/ethernet/netronome/nfp/flower/cmsg.c @@ -159,7 +159,7 @@ nfp_flower_cmsg_portmod_rx(struct nfp_app *app, struct sk_buff *skb) rtnl_lock(); rcu_read_lock(); - netdev = nfp_app_repr_get(app, be32_to_cpu(msg->portnum)); + netdev = nfp_app_dev_get(app, be32_to_cpu(msg->portnum), NULL); rcu_read_unlock(); if (!netdev) { nfp_flower_cmsg_warn(app, "ctrl msg for unknown port 0x%08x\n", @@ -192,7 +192,7 @@ nfp_flower_cmsg_portreify_rx(struct nfp_app *app, struct sk_buff *skb) msg = nfp_flower_cmsg_get_data(skb); rcu_read_lock(); - exists = !!nfp_app_repr_get(app, be32_to_cpu(msg->portnum)); + exists = !!nfp_app_dev_get(app, be32_to_cpu(msg->portnum), NULL); rcu_read_unlock(); if (!exists) { nfp_flower_cmsg_warn(app, "ctrl msg for unknown port 0x%08x\n", @@ -205,6 +205,50 @@ nfp_flower_cmsg_portreify_rx(struct nfp_app *app, struct sk_buff *skb) } static void +nfp_flower_cmsg_merge_hint_rx(struct nfp_app *app, struct sk_buff *skb) +{ + unsigned int msg_len = nfp_flower_cmsg_get_data_len(skb); + struct nfp_flower_cmsg_merge_hint *msg; + struct nfp_fl_payload *sub_flows[2]; + int err, i, flow_cnt; + + msg = nfp_flower_cmsg_get_data(skb); + /* msg->count starts at 0 and always assumes at least 1 entry. */ + flow_cnt = msg->count + 1; + + if (msg_len < struct_size(msg, flow, flow_cnt)) { + nfp_flower_cmsg_warn(app, "Merge hint ctrl msg too short - %d bytes but expect %zd\n", + msg_len, struct_size(msg, flow, flow_cnt)); + return; + } + + if (flow_cnt != 2) { + nfp_flower_cmsg_warn(app, "Merge hint contains %d flows - two are expected\n", + flow_cnt); + return; + } + + rtnl_lock(); + for (i = 0; i < flow_cnt; i++) { + u32 ctx = be32_to_cpu(msg->flow[i].host_ctx); + + sub_flows[i] = nfp_flower_get_fl_payload_from_ctx(app, ctx); + if (!sub_flows[i]) { + nfp_flower_cmsg_warn(app, "Invalid flow in merge hint\n"); + goto err_rtnl_unlock; + } + } + + err = nfp_flower_merge_offloaded_flows(app, sub_flows[0], sub_flows[1]); + /* Only warn on memory fail. Hint veto will not break functionality. */ + if (err == -ENOMEM) + nfp_flower_cmsg_warn(app, "Flow merge memory fail.\n"); + +err_rtnl_unlock: + rtnl_unlock(); +} + +static void nfp_flower_cmsg_process_one_rx(struct nfp_app *app, struct sk_buff *skb) { struct nfp_flower_priv *app_priv = app->priv; @@ -222,12 +266,21 @@ nfp_flower_cmsg_process_one_rx(struct nfp_app *app, struct sk_buff *skb) case NFP_FLOWER_CMSG_TYPE_PORT_MOD: nfp_flower_cmsg_portmod_rx(app, skb); break; + case NFP_FLOWER_CMSG_TYPE_MERGE_HINT: + if (app_priv->flower_ext_feats & NFP_FL_FEATS_FLOW_MERGE) { + nfp_flower_cmsg_merge_hint_rx(app, skb); + break; + } + goto err_default; case NFP_FLOWER_CMSG_TYPE_NO_NEIGH: nfp_tunnel_request_route(app, skb); break; case NFP_FLOWER_CMSG_TYPE_ACTIVE_TUNS: nfp_tunnel_keep_alive(app, skb); break; + case NFP_FLOWER_CMSG_TYPE_QOS_STATS: + nfp_flower_stats_rlim_reply(app, skb); + break; case NFP_FLOWER_CMSG_TYPE_LAG_CONFIG: if (app_priv->flower_ext_feats & NFP_FL_FEATS_LAG) { skb_stored = nfp_flower_lag_unprocessed_msg(app, skb); @@ -235,6 +288,7 @@ nfp_flower_cmsg_process_one_rx(struct nfp_app *app, struct sk_buff *skb) } /* fall through */ default: +err_default: nfp_flower_cmsg_warn(app, "Cannot handle invalid repr control type %u\n", type); goto out; diff --git a/drivers/net/ethernet/netronome/nfp/flower/cmsg.h b/drivers/net/ethernet/netronome/nfp/flower/cmsg.h index 0ed51e79db00..537f7fc19584 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/cmsg.h +++ b/drivers/net/ethernet/netronome/nfp/flower/cmsg.h @@ -402,11 +402,13 @@ struct nfp_flower_cmsg_hdr { /* Types defined for port related control messages */ enum nfp_flower_cmsg_type_port { NFP_FLOWER_CMSG_TYPE_FLOW_ADD = 0, + NFP_FLOWER_CMSG_TYPE_FLOW_MOD = 1, NFP_FLOWER_CMSG_TYPE_FLOW_DEL = 2, NFP_FLOWER_CMSG_TYPE_LAG_CONFIG = 4, NFP_FLOWER_CMSG_TYPE_PORT_REIFY = 6, NFP_FLOWER_CMSG_TYPE_MAC_REPR = 7, NFP_FLOWER_CMSG_TYPE_PORT_MOD = 8, + NFP_FLOWER_CMSG_TYPE_MERGE_HINT = 9, NFP_FLOWER_CMSG_TYPE_NO_NEIGH = 10, NFP_FLOWER_CMSG_TYPE_TUN_MAC = 11, NFP_FLOWER_CMSG_TYPE_ACTIVE_TUNS = 12, @@ -414,6 +416,9 @@ enum nfp_flower_cmsg_type_port { NFP_FLOWER_CMSG_TYPE_TUN_IPS = 14, NFP_FLOWER_CMSG_TYPE_FLOW_STATS = 15, NFP_FLOWER_CMSG_TYPE_PORT_ECHO = 16, + NFP_FLOWER_CMSG_TYPE_QOS_MOD = 18, + NFP_FLOWER_CMSG_TYPE_QOS_DEL = 19, + NFP_FLOWER_CMSG_TYPE_QOS_STATS = 20, NFP_FLOWER_CMSG_TYPE_MAX = 32, }; @@ -451,6 +456,16 @@ struct nfp_flower_cmsg_portreify { #define NFP_FLOWER_CMSG_PORTREIFY_INFO_EXIST BIT(0) +/* NFP_FLOWER_CMSG_TYPE_FLOW_MERGE_HINT */ +struct nfp_flower_cmsg_merge_hint { + u8 reserved[3]; + u8 count; + struct { + __be32 host_ctx; + __be64 host_cookie; + } __packed flow[0]; +}; + enum nfp_flower_cmsg_port_type { NFP_FLOWER_CMSG_PORT_TYPE_UNSPEC = 0x0, NFP_FLOWER_CMSG_PORT_TYPE_PHYS_PORT = 0x1, @@ -473,6 +488,13 @@ enum nfp_flower_cmsg_port_vnic_type { #define NFP_FLOWER_CMSG_PORT_PCIE_Q GENMASK(5, 0) #define NFP_FLOWER_CMSG_PORT_PHYS_PORT_NUM GENMASK(7, 0) +static inline u32 nfp_flower_internal_port_get_port_id(u8 internal_port) +{ + return FIELD_PREP(NFP_FLOWER_CMSG_PORT_PHYS_PORT_NUM, internal_port) | + FIELD_PREP(NFP_FLOWER_CMSG_PORT_TYPE, + NFP_FLOWER_CMSG_PORT_TYPE_OTHER_PORT); +} + static inline u32 nfp_flower_cmsg_phys_port(u8 phys_port) { return FIELD_PREP(NFP_FLOWER_CMSG_PORT_PHYS_PORT_NUM, phys_port) | diff --git a/drivers/net/ethernet/netronome/nfp/flower/main.c b/drivers/net/ethernet/netronome/nfp/flower/main.c index 408089133599..eb846133943b 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/main.c +++ b/drivers/net/ethernet/netronome/nfp/flower/main.c @@ -22,6 +22,9 @@ #define NFP_FLOWER_ALLOWED_VER 0x0001000000010000UL +#define NFP_MIN_INT_PORT_ID 1 +#define NFP_MAX_INT_PORT_ID 256 + static const char *nfp_flower_extra_cap(struct nfp_app *app, struct nfp_net *nn) { return "FLOWER"; @@ -32,6 +35,113 @@ static enum devlink_eswitch_mode eswitch_mode_get(struct nfp_app *app) return DEVLINK_ESWITCH_MODE_SWITCHDEV; } +static int +nfp_flower_lookup_internal_port_id(struct nfp_flower_priv *priv, + struct net_device *netdev) +{ + struct net_device *entry; + int i, id = 0; + + rcu_read_lock(); + idr_for_each_entry(&priv->internal_ports.port_ids, entry, i) + if (entry == netdev) { + id = i; + break; + } + rcu_read_unlock(); + + return id; +} + +static int +nfp_flower_get_internal_port_id(struct nfp_app *app, struct net_device *netdev) +{ + struct nfp_flower_priv *priv = app->priv; + int id; + + id = nfp_flower_lookup_internal_port_id(priv, netdev); + if (id > 0) + return id; + + idr_preload(GFP_ATOMIC); + spin_lock_bh(&priv->internal_ports.lock); + id = idr_alloc(&priv->internal_ports.port_ids, netdev, + NFP_MIN_INT_PORT_ID, NFP_MAX_INT_PORT_ID, GFP_ATOMIC); + spin_unlock_bh(&priv->internal_ports.lock); + idr_preload_end(); + + return id; +} + +u32 nfp_flower_get_port_id_from_netdev(struct nfp_app *app, + struct net_device *netdev) +{ + int ext_port; + + if (nfp_netdev_is_nfp_repr(netdev)) { + return nfp_repr_get_port_id(netdev); + } else if (nfp_flower_internal_port_can_offload(app, netdev)) { + ext_port = nfp_flower_get_internal_port_id(app, netdev); + if (ext_port < 0) + return 0; + + return nfp_flower_internal_port_get_port_id(ext_port); + } + + return 0; +} + +static struct net_device * +nfp_flower_get_netdev_from_internal_port_id(struct nfp_app *app, int port_id) +{ + struct nfp_flower_priv *priv = app->priv; + struct net_device *netdev; + + rcu_read_lock(); + netdev = idr_find(&priv->internal_ports.port_ids, port_id); + rcu_read_unlock(); + + return netdev; +} + +static void +nfp_flower_free_internal_port_id(struct nfp_app *app, struct net_device *netdev) +{ + struct nfp_flower_priv *priv = app->priv; + int id; + + id = nfp_flower_lookup_internal_port_id(priv, netdev); + if (!id) + return; + + spin_lock_bh(&priv->internal_ports.lock); + idr_remove(&priv->internal_ports.port_ids, id); + spin_unlock_bh(&priv->internal_ports.lock); +} + +static int +nfp_flower_internal_port_event_handler(struct nfp_app *app, + struct net_device *netdev, + unsigned long event) +{ + if (event == NETDEV_UNREGISTER && + nfp_flower_internal_port_can_offload(app, netdev)) + nfp_flower_free_internal_port_id(app, netdev); + + return NOTIFY_OK; +} + +static void nfp_flower_internal_port_init(struct nfp_flower_priv *priv) +{ + spin_lock_init(&priv->internal_ports.lock); + idr_init(&priv->internal_ports.port_ids); +} + +static void nfp_flower_internal_port_cleanup(struct nfp_flower_priv *priv) +{ + idr_destroy(&priv->internal_ports.port_ids); +} + static struct nfp_flower_non_repr_priv * nfp_flower_non_repr_priv_lookup(struct nfp_app *app, struct net_device *netdev) { @@ -119,12 +229,21 @@ nfp_flower_repr_get_type_and_port(struct nfp_app *app, u32 port_id, u8 *port) } static struct net_device * -nfp_flower_repr_get(struct nfp_app *app, u32 port_id) +nfp_flower_dev_get(struct nfp_app *app, u32 port_id, bool *redir_egress) { enum nfp_repr_type repr_type; struct nfp_reprs *reprs; u8 port = 0; + /* Check if the port is internal. */ + if (FIELD_GET(NFP_FLOWER_CMSG_PORT_TYPE, port_id) == + NFP_FLOWER_CMSG_PORT_TYPE_OTHER_PORT) { + if (redir_egress) + *redir_egress = true; + port = FIELD_GET(NFP_FLOWER_CMSG_PORT_PHYS_PORT_NUM, port_id); + return nfp_flower_get_netdev_from_internal_port_id(app, port); + } + repr_type = nfp_flower_repr_get_type_and_port(app, port_id, &port); if (repr_type > NFP_REPR_TYPE_MAX) return NULL; @@ -641,11 +760,33 @@ static int nfp_flower_init(struct nfp_app *app) goto err_cleanup_metadata; } + if (app_priv->flower_ext_feats & NFP_FL_FEATS_FLOW_MOD) { + /* Tell the firmware that the driver supports flow merging. */ + err = nfp_rtsym_write_le(app->pf->rtbl, + "_abi_flower_merge_hint_enable", 1); + if (!err) { + app_priv->flower_ext_feats |= NFP_FL_FEATS_FLOW_MERGE; + nfp_flower_internal_port_init(app_priv); + } else if (err == -ENOENT) { + nfp_warn(app->cpp, "Flow merge not supported by FW.\n"); + } else { + goto err_lag_clean; + } + } else { + nfp_warn(app->cpp, "Flow mod/merge not supported by FW.\n"); + } + + if (app_priv->flower_ext_feats & NFP_FL_FEATS_VF_RLIM) + nfp_flower_qos_init(app); + INIT_LIST_HEAD(&app_priv->indr_block_cb_priv); INIT_LIST_HEAD(&app_priv->non_repr_priv); return 0; +err_lag_clean: + if (app_priv->flower_ext_feats & NFP_FL_FEATS_LAG) + nfp_flower_lag_cleanup(&app_priv->nfp_lag); err_cleanup_metadata: nfp_flower_metadata_cleanup(app); err_free_app_priv: @@ -661,9 +802,15 @@ static void nfp_flower_clean(struct nfp_app *app) skb_queue_purge(&app_priv->cmsg_skbs_low); flush_work(&app_priv->cmsg_work); + if (app_priv->flower_ext_feats & NFP_FL_FEATS_VF_RLIM) + nfp_flower_qos_cleanup(app); + if (app_priv->flower_ext_feats & NFP_FL_FEATS_LAG) nfp_flower_lag_cleanup(&app_priv->nfp_lag); + if (app_priv->flower_ext_feats & NFP_FL_FEATS_FLOW_MERGE) + nfp_flower_internal_port_cleanup(app_priv); + nfp_flower_metadata_cleanup(app); vfree(app->priv); app->priv = NULL; @@ -762,6 +909,10 @@ nfp_flower_netdev_event(struct nfp_app *app, struct net_device *netdev, if (ret & NOTIFY_STOP_MASK) return ret; + ret = nfp_flower_internal_port_event_handler(app, netdev, event); + if (ret & NOTIFY_STOP_MASK) + return ret; + return nfp_tunnel_mac_event_handler(app, netdev, event, ptr); } @@ -800,7 +951,7 @@ const struct nfp_app_type app_flower = { .sriov_disable = nfp_flower_sriov_disable, .eswitch_mode_get = eswitch_mode_get, - .repr_get = nfp_flower_repr_get, + .dev_get = nfp_flower_dev_get, .setup_tc = nfp_flower_setup_tc, }; diff --git a/drivers/net/ethernet/netronome/nfp/flower/main.h b/drivers/net/ethernet/netronome/nfp/flower/main.h index c0945a5fd1a4..40957a8dbfe6 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/main.h +++ b/drivers/net/ethernet/netronome/nfp/flower/main.h @@ -5,6 +5,7 @@ #define __NFP_FLOWER_H__ 1 #include "cmsg.h" +#include "../nfp_net.h" #include <linux/circ_buf.h> #include <linux/hashtable.h> @@ -34,14 +35,14 @@ struct nfp_app; #define NFP_FL_MASK_REUSE_TIME_NS 40000 #define NFP_FL_MASK_ID_LOCATION 1 -#define NFP_FL_VXLAN_PORT 4789 -#define NFP_FL_GENEVE_PORT 6081 - /* Extra features bitmap. */ #define NFP_FL_FEATS_GENEVE BIT(0) #define NFP_FL_NBI_MTU_SETTING BIT(1) #define NFP_FL_FEATS_GENEVE_OPT BIT(2) #define NFP_FL_FEATS_VLAN_PCP BIT(3) +#define NFP_FL_FEATS_VF_RLIM BIT(4) +#define NFP_FL_FEATS_FLOW_MOD BIT(5) +#define NFP_FL_FEATS_FLOW_MERGE BIT(30) #define NFP_FL_FEATS_LAG BIT(31) struct nfp_fl_mask_id { @@ -118,6 +119,16 @@ struct nfp_fl_lag { }; /** + * struct nfp_fl_internal_ports - Flower APP priv data for additional ports + * @port_ids: Assignment of ids to any additional ports + * @lock: Lock for extra ports list + */ +struct nfp_fl_internal_ports { + struct idr port_ids; + spinlock_t lock; +}; + +/** * struct nfp_flower_priv - Flower APP per-vNIC priv data * @app: Back pointer to app * @nn: Pointer to vNIC @@ -131,6 +142,7 @@ struct nfp_fl_lag { * @flow_table: Hash table used to store flower rules * @stats: Stored stats updates for flower rules * @stats_lock: Lock for flower rule stats updates + * @stats_ctx_table: Hash table to map stats contexts to its flow rule * @cmsg_work: Workqueue for control messages processing * @cmsg_skbs_high: List of higher priority skbs for control message * processing @@ -146,6 +158,10 @@ struct nfp_fl_lag { * @non_repr_priv: List of offloaded non-repr ports and their priv data * @active_mem_unit: Current active memory unit for flower rules * @total_mem_units: Total number of available memory units for flower rules + * @internal_ports: Internal port ids used in offloaded rules + * @qos_stats_work: Workqueue for qos stats processing + * @qos_rate_limiters: Current active qos rate limiters + * @qos_stats_lock: Lock on qos stats updates */ struct nfp_flower_priv { struct nfp_app *app; @@ -160,6 +176,7 @@ struct nfp_flower_priv { struct rhashtable flow_table; struct nfp_fl_stats *stats; spinlock_t stats_lock; /* lock stats */ + struct rhashtable stats_ctx_table; struct work_struct cmsg_work; struct sk_buff_head cmsg_skbs_high; struct sk_buff_head cmsg_skbs_low; @@ -172,6 +189,24 @@ struct nfp_flower_priv { struct list_head non_repr_priv; unsigned int active_mem_unit; unsigned int total_mem_units; + struct nfp_fl_internal_ports internal_ports; + struct delayed_work qos_stats_work; + unsigned int qos_rate_limiters; + spinlock_t qos_stats_lock; /* Protect the qos stats */ +}; + +/** + * struct nfp_fl_qos - Flower APP priv data for quality of service + * @netdev_port_id: NFP port number of repr with qos info + * @curr_stats: Currently stored stats updates for qos info + * @prev_stats: Previously stored updates for qos info + * @last_update: Stored time when last stats were updated + */ +struct nfp_fl_qos { + u32 netdev_port_id; + struct nfp_stat_pair curr_stats; + struct nfp_stat_pair prev_stats; + u64 last_update; }; /** @@ -180,14 +215,18 @@ struct nfp_flower_priv { * @lag_port_flags: Extended port flags to record lag state of repr * @mac_offloaded: Flag indicating a MAC address is offloaded for repr * @offloaded_mac_addr: MAC address that has been offloaded for repr + * @block_shared: Flag indicating if offload applies to shared blocks * @mac_list: List entry of reprs that share the same offloaded MAC + * @qos_table: Stored info on filters implementing qos */ struct nfp_flower_repr_priv { struct nfp_repr *nfp_repr; unsigned long lag_port_flags; bool mac_offloaded; u8 offloaded_mac_addr[ETH_ALEN]; + bool block_shared; struct list_head mac_list; + struct nfp_fl_qos qos_table; }; /** @@ -239,6 +278,25 @@ struct nfp_fl_payload { char *unmasked_data; char *mask_data; char *action_data; + struct list_head linked_flows; + bool in_hw; +}; + +struct nfp_fl_payload_link { + /* A link contains a pointer to a merge flow and an associated sub_flow. + * Each merge flow will feature in 2 links to its underlying sub_flows. + * A sub_flow will have at least 1 link to a merge flow or more if it + * has been used to create multiple merge flows. + * + * For a merge flow, 'linked_flows' in its nfp_fl_payload struct lists + * all links to sub_flows (sub_flow.flow) via merge.list. + * For a sub_flow, 'linked_flows' gives all links to merge flows it has + * formed (merge_flow.flow) via sub_flow.list. + */ + struct { + struct list_head list; + struct nfp_fl_payload *flow; + } merge_flow, sub_flow; }; extern const struct rhashtable_params nfp_flower_table_params; @@ -250,12 +308,40 @@ struct nfp_fl_stats_frame { __be64 stats_cookie; }; +static inline bool +nfp_flower_internal_port_can_offload(struct nfp_app *app, + struct net_device *netdev) +{ + struct nfp_flower_priv *app_priv = app->priv; + + if (!(app_priv->flower_ext_feats & NFP_FL_FEATS_FLOW_MERGE)) + return false; + if (!netdev->rtnl_link_ops) + return false; + if (!strcmp(netdev->rtnl_link_ops->kind, "openvswitch")) + return true; + + return false; +} + +/* The address of the merged flow acts as its cookie. + * Cookies supplied to us by TC flower are also addresses to allocated + * memory and thus this scheme should not generate any collisions. + */ +static inline bool nfp_flower_is_merge_flow(struct nfp_fl_payload *flow_pay) +{ + return flow_pay->tc_flower_cookie == (unsigned long)flow_pay; +} + int nfp_flower_metadata_init(struct nfp_app *app, u64 host_ctx_count, unsigned int host_ctx_split); void nfp_flower_metadata_cleanup(struct nfp_app *app); int nfp_flower_setup_tc(struct nfp_app *app, struct net_device *netdev, enum tc_setup_type type, void *type_data); +int nfp_flower_merge_offloaded_flows(struct nfp_app *app, + struct nfp_fl_payload *sub_flow1, + struct nfp_fl_payload *sub_flow2); int nfp_flower_compile_flow_match(struct nfp_app *app, struct tc_cls_flower_offload *flow, struct nfp_fl_key_ls *key_ls, @@ -270,6 +356,8 @@ int nfp_compile_flow_metadata(struct nfp_app *app, struct tc_cls_flower_offload *flow, struct nfp_fl_payload *nfp_flow, struct net_device *netdev); +void __nfp_modify_flow_metadata(struct nfp_flower_priv *priv, + struct nfp_fl_payload *nfp_flow); int nfp_modify_flow_metadata(struct nfp_app *app, struct nfp_fl_payload *nfp_flow); @@ -277,6 +365,8 @@ struct nfp_fl_payload * nfp_flower_search_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie, struct net_device *netdev); struct nfp_fl_payload * +nfp_flower_get_fl_payload_from_ctx(struct nfp_app *app, u32 ctx_id); +struct nfp_fl_payload * nfp_flower_remove_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie); void nfp_flower_rx_flow_stats(struct nfp_app *app, struct sk_buff *skb); @@ -302,6 +392,11 @@ int nfp_flower_lag_populate_pre_action(struct nfp_app *app, struct nfp_fl_pre_lag *pre_act); int nfp_flower_lag_get_output_id(struct nfp_app *app, struct net_device *master); +void nfp_flower_qos_init(struct nfp_app *app); +void nfp_flower_qos_cleanup(struct nfp_app *app); +int nfp_flower_setup_qos_offload(struct nfp_app *app, struct net_device *netdev, + struct tc_cls_matchall_offload *flow); +void nfp_flower_stats_rlim_reply(struct nfp_app *app, struct sk_buff *skb); int nfp_flower_reg_indir_block_handler(struct nfp_app *app, struct net_device *netdev, unsigned long event); @@ -314,4 +409,6 @@ void __nfp_flower_non_repr_priv_put(struct nfp_flower_non_repr_priv *non_repr_priv); void nfp_flower_non_repr_priv_put(struct nfp_app *app, struct net_device *netdev); +u32 nfp_flower_get_port_id_from_netdev(struct nfp_app *app, + struct net_device *netdev); #endif diff --git a/drivers/net/ethernet/netronome/nfp/flower/match.c b/drivers/net/ethernet/netronome/nfp/flower/match.c index 9b8b843d0340..bfa4bf34911d 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/match.c +++ b/drivers/net/ethernet/netronome/nfp/flower/match.c @@ -326,13 +326,12 @@ int nfp_flower_compile_flow_match(struct nfp_app *app, struct nfp_fl_payload *nfp_flow, enum nfp_flower_tun_type tun_type) { - u32 cmsg_port = 0; + u32 port_id; int err; u8 *ext; u8 *msk; - if (nfp_netdev_is_nfp_repr(netdev)) - cmsg_port = nfp_repr_get_port_id(netdev); + port_id = nfp_flower_get_port_id_from_netdev(app, netdev); memset(nfp_flow->unmasked_data, 0, key_ls->key_size); memset(nfp_flow->mask_data, 0, key_ls->key_size); @@ -358,13 +357,13 @@ int nfp_flower_compile_flow_match(struct nfp_app *app, /* Populate Exact Port data. */ err = nfp_flower_compile_port((struct nfp_flower_in_port *)ext, - cmsg_port, false, tun_type); + port_id, false, tun_type); if (err) return err; /* Populate Mask Port Data. */ err = nfp_flower_compile_port((struct nfp_flower_in_port *)msk, - cmsg_port, true, tun_type); + port_id, true, tun_type); if (err) return err; diff --git a/drivers/net/ethernet/netronome/nfp/flower/metadata.c b/drivers/net/ethernet/netronome/nfp/flower/metadata.c index 492837b852b6..3d326efdc814 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/metadata.c +++ b/drivers/net/ethernet/netronome/nfp/flower/metadata.c @@ -24,6 +24,18 @@ struct nfp_fl_flow_table_cmp_arg { unsigned long cookie; }; +struct nfp_fl_stats_ctx_to_flow { + struct rhash_head ht_node; + u32 stats_cxt; + struct nfp_fl_payload *flow; +}; + +static const struct rhashtable_params stats_ctx_table_params = { + .key_offset = offsetof(struct nfp_fl_stats_ctx_to_flow, stats_cxt), + .head_offset = offsetof(struct nfp_fl_stats_ctx_to_flow, ht_node), + .key_len = sizeof(u32), +}; + static int nfp_release_stats_entry(struct nfp_app *app, u32 stats_context_id) { struct nfp_flower_priv *priv = app->priv; @@ -264,9 +276,6 @@ nfp_check_mask_remove(struct nfp_app *app, char *mask_data, u32 mask_len, if (!mask_entry) return false; - if (meta_flags) - *meta_flags &= ~NFP_FL_META_FLAG_MANAGE_MASK; - *mask_id = mask_entry->mask_id; mask_entry->ref_cnt--; if (!mask_entry->ref_cnt) { @@ -285,25 +294,42 @@ int nfp_compile_flow_metadata(struct nfp_app *app, struct nfp_fl_payload *nfp_flow, struct net_device *netdev) { + struct nfp_fl_stats_ctx_to_flow *ctx_entry; struct nfp_flower_priv *priv = app->priv; struct nfp_fl_payload *check_entry; u8 new_mask_id; u32 stats_cxt; + int err; - if (nfp_get_stats_entry(app, &stats_cxt)) - return -ENOENT; + err = nfp_get_stats_entry(app, &stats_cxt); + if (err) + return err; nfp_flow->meta.host_ctx_id = cpu_to_be32(stats_cxt); nfp_flow->meta.host_cookie = cpu_to_be64(flow->cookie); nfp_flow->ingress_dev = netdev; + ctx_entry = kzalloc(sizeof(*ctx_entry), GFP_KERNEL); + if (!ctx_entry) { + err = -ENOMEM; + goto err_release_stats; + } + + ctx_entry->stats_cxt = stats_cxt; + ctx_entry->flow = nfp_flow; + + if (rhashtable_insert_fast(&priv->stats_ctx_table, &ctx_entry->ht_node, + stats_ctx_table_params)) { + err = -ENOMEM; + goto err_free_ctx_entry; + } + new_mask_id = 0; if (!nfp_check_mask_add(app, nfp_flow->mask_data, nfp_flow->meta.mask_len, &nfp_flow->meta.flags, &new_mask_id)) { - if (nfp_release_stats_entry(app, stats_cxt)) - return -EINVAL; - return -ENOENT; + err = -ENOENT; + goto err_remove_rhash; } nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version); @@ -317,43 +343,82 @@ int nfp_compile_flow_metadata(struct nfp_app *app, check_entry = nfp_flower_search_fl_table(app, flow->cookie, netdev); if (check_entry) { - if (nfp_release_stats_entry(app, stats_cxt)) - return -EINVAL; - - if (!nfp_check_mask_remove(app, nfp_flow->mask_data, - nfp_flow->meta.mask_len, - NULL, &new_mask_id)) - return -EINVAL; - - return -EEXIST; + err = -EEXIST; + goto err_remove_mask; } return 0; + +err_remove_mask: + nfp_check_mask_remove(app, nfp_flow->mask_data, nfp_flow->meta.mask_len, + NULL, &new_mask_id); +err_remove_rhash: + WARN_ON_ONCE(rhashtable_remove_fast(&priv->stats_ctx_table, + &ctx_entry->ht_node, + stats_ctx_table_params)); +err_free_ctx_entry: + kfree(ctx_entry); +err_release_stats: + nfp_release_stats_entry(app, stats_cxt); + + return err; +} + +void __nfp_modify_flow_metadata(struct nfp_flower_priv *priv, + struct nfp_fl_payload *nfp_flow) +{ + nfp_flow->meta.flags &= ~NFP_FL_META_FLAG_MANAGE_MASK; + nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version); + priv->flower_version++; } int nfp_modify_flow_metadata(struct nfp_app *app, struct nfp_fl_payload *nfp_flow) { + struct nfp_fl_stats_ctx_to_flow *ctx_entry; struct nfp_flower_priv *priv = app->priv; u8 new_mask_id = 0; u32 temp_ctx_id; + __nfp_modify_flow_metadata(priv, nfp_flow); + nfp_check_mask_remove(app, nfp_flow->mask_data, nfp_flow->meta.mask_len, &nfp_flow->meta.flags, &new_mask_id); - nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version); - priv->flower_version++; - /* Update flow payload with mask ids. */ nfp_flow->unmasked_data[NFP_FL_MASK_ID_LOCATION] = new_mask_id; - /* Release the stats ctx id. */ + /* Release the stats ctx id and ctx to flow table entry. */ temp_ctx_id = be32_to_cpu(nfp_flow->meta.host_ctx_id); + ctx_entry = rhashtable_lookup_fast(&priv->stats_ctx_table, &temp_ctx_id, + stats_ctx_table_params); + if (!ctx_entry) + return -ENOENT; + + WARN_ON_ONCE(rhashtable_remove_fast(&priv->stats_ctx_table, + &ctx_entry->ht_node, + stats_ctx_table_params)); + kfree(ctx_entry); + return nfp_release_stats_entry(app, temp_ctx_id); } +struct nfp_fl_payload * +nfp_flower_get_fl_payload_from_ctx(struct nfp_app *app, u32 ctx_id) +{ + struct nfp_fl_stats_ctx_to_flow *ctx_entry; + struct nfp_flower_priv *priv = app->priv; + + ctx_entry = rhashtable_lookup_fast(&priv->stats_ctx_table, &ctx_id, + stats_ctx_table_params); + if (!ctx_entry) + return NULL; + + return ctx_entry->flow; +} + static int nfp_fl_obj_cmpfn(struct rhashtable_compare_arg *arg, const void *obj) { @@ -403,6 +468,10 @@ int nfp_flower_metadata_init(struct nfp_app *app, u64 host_ctx_count, if (err) return err; + err = rhashtable_init(&priv->stats_ctx_table, &stats_ctx_table_params); + if (err) + goto err_free_flow_table; + get_random_bytes(&priv->mask_id_seed, sizeof(priv->mask_id_seed)); /* Init ring buffer and unallocated mask_ids. */ @@ -410,7 +479,7 @@ int nfp_flower_metadata_init(struct nfp_app *app, u64 host_ctx_count, kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS, NFP_FLOWER_MASK_ELEMENT_RS, GFP_KERNEL); if (!priv->mask_ids.mask_id_free_list.buf) - goto err_free_flow_table; + goto err_free_stats_ctx_table; priv->mask_ids.init_unallocated = NFP_FLOWER_MASK_ENTRY_RS - 1; @@ -447,6 +516,8 @@ err_free_last_used: kfree(priv->mask_ids.last_used); err_free_mask_id: kfree(priv->mask_ids.mask_id_free_list.buf); +err_free_stats_ctx_table: + rhashtable_destroy(&priv->stats_ctx_table); err_free_flow_table: rhashtable_destroy(&priv->flow_table); return -ENOMEM; @@ -461,6 +532,8 @@ void nfp_flower_metadata_cleanup(struct nfp_app *app) rhashtable_free_and_destroy(&priv->flow_table, nfp_check_rhashtable_empty, NULL); + rhashtable_free_and_destroy(&priv->stats_ctx_table, + nfp_check_rhashtable_empty, NULL); kvfree(priv->stats); kfree(priv->mask_ids.mask_id_free_list.buf); kfree(priv->mask_ids.last_used); diff --git a/drivers/net/ethernet/netronome/nfp/flower/offload.c b/drivers/net/ethernet/netronome/nfp/flower/offload.c index 450d7296fd57..1fbfeb43c538 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/offload.c +++ b/drivers/net/ethernet/netronome/nfp/flower/offload.c @@ -55,6 +55,28 @@ BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | \ BIT(FLOW_DISSECTOR_KEY_ENC_PORTS)) +#define NFP_FLOWER_MERGE_FIELDS \ + (NFP_FLOWER_LAYER_PORT | \ + NFP_FLOWER_LAYER_MAC | \ + NFP_FLOWER_LAYER_TP | \ + NFP_FLOWER_LAYER_IPV4 | \ + NFP_FLOWER_LAYER_IPV6) + +struct nfp_flower_merge_check { + union { + struct { + __be16 tci; + struct nfp_flower_mac_mpls l2; + struct nfp_flower_tp_ports l4; + union { + struct nfp_flower_ipv4 ipv4; + struct nfp_flower_ipv6 ipv6; + }; + }; + unsigned long vals[8]; + }; +}; + static int nfp_flower_xmit_flow(struct nfp_app *app, struct nfp_fl_payload *nfp_flow, u8 mtype) @@ -195,7 +217,7 @@ nfp_flower_calculate_key_layers(struct nfp_app *app, flow_rule_match_enc_opts(rule, &enc_op); switch (enc_ports.key->dst) { - case htons(NFP_FL_VXLAN_PORT): + case htons(IANA_VXLAN_UDP_PORT): *tun_type = NFP_FL_TUNNEL_VXLAN; key_layer |= NFP_FLOWER_LAYER_VXLAN; key_size += sizeof(struct nfp_flower_ipv4_udp_tun); @@ -203,7 +225,7 @@ nfp_flower_calculate_key_layers(struct nfp_app *app, if (enc_op.key) return -EOPNOTSUPP; break; - case htons(NFP_FL_GENEVE_PORT): + case htons(GENEVE_UDP_PORT): if (!(priv->flower_ext_feats & NFP_FL_FEATS_GENEVE)) return -EOPNOTSUPP; *tun_type = NFP_FL_TUNNEL_GENEVE; @@ -326,7 +348,7 @@ nfp_flower_calculate_key_layers(struct nfp_app *app, break; case cpu_to_be16(ETH_P_IPV6): - key_layer |= NFP_FLOWER_LAYER_IPV6; + key_layer |= NFP_FLOWER_LAYER_IPV6; key_size += sizeof(struct nfp_flower_ipv6); break; @@ -376,6 +398,8 @@ nfp_flower_allocate_new(struct nfp_fl_key_ls *key_layer) flow_pay->nfp_tun_ipv4_addr = 0; flow_pay->meta.flags = 0; + INIT_LIST_HEAD(&flow_pay->linked_flows); + flow_pay->in_hw = false; return flow_pay; @@ -388,6 +412,447 @@ err_free_flow: return NULL; } +static int +nfp_flower_update_merge_with_actions(struct nfp_fl_payload *flow, + struct nfp_flower_merge_check *merge, + u8 *last_act_id, int *act_out) +{ + struct nfp_fl_set_ipv6_tc_hl_fl *ipv6_tc_hl_fl; + struct nfp_fl_set_ip4_ttl_tos *ipv4_ttl_tos; + struct nfp_fl_set_ip4_addrs *ipv4_add; + struct nfp_fl_set_ipv6_addr *ipv6_add; + struct nfp_fl_push_vlan *push_vlan; + struct nfp_fl_set_tport *tport; + struct nfp_fl_set_eth *eth; + struct nfp_fl_act_head *a; + unsigned int act_off = 0; + u8 act_id = 0; + u8 *ports; + int i; + + while (act_off < flow->meta.act_len) { + a = (struct nfp_fl_act_head *)&flow->action_data[act_off]; + act_id = a->jump_id; + + switch (act_id) { + case NFP_FL_ACTION_OPCODE_OUTPUT: + if (act_out) + (*act_out)++; + break; + case NFP_FL_ACTION_OPCODE_PUSH_VLAN: + push_vlan = (struct nfp_fl_push_vlan *)a; + if (push_vlan->vlan_tci) + merge->tci = cpu_to_be16(0xffff); + break; + case NFP_FL_ACTION_OPCODE_POP_VLAN: + merge->tci = cpu_to_be16(0); + break; + case NFP_FL_ACTION_OPCODE_SET_IPV4_TUNNEL: + /* New tunnel header means l2 to l4 can be matched. */ + eth_broadcast_addr(&merge->l2.mac_dst[0]); + eth_broadcast_addr(&merge->l2.mac_src[0]); + memset(&merge->l4, 0xff, + sizeof(struct nfp_flower_tp_ports)); + memset(&merge->ipv4, 0xff, + sizeof(struct nfp_flower_ipv4)); + break; + case NFP_FL_ACTION_OPCODE_SET_ETHERNET: + eth = (struct nfp_fl_set_eth *)a; + for (i = 0; i < ETH_ALEN; i++) + merge->l2.mac_dst[i] |= eth->eth_addr_mask[i]; + for (i = 0; i < ETH_ALEN; i++) + merge->l2.mac_src[i] |= + eth->eth_addr_mask[ETH_ALEN + i]; + break; + case NFP_FL_ACTION_OPCODE_SET_IPV4_ADDRS: + ipv4_add = (struct nfp_fl_set_ip4_addrs *)a; + merge->ipv4.ipv4_src |= ipv4_add->ipv4_src_mask; + merge->ipv4.ipv4_dst |= ipv4_add->ipv4_dst_mask; + break; + case NFP_FL_ACTION_OPCODE_SET_IPV4_TTL_TOS: + ipv4_ttl_tos = (struct nfp_fl_set_ip4_ttl_tos *)a; + merge->ipv4.ip_ext.ttl |= ipv4_ttl_tos->ipv4_ttl_mask; + merge->ipv4.ip_ext.tos |= ipv4_ttl_tos->ipv4_tos_mask; + break; + case NFP_FL_ACTION_OPCODE_SET_IPV6_SRC: + ipv6_add = (struct nfp_fl_set_ipv6_addr *)a; + for (i = 0; i < 4; i++) + merge->ipv6.ipv6_src.in6_u.u6_addr32[i] |= + ipv6_add->ipv6[i].mask; + break; + case NFP_FL_ACTION_OPCODE_SET_IPV6_DST: + ipv6_add = (struct nfp_fl_set_ipv6_addr *)a; + for (i = 0; i < 4; i++) + merge->ipv6.ipv6_dst.in6_u.u6_addr32[i] |= + ipv6_add->ipv6[i].mask; + break; + case NFP_FL_ACTION_OPCODE_SET_IPV6_TC_HL_FL: + ipv6_tc_hl_fl = (struct nfp_fl_set_ipv6_tc_hl_fl *)a; + merge->ipv6.ip_ext.ttl |= + ipv6_tc_hl_fl->ipv6_hop_limit_mask; + merge->ipv6.ip_ext.tos |= ipv6_tc_hl_fl->ipv6_tc_mask; + merge->ipv6.ipv6_flow_label_exthdr |= + ipv6_tc_hl_fl->ipv6_label_mask; + break; + case NFP_FL_ACTION_OPCODE_SET_UDP: + case NFP_FL_ACTION_OPCODE_SET_TCP: + tport = (struct nfp_fl_set_tport *)a; + ports = (u8 *)&merge->l4.port_src; + for (i = 0; i < 4; i++) + ports[i] |= tport->tp_port_mask[i]; + break; + case NFP_FL_ACTION_OPCODE_PRE_TUNNEL: + case NFP_FL_ACTION_OPCODE_PRE_LAG: + case NFP_FL_ACTION_OPCODE_PUSH_GENEVE: + break; + default: + return -EOPNOTSUPP; + } + + act_off += a->len_lw << NFP_FL_LW_SIZ; + } + + if (last_act_id) + *last_act_id = act_id; + + return 0; +} + +static int +nfp_flower_populate_merge_match(struct nfp_fl_payload *flow, + struct nfp_flower_merge_check *merge, + bool extra_fields) +{ + struct nfp_flower_meta_tci *meta_tci; + u8 *mask = flow->mask_data; + u8 key_layer, match_size; + + memset(merge, 0, sizeof(struct nfp_flower_merge_check)); + + meta_tci = (struct nfp_flower_meta_tci *)mask; + key_layer = meta_tci->nfp_flow_key_layer; + + if (key_layer & ~NFP_FLOWER_MERGE_FIELDS && !extra_fields) + return -EOPNOTSUPP; + + merge->tci = meta_tci->tci; + mask += sizeof(struct nfp_flower_meta_tci); + + if (key_layer & NFP_FLOWER_LAYER_EXT_META) + mask += sizeof(struct nfp_flower_ext_meta); + + mask += sizeof(struct nfp_flower_in_port); + + if (key_layer & NFP_FLOWER_LAYER_MAC) { + match_size = sizeof(struct nfp_flower_mac_mpls); + memcpy(&merge->l2, mask, match_size); + mask += match_size; + } + + if (key_layer & NFP_FLOWER_LAYER_TP) { + match_size = sizeof(struct nfp_flower_tp_ports); + memcpy(&merge->l4, mask, match_size); + mask += match_size; + } + + if (key_layer & NFP_FLOWER_LAYER_IPV4) { + match_size = sizeof(struct nfp_flower_ipv4); + memcpy(&merge->ipv4, mask, match_size); + } + + if (key_layer & NFP_FLOWER_LAYER_IPV6) { + match_size = sizeof(struct nfp_flower_ipv6); + memcpy(&merge->ipv6, mask, match_size); + } + + return 0; +} + +static int +nfp_flower_can_merge(struct nfp_fl_payload *sub_flow1, + struct nfp_fl_payload *sub_flow2) +{ + /* Two flows can be merged if sub_flow2 only matches on bits that are + * either matched by sub_flow1 or set by a sub_flow1 action. This + * ensures that every packet that hits sub_flow1 and recirculates is + * guaranteed to hit sub_flow2. + */ + struct nfp_flower_merge_check sub_flow1_merge, sub_flow2_merge; + int err, act_out = 0; + u8 last_act_id = 0; + + err = nfp_flower_populate_merge_match(sub_flow1, &sub_flow1_merge, + true); + if (err) + return err; + + err = nfp_flower_populate_merge_match(sub_flow2, &sub_flow2_merge, + false); + if (err) + return err; + + err = nfp_flower_update_merge_with_actions(sub_flow1, &sub_flow1_merge, + &last_act_id, &act_out); + if (err) + return err; + + /* Must only be 1 output action and it must be the last in sequence. */ + if (act_out != 1 || last_act_id != NFP_FL_ACTION_OPCODE_OUTPUT) + return -EOPNOTSUPP; + + /* Reject merge if sub_flow2 matches on something that is not matched + * on or set in an action by sub_flow1. + */ + err = bitmap_andnot(sub_flow2_merge.vals, sub_flow2_merge.vals, + sub_flow1_merge.vals, + sizeof(struct nfp_flower_merge_check) * 8); + if (err) + return -EINVAL; + + return 0; +} + +static unsigned int +nfp_flower_copy_pre_actions(char *act_dst, char *act_src, int len, + bool *tunnel_act) +{ + unsigned int act_off = 0, act_len; + struct nfp_fl_act_head *a; + u8 act_id = 0; + + while (act_off < len) { + a = (struct nfp_fl_act_head *)&act_src[act_off]; + act_len = a->len_lw << NFP_FL_LW_SIZ; + act_id = a->jump_id; + + switch (act_id) { + case NFP_FL_ACTION_OPCODE_PRE_TUNNEL: + if (tunnel_act) + *tunnel_act = true; + /* fall through */ + case NFP_FL_ACTION_OPCODE_PRE_LAG: + memcpy(act_dst + act_off, act_src + act_off, act_len); + break; + default: + return act_off; + } + + act_off += act_len; + } + + return act_off; +} + +static int nfp_fl_verify_post_tun_acts(char *acts, int len) +{ + struct nfp_fl_act_head *a; + unsigned int act_off = 0; + + while (act_off < len) { + a = (struct nfp_fl_act_head *)&acts[act_off]; + if (a->jump_id != NFP_FL_ACTION_OPCODE_OUTPUT) + return -EOPNOTSUPP; + + act_off += a->len_lw << NFP_FL_LW_SIZ; + } + + return 0; +} + +static int +nfp_flower_merge_action(struct nfp_fl_payload *sub_flow1, + struct nfp_fl_payload *sub_flow2, + struct nfp_fl_payload *merge_flow) +{ + unsigned int sub1_act_len, sub2_act_len, pre_off1, pre_off2; + bool tunnel_act = false; + char *merge_act; + int err; + + /* The last action of sub_flow1 must be output - do not merge this. */ + sub1_act_len = sub_flow1->meta.act_len - sizeof(struct nfp_fl_output); + sub2_act_len = sub_flow2->meta.act_len; + + if (!sub2_act_len) + return -EINVAL; + + if (sub1_act_len + sub2_act_len > NFP_FL_MAX_A_SIZ) + return -EINVAL; + + /* A shortcut can only be applied if there is a single action. */ + if (sub1_act_len) + merge_flow->meta.shortcut = cpu_to_be32(NFP_FL_SC_ACT_NULL); + else + merge_flow->meta.shortcut = sub_flow2->meta.shortcut; + + merge_flow->meta.act_len = sub1_act_len + sub2_act_len; + merge_act = merge_flow->action_data; + + /* Copy any pre-actions to the start of merge flow action list. */ + pre_off1 = nfp_flower_copy_pre_actions(merge_act, + sub_flow1->action_data, + sub1_act_len, &tunnel_act); + merge_act += pre_off1; + sub1_act_len -= pre_off1; + pre_off2 = nfp_flower_copy_pre_actions(merge_act, + sub_flow2->action_data, + sub2_act_len, NULL); + merge_act += pre_off2; + sub2_act_len -= pre_off2; + + /* FW does a tunnel push when egressing, therefore, if sub_flow 1 pushes + * a tunnel, sub_flow 2 can only have output actions for a valid merge. + */ + if (tunnel_act) { + char *post_tun_acts = &sub_flow2->action_data[pre_off2]; + + err = nfp_fl_verify_post_tun_acts(post_tun_acts, sub2_act_len); + if (err) + return err; + } + + /* Copy remaining actions from sub_flows 1 and 2. */ + memcpy(merge_act, sub_flow1->action_data + pre_off1, sub1_act_len); + merge_act += sub1_act_len; + memcpy(merge_act, sub_flow2->action_data + pre_off2, sub2_act_len); + + return 0; +} + +/* Flow link code should only be accessed under RTNL. */ +static void nfp_flower_unlink_flow(struct nfp_fl_payload_link *link) +{ + list_del(&link->merge_flow.list); + list_del(&link->sub_flow.list); + kfree(link); +} + +static void nfp_flower_unlink_flows(struct nfp_fl_payload *merge_flow, + struct nfp_fl_payload *sub_flow) +{ + struct nfp_fl_payload_link *link; + + list_for_each_entry(link, &merge_flow->linked_flows, merge_flow.list) + if (link->sub_flow.flow == sub_flow) { + nfp_flower_unlink_flow(link); + return; + } +} + +static int nfp_flower_link_flows(struct nfp_fl_payload *merge_flow, + struct nfp_fl_payload *sub_flow) +{ + struct nfp_fl_payload_link *link; + + link = kmalloc(sizeof(*link), GFP_KERNEL); + if (!link) + return -ENOMEM; + + link->merge_flow.flow = merge_flow; + list_add_tail(&link->merge_flow.list, &merge_flow->linked_flows); + link->sub_flow.flow = sub_flow; + list_add_tail(&link->sub_flow.list, &sub_flow->linked_flows); + + return 0; +} + +/** + * nfp_flower_merge_offloaded_flows() - Merge 2 existing flows to single flow. + * @app: Pointer to the APP handle + * @sub_flow1: Initial flow matched to produce merge hint + * @sub_flow2: Post recirculation flow matched in merge hint + * + * Combines 2 flows (if valid) to a single flow, removing the initial from hw + * and offloading the new, merged flow. + * + * Return: negative value on error, 0 in success. + */ +int nfp_flower_merge_offloaded_flows(struct nfp_app *app, + struct nfp_fl_payload *sub_flow1, + struct nfp_fl_payload *sub_flow2) +{ + struct tc_cls_flower_offload merge_tc_off; + struct nfp_flower_priv *priv = app->priv; + struct nfp_fl_payload *merge_flow; + struct nfp_fl_key_ls merge_key_ls; + int err; + + ASSERT_RTNL(); + + if (sub_flow1 == sub_flow2 || + nfp_flower_is_merge_flow(sub_flow1) || + nfp_flower_is_merge_flow(sub_flow2)) + return -EINVAL; + + err = nfp_flower_can_merge(sub_flow1, sub_flow2); + if (err) + return err; + + merge_key_ls.key_size = sub_flow1->meta.key_len; + + merge_flow = nfp_flower_allocate_new(&merge_key_ls); + if (!merge_flow) + return -ENOMEM; + + merge_flow->tc_flower_cookie = (unsigned long)merge_flow; + merge_flow->ingress_dev = sub_flow1->ingress_dev; + + memcpy(merge_flow->unmasked_data, sub_flow1->unmasked_data, + sub_flow1->meta.key_len); + memcpy(merge_flow->mask_data, sub_flow1->mask_data, + sub_flow1->meta.mask_len); + + err = nfp_flower_merge_action(sub_flow1, sub_flow2, merge_flow); + if (err) + goto err_destroy_merge_flow; + + err = nfp_flower_link_flows(merge_flow, sub_flow1); + if (err) + goto err_destroy_merge_flow; + + err = nfp_flower_link_flows(merge_flow, sub_flow2); + if (err) + goto err_unlink_sub_flow1; + + merge_tc_off.cookie = merge_flow->tc_flower_cookie; + err = nfp_compile_flow_metadata(app, &merge_tc_off, merge_flow, + merge_flow->ingress_dev); + if (err) + goto err_unlink_sub_flow2; + + err = rhashtable_insert_fast(&priv->flow_table, &merge_flow->fl_node, + nfp_flower_table_params); + if (err) + goto err_release_metadata; + + err = nfp_flower_xmit_flow(app, merge_flow, + NFP_FLOWER_CMSG_TYPE_FLOW_MOD); + if (err) + goto err_remove_rhash; + + merge_flow->in_hw = true; + sub_flow1->in_hw = false; + + return 0; + +err_remove_rhash: + WARN_ON_ONCE(rhashtable_remove_fast(&priv->flow_table, + &merge_flow->fl_node, + nfp_flower_table_params)); +err_release_metadata: + nfp_modify_flow_metadata(app, merge_flow); +err_unlink_sub_flow2: + nfp_flower_unlink_flows(merge_flow, sub_flow2); +err_unlink_sub_flow1: + nfp_flower_unlink_flows(merge_flow, sub_flow1); +err_destroy_merge_flow: + kfree(merge_flow->action_data); + kfree(merge_flow->mask_data); + kfree(merge_flow->unmasked_data); + kfree(merge_flow); + return err; +} + /** * nfp_flower_add_offload() - Adds a new flow to hardware. * @app: Pointer to the APP handle @@ -454,6 +919,8 @@ nfp_flower_add_offload(struct nfp_app *app, struct net_device *netdev, if (port) port->tc_offload_cnt++; + flow_pay->in_hw = true; + /* Deallocate flow payload when flower rule has been destroyed. */ kfree(key_layer); @@ -475,6 +942,75 @@ err_free_key_ls: return err; } +static void +nfp_flower_remove_merge_flow(struct nfp_app *app, + struct nfp_fl_payload *del_sub_flow, + struct nfp_fl_payload *merge_flow) +{ + struct nfp_flower_priv *priv = app->priv; + struct nfp_fl_payload_link *link, *temp; + struct nfp_fl_payload *origin; + bool mod = false; + int err; + + link = list_first_entry(&merge_flow->linked_flows, + struct nfp_fl_payload_link, merge_flow.list); + origin = link->sub_flow.flow; + + /* Re-add rule the merge had overwritten if it has not been deleted. */ + if (origin != del_sub_flow) + mod = true; + + err = nfp_modify_flow_metadata(app, merge_flow); + if (err) { + nfp_flower_cmsg_warn(app, "Metadata fail for merge flow delete.\n"); + goto err_free_links; + } + + if (!mod) { + err = nfp_flower_xmit_flow(app, merge_flow, + NFP_FLOWER_CMSG_TYPE_FLOW_DEL); + if (err) { + nfp_flower_cmsg_warn(app, "Failed to delete merged flow.\n"); + goto err_free_links; + } + } else { + __nfp_modify_flow_metadata(priv, origin); + err = nfp_flower_xmit_flow(app, origin, + NFP_FLOWER_CMSG_TYPE_FLOW_MOD); + if (err) + nfp_flower_cmsg_warn(app, "Failed to revert merge flow.\n"); + origin->in_hw = true; + } + +err_free_links: + /* Clean any links connected with the merged flow. */ + list_for_each_entry_safe(link, temp, &merge_flow->linked_flows, + merge_flow.list) + nfp_flower_unlink_flow(link); + + kfree(merge_flow->action_data); + kfree(merge_flow->mask_data); + kfree(merge_flow->unmasked_data); + WARN_ON_ONCE(rhashtable_remove_fast(&priv->flow_table, + &merge_flow->fl_node, + nfp_flower_table_params)); + kfree_rcu(merge_flow, rcu); +} + +static void +nfp_flower_del_linked_merge_flows(struct nfp_app *app, + struct nfp_fl_payload *sub_flow) +{ + struct nfp_fl_payload_link *link, *temp; + + /* Remove any merge flow formed from the deleted sub_flow. */ + list_for_each_entry_safe(link, temp, &sub_flow->linked_flows, + sub_flow.list) + nfp_flower_remove_merge_flow(app, sub_flow, + link->merge_flow.flow); +} + /** * nfp_flower_del_offload() - Removes a flow from hardware. * @app: Pointer to the APP handle @@ -482,7 +1018,7 @@ err_free_key_ls: * @flow: TC flower classifier offload structure * * Removes a flow from the repeated hash structure and clears the - * action payload. + * action payload. Any flows merged from this are also deleted. * * Return: negative value on error, 0 if removed successfully. */ @@ -504,17 +1040,22 @@ nfp_flower_del_offload(struct nfp_app *app, struct net_device *netdev, err = nfp_modify_flow_metadata(app, nfp_flow); if (err) - goto err_free_flow; + goto err_free_merge_flow; if (nfp_flow->nfp_tun_ipv4_addr) nfp_tunnel_del_ipv4_off(app, nfp_flow->nfp_tun_ipv4_addr); + if (!nfp_flow->in_hw) { + err = 0; + goto err_free_merge_flow; + } + err = nfp_flower_xmit_flow(app, nfp_flow, NFP_FLOWER_CMSG_TYPE_FLOW_DEL); - if (err) - goto err_free_flow; + /* Fall through on error. */ -err_free_flow: +err_free_merge_flow: + nfp_flower_del_linked_merge_flows(app, nfp_flow); if (port) port->tc_offload_cnt--; kfree(nfp_flow->action_data); @@ -527,6 +1068,52 @@ err_free_flow: return err; } +static void +__nfp_flower_update_merge_stats(struct nfp_app *app, + struct nfp_fl_payload *merge_flow) +{ + struct nfp_flower_priv *priv = app->priv; + struct nfp_fl_payload_link *link; + struct nfp_fl_payload *sub_flow; + u64 pkts, bytes, used; + u32 ctx_id; + + ctx_id = be32_to_cpu(merge_flow->meta.host_ctx_id); + pkts = priv->stats[ctx_id].pkts; + /* Do not cycle subflows if no stats to distribute. */ + if (!pkts) + return; + bytes = priv->stats[ctx_id].bytes; + used = priv->stats[ctx_id].used; + + /* Reset stats for the merge flow. */ + priv->stats[ctx_id].pkts = 0; + priv->stats[ctx_id].bytes = 0; + + /* The merge flow has received stats updates from firmware. + * Distribute these stats to all subflows that form the merge. + * The stats will collected from TC via the subflows. + */ + list_for_each_entry(link, &merge_flow->linked_flows, merge_flow.list) { + sub_flow = link->sub_flow.flow; + ctx_id = be32_to_cpu(sub_flow->meta.host_ctx_id); + priv->stats[ctx_id].pkts += pkts; + priv->stats[ctx_id].bytes += bytes; + max_t(u64, priv->stats[ctx_id].used, used); + } +} + +static void +nfp_flower_update_merge_stats(struct nfp_app *app, + struct nfp_fl_payload *sub_flow) +{ + struct nfp_fl_payload_link *link; + + /* Get merge flows that the subflow forms to distribute their stats. */ + list_for_each_entry(link, &sub_flow->linked_flows, sub_flow.list) + __nfp_flower_update_merge_stats(app, link->merge_flow.flow); +} + /** * nfp_flower_get_stats() - Populates flow stats obtained from hardware. * @app: Pointer to the APP handle @@ -553,6 +1140,10 @@ nfp_flower_get_stats(struct nfp_app *app, struct net_device *netdev, ctx_id = be32_to_cpu(nfp_flow->meta.host_ctx_id); spin_lock_bh(&priv->stats_lock); + /* If request is for a sub_flow, update stats from merged flows. */ + if (!list_empty(&nfp_flow->linked_flows)) + nfp_flower_update_merge_stats(app, nfp_flow); + flow_stats_update(&flow->stats, priv->stats[ctx_id].bytes, priv->stats[ctx_id].pkts, priv->stats[ctx_id].used); @@ -594,6 +1185,9 @@ static int nfp_flower_setup_tc_block_cb(enum tc_setup_type type, case TC_SETUP_CLSFLOWER: return nfp_flower_repr_offload(repr->app, repr->netdev, type_data); + case TC_SETUP_CLSMATCHALL: + return nfp_flower_setup_qos_offload(repr->app, repr->netdev, + type_data); default: return -EOPNOTSUPP; } @@ -603,10 +1197,14 @@ static int nfp_flower_setup_tc_block(struct net_device *netdev, struct tc_block_offload *f) { struct nfp_repr *repr = netdev_priv(netdev); + struct nfp_flower_repr_priv *repr_priv; if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) return -EOPNOTSUPP; + repr_priv = repr->app_priv; + repr_priv->block_shared = tcf_block_shared(f->block); + switch (f->command) { case TC_BLOCK_BIND: return tcf_block_cb_register(f->block, @@ -682,7 +1280,9 @@ nfp_flower_setup_indr_tc_block(struct net_device *netdev, struct nfp_app *app, struct nfp_flower_priv *priv = app->priv; int err; - if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) + if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS && + !(f->binder_type == TCF_BLOCK_BINDER_TYPE_CLSACT_EGRESS && + nfp_flower_internal_port_can_offload(app, netdev))) return -EOPNOTSUPP; switch (f->command) { diff --git a/drivers/net/ethernet/netronome/nfp/flower/qos_conf.c b/drivers/net/ethernet/netronome/nfp/flower/qos_conf.c new file mode 100644 index 000000000000..86e968cd5ffd --- /dev/null +++ b/drivers/net/ethernet/netronome/nfp/flower/qos_conf.c @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* Copyright (C) 2019 Netronome Systems, Inc. */ + +#include <linux/math64.h> +#include <net/pkt_cls.h> +#include <net/pkt_sched.h> + +#include "cmsg.h" +#include "main.h" +#include "../nfp_port.h" + +#define NFP_FL_QOS_UPDATE msecs_to_jiffies(1000) + +struct nfp_police_cfg_head { + __be32 flags_opts; + __be32 port; +}; + +/* Police cmsg for configuring a trTCM traffic conditioner (8W/32B) + * See RFC 2698 for more details. + * ---------------------------------------------------------------- + * 3 2 1 + * 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | Flag options | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | Port Ingress | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | Token Bucket Peak | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | Token Bucket Committed | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | Peak Burst Size | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | Committed Burst Size | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | Peak Information Rate | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | Committed Information Rate | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + */ +struct nfp_police_config { + struct nfp_police_cfg_head head; + __be32 bkt_tkn_p; + __be32 bkt_tkn_c; + __be32 pbs; + __be32 cbs; + __be32 pir; + __be32 cir; +}; + +struct nfp_police_stats_reply { + struct nfp_police_cfg_head head; + __be64 pass_bytes; + __be64 pass_pkts; + __be64 drop_bytes; + __be64 drop_pkts; +}; + +static int +nfp_flower_install_rate_limiter(struct nfp_app *app, struct net_device *netdev, + struct tc_cls_matchall_offload *flow, + struct netlink_ext_ack *extack) +{ + struct flow_action_entry *action = &flow->rule->action.entries[0]; + struct nfp_flower_priv *fl_priv = app->priv; + struct nfp_flower_repr_priv *repr_priv; + struct nfp_police_config *config; + struct nfp_repr *repr; + struct sk_buff *skb; + u32 netdev_port_id; + u64 burst, rate; + + if (!nfp_netdev_is_nfp_repr(netdev)) { + NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on higher level port"); + return -EOPNOTSUPP; + } + repr = netdev_priv(netdev); + repr_priv = repr->app_priv; + + if (repr_priv->block_shared) { + NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on shared blocks"); + return -EOPNOTSUPP; + } + + if (repr->port->type != NFP_PORT_VF_PORT) { + NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on non-VF ports"); + return -EOPNOTSUPP; + } + + if (!flow_offload_has_one_action(&flow->rule->action)) { + NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload requires a single action"); + return -EOPNOTSUPP; + } + + if (flow->common.prio != (1 << 16)) { + NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload requires highest priority"); + return -EOPNOTSUPP; + } + + if (action->id != FLOW_ACTION_POLICE) { + NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload requires police action"); + return -EOPNOTSUPP; + } + + rate = action->police.rate_bytes_ps; + burst = div_u64(rate * PSCHED_NS2TICKS(action->police.burst), + PSCHED_TICKS_PER_SEC); + netdev_port_id = nfp_repr_get_port_id(netdev); + + skb = nfp_flower_cmsg_alloc(repr->app, sizeof(struct nfp_police_config), + NFP_FLOWER_CMSG_TYPE_QOS_MOD, GFP_KERNEL); + if (!skb) + return -ENOMEM; + + config = nfp_flower_cmsg_get_data(skb); + memset(config, 0, sizeof(struct nfp_police_config)); + config->head.port = cpu_to_be32(netdev_port_id); + config->bkt_tkn_p = cpu_to_be32(burst); + config->bkt_tkn_c = cpu_to_be32(burst); + config->pbs = cpu_to_be32(burst); + config->cbs = cpu_to_be32(burst); + config->pir = cpu_to_be32(rate); + config->cir = cpu_to_be32(rate); + nfp_ctrl_tx(repr->app->ctrl, skb); + + repr_priv->qos_table.netdev_port_id = netdev_port_id; + fl_priv->qos_rate_limiters++; + if (fl_priv->qos_rate_limiters == 1) + schedule_delayed_work(&fl_priv->qos_stats_work, + NFP_FL_QOS_UPDATE); + + return 0; +} + +static int +nfp_flower_remove_rate_limiter(struct nfp_app *app, struct net_device *netdev, + struct tc_cls_matchall_offload *flow, + struct netlink_ext_ack *extack) +{ + struct nfp_flower_priv *fl_priv = app->priv; + struct nfp_flower_repr_priv *repr_priv; + struct nfp_police_config *config; + struct nfp_repr *repr; + struct sk_buff *skb; + u32 netdev_port_id; + + if (!nfp_netdev_is_nfp_repr(netdev)) { + NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on higher level port"); + return -EOPNOTSUPP; + } + repr = netdev_priv(netdev); + + netdev_port_id = nfp_repr_get_port_id(netdev); + repr_priv = repr->app_priv; + + if (!repr_priv->qos_table.netdev_port_id) { + NL_SET_ERR_MSG_MOD(extack, "unsupported offload: cannot remove qos entry that does not exist"); + return -EOPNOTSUPP; + } + + skb = nfp_flower_cmsg_alloc(repr->app, sizeof(struct nfp_police_config), + NFP_FLOWER_CMSG_TYPE_QOS_DEL, GFP_KERNEL); + if (!skb) + return -ENOMEM; + + /* Clear all qos associate data for this interface */ + memset(&repr_priv->qos_table, 0, sizeof(struct nfp_fl_qos)); + fl_priv->qos_rate_limiters--; + if (!fl_priv->qos_rate_limiters) + cancel_delayed_work_sync(&fl_priv->qos_stats_work); + + config = nfp_flower_cmsg_get_data(skb); + memset(config, 0, sizeof(struct nfp_police_config)); + config->head.port = cpu_to_be32(netdev_port_id); + nfp_ctrl_tx(repr->app->ctrl, skb); + + return 0; +} + +void nfp_flower_stats_rlim_reply(struct nfp_app *app, struct sk_buff *skb) +{ + struct nfp_flower_priv *fl_priv = app->priv; + struct nfp_flower_repr_priv *repr_priv; + struct nfp_police_stats_reply *msg; + struct nfp_stat_pair *curr_stats; + struct nfp_stat_pair *prev_stats; + struct net_device *netdev; + struct nfp_repr *repr; + u32 netdev_port_id; + + msg = nfp_flower_cmsg_get_data(skb); + netdev_port_id = be32_to_cpu(msg->head.port); + rcu_read_lock(); + netdev = nfp_app_dev_get(app, netdev_port_id, NULL); + if (!netdev) + goto exit_unlock_rcu; + + repr = netdev_priv(netdev); + repr_priv = repr->app_priv; + curr_stats = &repr_priv->qos_table.curr_stats; + prev_stats = &repr_priv->qos_table.prev_stats; + + spin_lock_bh(&fl_priv->qos_stats_lock); + curr_stats->pkts = be64_to_cpu(msg->pass_pkts) + + be64_to_cpu(msg->drop_pkts); + curr_stats->bytes = be64_to_cpu(msg->pass_bytes) + + be64_to_cpu(msg->drop_bytes); + + if (!repr_priv->qos_table.last_update) { + prev_stats->pkts = curr_stats->pkts; + prev_stats->bytes = curr_stats->bytes; + } + + repr_priv->qos_table.last_update = jiffies; + spin_unlock_bh(&fl_priv->qos_stats_lock); + +exit_unlock_rcu: + rcu_read_unlock(); +} + +static void +nfp_flower_stats_rlim_request(struct nfp_flower_priv *fl_priv, + u32 netdev_port_id) +{ + struct nfp_police_cfg_head *head; + struct sk_buff *skb; + + skb = nfp_flower_cmsg_alloc(fl_priv->app, + sizeof(struct nfp_police_cfg_head), + NFP_FLOWER_CMSG_TYPE_QOS_STATS, + GFP_ATOMIC); + if (!skb) + return; + + head = nfp_flower_cmsg_get_data(skb); + memset(head, 0, sizeof(struct nfp_police_cfg_head)); + head->port = cpu_to_be32(netdev_port_id); + + nfp_ctrl_tx(fl_priv->app->ctrl, skb); +} + +static void +nfp_flower_stats_rlim_request_all(struct nfp_flower_priv *fl_priv) +{ + struct nfp_reprs *repr_set; + int i; + + rcu_read_lock(); + repr_set = rcu_dereference(fl_priv->app->reprs[NFP_REPR_TYPE_VF]); + if (!repr_set) + goto exit_unlock_rcu; + + for (i = 0; i < repr_set->num_reprs; i++) { + struct net_device *netdev; + + netdev = rcu_dereference(repr_set->reprs[i]); + if (netdev) { + struct nfp_repr *priv = netdev_priv(netdev); + struct nfp_flower_repr_priv *repr_priv; + u32 netdev_port_id; + + repr_priv = priv->app_priv; + netdev_port_id = repr_priv->qos_table.netdev_port_id; + if (!netdev_port_id) + continue; + + nfp_flower_stats_rlim_request(fl_priv, netdev_port_id); + } + } + +exit_unlock_rcu: + rcu_read_unlock(); +} + +static void update_stats_cache(struct work_struct *work) +{ + struct delayed_work *delayed_work; + struct nfp_flower_priv *fl_priv; + + delayed_work = to_delayed_work(work); + fl_priv = container_of(delayed_work, struct nfp_flower_priv, + qos_stats_work); + + nfp_flower_stats_rlim_request_all(fl_priv); + schedule_delayed_work(&fl_priv->qos_stats_work, NFP_FL_QOS_UPDATE); +} + +static int +nfp_flower_stats_rate_limiter(struct nfp_app *app, struct net_device *netdev, + struct tc_cls_matchall_offload *flow, + struct netlink_ext_ack *extack) +{ + struct nfp_flower_priv *fl_priv = app->priv; + struct nfp_flower_repr_priv *repr_priv; + struct nfp_stat_pair *curr_stats; + struct nfp_stat_pair *prev_stats; + u64 diff_bytes, diff_pkts; + struct nfp_repr *repr; + + if (!nfp_netdev_is_nfp_repr(netdev)) { + NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on higher level port"); + return -EOPNOTSUPP; + } + repr = netdev_priv(netdev); + + repr_priv = repr->app_priv; + if (!repr_priv->qos_table.netdev_port_id) { + NL_SET_ERR_MSG_MOD(extack, "unsupported offload: cannot find qos entry for stats update"); + return -EOPNOTSUPP; + } + + spin_lock_bh(&fl_priv->qos_stats_lock); + curr_stats = &repr_priv->qos_table.curr_stats; + prev_stats = &repr_priv->qos_table.prev_stats; + diff_pkts = curr_stats->pkts - prev_stats->pkts; + diff_bytes = curr_stats->bytes - prev_stats->bytes; + prev_stats->pkts = curr_stats->pkts; + prev_stats->bytes = curr_stats->bytes; + spin_unlock_bh(&fl_priv->qos_stats_lock); + + flow_stats_update(&flow->stats, diff_bytes, diff_pkts, + repr_priv->qos_table.last_update); + return 0; +} + +void nfp_flower_qos_init(struct nfp_app *app) +{ + struct nfp_flower_priv *fl_priv = app->priv; + + spin_lock_init(&fl_priv->qos_stats_lock); + INIT_DELAYED_WORK(&fl_priv->qos_stats_work, &update_stats_cache); +} + +void nfp_flower_qos_cleanup(struct nfp_app *app) +{ + struct nfp_flower_priv *fl_priv = app->priv; + + cancel_delayed_work_sync(&fl_priv->qos_stats_work); +} + +int nfp_flower_setup_qos_offload(struct nfp_app *app, struct net_device *netdev, + struct tc_cls_matchall_offload *flow) +{ + struct netlink_ext_ack *extack = flow->common.extack; + struct nfp_flower_priv *fl_priv = app->priv; + + if (!(fl_priv->flower_ext_feats & NFP_FL_FEATS_VF_RLIM)) { + NL_SET_ERR_MSG_MOD(extack, "unsupported offload: loaded firmware does not support qos rate limit offload"); + return -EOPNOTSUPP; + } + + switch (flow->command) { + case TC_CLSMATCHALL_REPLACE: + return nfp_flower_install_rate_limiter(app, netdev, flow, + extack); + case TC_CLSMATCHALL_DESTROY: + return nfp_flower_remove_rate_limiter(app, netdev, flow, + extack); + case TC_CLSMATCHALL_STATS: + return nfp_flower_stats_rate_limiter(app, netdev, flow, + extack); + default: + return -EOPNOTSUPP; + } +} diff --git a/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c b/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c index 4d78be4ec4e9..faa06edf95ac 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c +++ b/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c @@ -171,7 +171,7 @@ void nfp_tunnel_keep_alive(struct nfp_app *app, struct sk_buff *skb) for (i = 0; i < count; i++) { ipv4_addr = payload->tun_info[i].ipv4; port = be32_to_cpu(payload->tun_info[i].egress_port); - netdev = nfp_app_repr_get(app, port); + netdev = nfp_app_dev_get(app, port, NULL); if (!netdev) continue; @@ -270,9 +270,10 @@ nfp_tun_write_neigh(struct net_device *netdev, struct nfp_app *app, struct flowi4 *flow, struct neighbour *neigh, gfp_t flag) { struct nfp_tun_neigh payload; + u32 port_id; - /* Only offload representor IPv4s for now. */ - if (!nfp_netdev_is_nfp_repr(netdev)) + port_id = nfp_flower_get_port_id_from_netdev(app, netdev); + if (!port_id) return; memset(&payload, 0, sizeof(struct nfp_tun_neigh)); @@ -290,7 +291,7 @@ nfp_tun_write_neigh(struct net_device *netdev, struct nfp_app *app, payload.src_ipv4 = flow->saddr; ether_addr_copy(payload.src_addr, netdev->dev_addr); neigh_ha_snapshot(payload.dst_addr, neigh, netdev); - payload.port_id = cpu_to_be32(nfp_repr_get_port_id(netdev)); + payload.port_id = cpu_to_be32(port_id); /* Add destination of new route to NFP cache. */ nfp_tun_add_route_to_cache(app, payload.dst_ipv4); @@ -366,7 +367,7 @@ void nfp_tunnel_request_route(struct nfp_app *app, struct sk_buff *skb) payload = nfp_flower_cmsg_get_data(skb); - netdev = nfp_app_repr_get(app, be32_to_cpu(payload->ingress_port)); + netdev = nfp_app_dev_get(app, be32_to_cpu(payload->ingress_port), NULL); if (!netdev) goto route_fail_warning; |