diff options
Diffstat (limited to 'drivers/net/ethernet/mediatek/mtk_ppe.c')
| -rw-r--r-- | drivers/net/ethernet/mediatek/mtk_ppe.c | 571 |
1 files changed, 479 insertions, 92 deletions
diff --git a/drivers/net/ethernet/mediatek/mtk_ppe.c b/drivers/net/ethernet/mediatek/mtk_ppe.c index 3ad10c793308..2d8ca99f2467 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe.c +++ b/drivers/net/ethernet/mediatek/mtk_ppe.c @@ -6,9 +6,22 @@ #include <linux/iopoll.h> #include <linux/etherdevice.h> #include <linux/platform_device.h> +#include <linux/if_ether.h> +#include <linux/if_vlan.h> +#include <net/dsa.h> +#include "mtk_eth_soc.h" #include "mtk_ppe.h" #include "mtk_ppe_regs.h" +static DEFINE_SPINLOCK(ppe_lock); + +static const struct rhashtable_params mtk_flow_l2_ht_params = { + .head_offset = offsetof(struct mtk_flow_entry, l2_node), + .key_offset = offsetof(struct mtk_flow_entry, data.bridge), + .key_len = offsetof(struct mtk_foe_bridge, key_end), + .automatic_shrinking = true, +}; + static void ppe_w32(struct mtk_ppe *ppe, u32 reg, u32 val) { writel(val, ppe->base + reg); @@ -41,6 +54,11 @@ static u32 ppe_clear(struct mtk_ppe *ppe, u32 reg, u32 val) return ppe_m32(ppe, reg, val, 0); } +static u32 mtk_eth_timestamp(struct mtk_eth *eth) +{ + return mtk_r32(eth, 0x0010) & mtk_get_ib1_ts_mask(eth); +} + static int mtk_ppe_wait_busy(struct mtk_ppe *ppe) { int ret; @@ -70,19 +88,12 @@ static void mtk_ppe_cache_enable(struct mtk_ppe *ppe, bool enable) enable * MTK_PPE_CACHE_CTL_EN); } -static u32 mtk_ppe_hash_entry(struct mtk_foe_entry *e) +static u32 mtk_ppe_hash_entry(struct mtk_eth *eth, struct mtk_foe_entry *e) { u32 hv1, hv2, hv3; u32 hash; - switch (FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, e->ib1)) { - case MTK_PPE_PKT_TYPE_BRIDGE: - hv1 = e->bridge.src_mac_lo; - hv1 ^= ((e->bridge.src_mac_hi & 0xffff) << 16); - hv2 = e->bridge.src_mac_hi >> 16; - hv2 ^= e->bridge.dest_mac_lo; - hv3 = e->bridge.dest_mac_hi; - break; + switch (mtk_get_ib1_pkt_type(eth, e->ib1)) { case MTK_PPE_PKT_TYPE_IPV4_ROUTE: case MTK_PPE_PKT_TYPE_IPV4_HNAPT: hv1 = e->ipv4.orig.ports; @@ -111,16 +122,19 @@ static u32 mtk_ppe_hash_entry(struct mtk_foe_entry *e) hash = (hash >> 24) | ((hash & 0xffffff) << 8); hash ^= hv1 ^ hv2 ^ hv3; hash ^= hash >> 16; - hash <<= 1; + hash <<= (ffs(eth->soc->hash_offset) - 1); hash &= MTK_PPE_ENTRIES - 1; return hash; } static inline struct mtk_foe_mac_info * -mtk_foe_entry_l2(struct mtk_foe_entry *entry) +mtk_foe_entry_l2(struct mtk_eth *eth, struct mtk_foe_entry *entry) { - int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1); + int type = mtk_get_ib1_pkt_type(eth, entry->ib1); + + if (type == MTK_PPE_PKT_TYPE_BRIDGE) + return &entry->bridge.l2; if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) return &entry->ipv6.l2; @@ -129,9 +143,12 @@ mtk_foe_entry_l2(struct mtk_foe_entry *entry) } static inline u32 * -mtk_foe_entry_ib2(struct mtk_foe_entry *entry) +mtk_foe_entry_ib2(struct mtk_eth *eth, struct mtk_foe_entry *entry) { - int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1); + int type = mtk_get_ib1_pkt_type(eth, entry->ib1); + + if (type == MTK_PPE_PKT_TYPE_BRIDGE) + return &entry->bridge.ib2; if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) return &entry->ipv6.ib2; @@ -139,27 +156,38 @@ mtk_foe_entry_ib2(struct mtk_foe_entry *entry) return &entry->ipv4.ib2; } -int mtk_foe_entry_prepare(struct mtk_foe_entry *entry, int type, int l4proto, - u8 pse_port, u8 *src_mac, u8 *dest_mac) +int mtk_foe_entry_prepare(struct mtk_eth *eth, struct mtk_foe_entry *entry, + int type, int l4proto, u8 pse_port, u8 *src_mac, + u8 *dest_mac) { struct mtk_foe_mac_info *l2; u32 ports_pad, val; memset(entry, 0, sizeof(*entry)); - val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) | - FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE, type) | - FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) | - MTK_FOE_IB1_BIND_TTL | - MTK_FOE_IB1_BIND_CACHE; - entry->ib1 = val; + if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) | + FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE_V2, type) | + FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) | + MTK_FOE_IB1_BIND_CACHE_V2 | MTK_FOE_IB1_BIND_TTL_V2; + entry->ib1 = val; - val = FIELD_PREP(MTK_FOE_IB2_PORT_MG, 0x3f) | - FIELD_PREP(MTK_FOE_IB2_PORT_AG, 0x1f) | - FIELD_PREP(MTK_FOE_IB2_DEST_PORT, pse_port); + val = FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, pse_port) | + FIELD_PREP(MTK_FOE_IB2_PORT_AG_V2, 0xf); + } else { + val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) | + FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE, type) | + FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) | + MTK_FOE_IB1_BIND_CACHE | MTK_FOE_IB1_BIND_TTL; + entry->ib1 = val; + + val = FIELD_PREP(MTK_FOE_IB2_DEST_PORT, pse_port) | + FIELD_PREP(MTK_FOE_IB2_PORT_MG, 0x3f) | + FIELD_PREP(MTK_FOE_IB2_PORT_AG, 0x1f); + } if (is_multicast_ether_addr(dest_mac)) - val |= MTK_FOE_IB2_MULTICAST; + val |= mtk_get_ib2_multicast_mask(eth); ports_pad = 0xa5a5a500 | (l4proto & 0xff); if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE) @@ -167,7 +195,12 @@ int mtk_foe_entry_prepare(struct mtk_foe_entry *entry, int type, int l4proto, if (type == MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T) entry->ipv6.ports = ports_pad; - if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) { + if (type == MTK_PPE_PKT_TYPE_BRIDGE) { + ether_addr_copy(entry->bridge.src_mac, src_mac); + ether_addr_copy(entry->bridge.dest_mac, dest_mac); + entry->bridge.ib2 = val; + l2 = &entry->bridge.l2; + } else if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) { entry->ipv6.ib2 = val; l2 = &entry->ipv6.l2; } else { @@ -188,24 +221,30 @@ int mtk_foe_entry_prepare(struct mtk_foe_entry *entry, int type, int l4proto, return 0; } -int mtk_foe_entry_set_pse_port(struct mtk_foe_entry *entry, u8 port) +int mtk_foe_entry_set_pse_port(struct mtk_eth *eth, + struct mtk_foe_entry *entry, u8 port) { - u32 *ib2 = mtk_foe_entry_ib2(entry); - u32 val; + u32 *ib2 = mtk_foe_entry_ib2(eth, entry); + u32 val = *ib2; - val = *ib2; - val &= ~MTK_FOE_IB2_DEST_PORT; - val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT, port); + if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + val &= ~MTK_FOE_IB2_DEST_PORT_V2; + val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, port); + } else { + val &= ~MTK_FOE_IB2_DEST_PORT; + val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT, port); + } *ib2 = val; return 0; } -int mtk_foe_entry_set_ipv4_tuple(struct mtk_foe_entry *entry, bool egress, +int mtk_foe_entry_set_ipv4_tuple(struct mtk_eth *eth, + struct mtk_foe_entry *entry, bool egress, __be32 src_addr, __be16 src_port, __be32 dest_addr, __be16 dest_port) { - int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1); + int type = mtk_get_ib1_pkt_type(eth, entry->ib1); struct mtk_ipv4_tuple *t; switch (type) { @@ -240,11 +279,12 @@ int mtk_foe_entry_set_ipv4_tuple(struct mtk_foe_entry *entry, bool egress, return 0; } -int mtk_foe_entry_set_ipv6_tuple(struct mtk_foe_entry *entry, +int mtk_foe_entry_set_ipv6_tuple(struct mtk_eth *eth, + struct mtk_foe_entry *entry, __be32 *src_addr, __be16 src_port, __be32 *dest_addr, __be16 dest_port) { - int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1); + int type = mtk_get_ib1_pkt_type(eth, entry->ib1); u32 *src, *dest; int i; @@ -275,39 +315,41 @@ int mtk_foe_entry_set_ipv6_tuple(struct mtk_foe_entry *entry, return 0; } -int mtk_foe_entry_set_dsa(struct mtk_foe_entry *entry, int port) +int mtk_foe_entry_set_dsa(struct mtk_eth *eth, struct mtk_foe_entry *entry, + int port) { - struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(entry); + struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); l2->etype = BIT(port); - if (!(entry->ib1 & MTK_FOE_IB1_BIND_VLAN_LAYER)) - entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_VLAN_LAYER, 1); + if (!(entry->ib1 & mtk_get_ib1_vlan_layer_mask(eth))) + entry->ib1 |= mtk_prep_ib1_vlan_layer(eth, 1); else l2->etype |= BIT(8); - entry->ib1 &= ~MTK_FOE_IB1_BIND_VLAN_TAG; + entry->ib1 &= ~mtk_get_ib1_vlan_tag_mask(eth); return 0; } -int mtk_foe_entry_set_vlan(struct mtk_foe_entry *entry, int vid) +int mtk_foe_entry_set_vlan(struct mtk_eth *eth, struct mtk_foe_entry *entry, + int vid) { - struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(entry); + struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); - switch (FIELD_GET(MTK_FOE_IB1_BIND_VLAN_LAYER, entry->ib1)) { + switch (mtk_get_ib1_vlan_layer(eth, entry->ib1)) { case 0: - entry->ib1 |= MTK_FOE_IB1_BIND_VLAN_TAG | - FIELD_PREP(MTK_FOE_IB1_BIND_VLAN_LAYER, 1); + entry->ib1 |= mtk_get_ib1_vlan_tag_mask(eth) | + mtk_prep_ib1_vlan_layer(eth, 1); l2->vlan1 = vid; return 0; case 1: - if (!(entry->ib1 & MTK_FOE_IB1_BIND_VLAN_TAG)) { + if (!(entry->ib1 & mtk_get_ib1_vlan_tag_mask(eth))) { l2->vlan1 = vid; l2->etype |= BIT(8); } else { l2->vlan2 = vid; - entry->ib1 += FIELD_PREP(MTK_FOE_IB1_BIND_VLAN_LAYER, 1); + entry->ib1 += mtk_prep_ib1_vlan_layer(eth, 1); } return 0; default: @@ -315,79 +357,399 @@ int mtk_foe_entry_set_vlan(struct mtk_foe_entry *entry, int vid) } } -int mtk_foe_entry_set_pppoe(struct mtk_foe_entry *entry, int sid) +int mtk_foe_entry_set_pppoe(struct mtk_eth *eth, struct mtk_foe_entry *entry, + int sid) { - struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(entry); + struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); - if (!(entry->ib1 & MTK_FOE_IB1_BIND_VLAN_LAYER) || - (entry->ib1 & MTK_FOE_IB1_BIND_VLAN_TAG)) + if (!(entry->ib1 & mtk_get_ib1_vlan_layer_mask(eth)) || + (entry->ib1 & mtk_get_ib1_vlan_tag_mask(eth))) l2->etype = ETH_P_PPP_SES; - entry->ib1 |= MTK_FOE_IB1_BIND_PPPOE; + entry->ib1 |= mtk_get_ib1_ppoe_mask(eth); l2->pppoe_id = sid; return 0; } -static inline bool mtk_foe_entry_usable(struct mtk_foe_entry *entry) +int mtk_foe_entry_set_wdma(struct mtk_eth *eth, struct mtk_foe_entry *entry, + int wdma_idx, int txq, int bss, int wcid) { - return !(entry->ib1 & MTK_FOE_IB1_STATIC) && - FIELD_GET(MTK_FOE_IB1_STATE, entry->ib1) != MTK_FOE_STATE_BIND; + struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); + u32 *ib2 = mtk_foe_entry_ib2(eth, entry); + + if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + *ib2 &= ~MTK_FOE_IB2_PORT_MG_V2; + *ib2 |= FIELD_PREP(MTK_FOE_IB2_RX_IDX, txq) | + MTK_FOE_IB2_WDMA_WINFO_V2; + l2->winfo = FIELD_PREP(MTK_FOE_WINFO_WCID, wcid) | + FIELD_PREP(MTK_FOE_WINFO_BSS, bss); + } else { + *ib2 &= ~MTK_FOE_IB2_PORT_MG; + *ib2 |= MTK_FOE_IB2_WDMA_WINFO; + if (wdma_idx) + *ib2 |= MTK_FOE_IB2_WDMA_DEVIDX; + l2->vlan2 = FIELD_PREP(MTK_FOE_VLAN2_WINFO_BSS, bss) | + FIELD_PREP(MTK_FOE_VLAN2_WINFO_WCID, wcid) | + FIELD_PREP(MTK_FOE_VLAN2_WINFO_RING, txq); + } + + return 0; } -int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry, - u16 timestamp) +static bool +mtk_flow_entry_match(struct mtk_eth *eth, struct mtk_flow_entry *entry, + struct mtk_foe_entry *data) { + int type, len; + + if ((data->ib1 ^ entry->data.ib1) & MTK_FOE_IB1_UDP) + return false; + + type = mtk_get_ib1_pkt_type(eth, entry->data.ib1); + if (type > MTK_PPE_PKT_TYPE_IPV4_DSLITE) + len = offsetof(struct mtk_foe_entry, ipv6._rsv); + else + len = offsetof(struct mtk_foe_entry, ipv4.ib2); + + return !memcmp(&entry->data.data, &data->data, len - 4); +} + +static void +__mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) +{ + struct hlist_head *head; + struct hlist_node *tmp; + + if (entry->type == MTK_FLOW_TYPE_L2) { + rhashtable_remove_fast(&ppe->l2_flows, &entry->l2_node, + mtk_flow_l2_ht_params); + + head = &entry->l2_flows; + hlist_for_each_entry_safe(entry, tmp, head, l2_data.list) + __mtk_foe_entry_clear(ppe, entry); + return; + } + + hlist_del_init(&entry->list); + if (entry->hash != 0xffff) { + struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, entry->hash); + + hwe->ib1 &= ~MTK_FOE_IB1_STATE; + hwe->ib1 |= FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_INVALID); + dma_wmb(); + } + entry->hash = 0xffff; + + if (entry->type != MTK_FLOW_TYPE_L2_SUBFLOW) + return; + + hlist_del_init(&entry->l2_data.list); + kfree(entry); +} + +static int __mtk_foe_entry_idle_time(struct mtk_ppe *ppe, u32 ib1) +{ + u32 ib1_ts_mask = mtk_get_ib1_ts_mask(ppe->eth); + u16 now = mtk_eth_timestamp(ppe->eth); + u16 timestamp = ib1 & ib1_ts_mask; + + if (timestamp > now) + return ib1_ts_mask + 1 - timestamp + now; + else + return now - timestamp; +} + +static void +mtk_flow_entry_update_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) +{ + u32 ib1_ts_mask = mtk_get_ib1_ts_mask(ppe->eth); + struct mtk_flow_entry *cur; struct mtk_foe_entry *hwe; - u32 hash; + struct hlist_node *tmp; + int idle; + + idle = __mtk_foe_entry_idle_time(ppe, entry->data.ib1); + hlist_for_each_entry_safe(cur, tmp, &entry->l2_flows, l2_data.list) { + int cur_idle; + u32 ib1; + + hwe = mtk_foe_get_entry(ppe, cur->hash); + ib1 = READ_ONCE(hwe->ib1); + + if (FIELD_GET(MTK_FOE_IB1_STATE, ib1) != MTK_FOE_STATE_BIND) { + cur->hash = 0xffff; + __mtk_foe_entry_clear(ppe, cur); + continue; + } + + cur_idle = __mtk_foe_entry_idle_time(ppe, ib1); + if (cur_idle >= idle) + continue; + + idle = cur_idle; + entry->data.ib1 &= ~ib1_ts_mask; + entry->data.ib1 |= hwe->ib1 & ib1_ts_mask; + } +} + +static void +mtk_flow_entry_update(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) +{ + struct mtk_foe_entry foe = {}; + struct mtk_foe_entry *hwe; + + spin_lock_bh(&ppe_lock); + + if (entry->type == MTK_FLOW_TYPE_L2) { + mtk_flow_entry_update_l2(ppe, entry); + goto out; + } - timestamp &= MTK_FOE_IB1_BIND_TIMESTAMP; - entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP; - entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP, timestamp); + if (entry->hash == 0xffff) + goto out; - hash = mtk_ppe_hash_entry(entry); - hwe = &ppe->foe_table[hash]; - if (!mtk_foe_entry_usable(hwe)) { - hwe++; - hash++; + hwe = mtk_foe_get_entry(ppe, entry->hash); + memcpy(&foe, hwe, ppe->eth->soc->foe_entry_size); + if (!mtk_flow_entry_match(ppe->eth, entry, &foe)) { + entry->hash = 0xffff; + goto out; + } + + entry->data.ib1 = foe.ib1; + +out: + spin_unlock_bh(&ppe_lock); +} - if (!mtk_foe_entry_usable(hwe)) - return -ENOSPC; +static void +__mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry, + u16 hash) +{ + struct mtk_eth *eth = ppe->eth; + u16 timestamp = mtk_eth_timestamp(eth); + struct mtk_foe_entry *hwe; + + if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP_V2; + entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP_V2, + timestamp); + } else { + entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP; + entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP, + timestamp); } - memcpy(&hwe->data, &entry->data, sizeof(hwe->data)); + hwe = mtk_foe_get_entry(ppe, hash); + memcpy(&hwe->data, &entry->data, eth->soc->foe_entry_size - sizeof(hwe->ib1)); wmb(); hwe->ib1 = entry->ib1; dma_wmb(); mtk_ppe_cache_clear(ppe); +} - return hash; +void mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) +{ + spin_lock_bh(&ppe_lock); + __mtk_foe_entry_clear(ppe, entry); + spin_unlock_bh(&ppe_lock); +} + +static int +mtk_foe_entry_commit_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) +{ + entry->type = MTK_FLOW_TYPE_L2; + + return rhashtable_insert_fast(&ppe->l2_flows, &entry->l2_node, + mtk_flow_l2_ht_params); +} + +int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) +{ + const struct mtk_soc_data *soc = ppe->eth->soc; + int type = mtk_get_ib1_pkt_type(ppe->eth, entry->data.ib1); + u32 hash; + + if (type == MTK_PPE_PKT_TYPE_BRIDGE) + return mtk_foe_entry_commit_l2(ppe, entry); + + hash = mtk_ppe_hash_entry(ppe->eth, &entry->data); + entry->hash = 0xffff; + spin_lock_bh(&ppe_lock); + hlist_add_head(&entry->list, &ppe->foe_flow[hash / soc->hash_offset]); + spin_unlock_bh(&ppe_lock); + + return 0; +} + +static void +mtk_foe_entry_commit_subflow(struct mtk_ppe *ppe, struct mtk_flow_entry *entry, + u16 hash) +{ + const struct mtk_soc_data *soc = ppe->eth->soc; + struct mtk_flow_entry *flow_info; + struct mtk_foe_entry foe = {}, *hwe; + struct mtk_foe_mac_info *l2; + u32 ib1_mask = mtk_get_ib1_pkt_type_mask(ppe->eth) | MTK_FOE_IB1_UDP; + int type; + + flow_info = kzalloc(offsetof(struct mtk_flow_entry, l2_data.end), + GFP_ATOMIC); + if (!flow_info) + return; + + flow_info->l2_data.base_flow = entry; + flow_info->type = MTK_FLOW_TYPE_L2_SUBFLOW; + flow_info->hash = hash; + hlist_add_head(&flow_info->list, + &ppe->foe_flow[hash / soc->hash_offset]); + hlist_add_head(&flow_info->l2_data.list, &entry->l2_flows); + + hwe = mtk_foe_get_entry(ppe, hash); + memcpy(&foe, hwe, soc->foe_entry_size); + foe.ib1 &= ib1_mask; + foe.ib1 |= entry->data.ib1 & ~ib1_mask; + + l2 = mtk_foe_entry_l2(ppe->eth, &foe); + memcpy(l2, &entry->data.bridge.l2, sizeof(*l2)); + + type = mtk_get_ib1_pkt_type(ppe->eth, foe.ib1); + if (type == MTK_PPE_PKT_TYPE_IPV4_HNAPT) + memcpy(&foe.ipv4.new, &foe.ipv4.orig, sizeof(foe.ipv4.new)); + else if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T && l2->etype == ETH_P_IP) + l2->etype = ETH_P_IPV6; + + *mtk_foe_entry_ib2(ppe->eth, &foe) = entry->data.bridge.ib2; + + __mtk_foe_entry_commit(ppe, &foe, hash); +} + +void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash) +{ + const struct mtk_soc_data *soc = ppe->eth->soc; + struct hlist_head *head = &ppe->foe_flow[hash / soc->hash_offset]; + struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, hash); + struct mtk_flow_entry *entry; + struct mtk_foe_bridge key = {}; + struct hlist_node *n; + struct ethhdr *eh; + bool found = false; + u8 *tag; + + spin_lock_bh(&ppe_lock); + + if (FIELD_GET(MTK_FOE_IB1_STATE, hwe->ib1) == MTK_FOE_STATE_BIND) + goto out; + + hlist_for_each_entry_safe(entry, n, head, list) { + if (entry->type == MTK_FLOW_TYPE_L2_SUBFLOW) { + if (unlikely(FIELD_GET(MTK_FOE_IB1_STATE, hwe->ib1) == + MTK_FOE_STATE_BIND)) + continue; + + entry->hash = 0xffff; + __mtk_foe_entry_clear(ppe, entry); + continue; + } + + if (found || !mtk_flow_entry_match(ppe->eth, entry, hwe)) { + if (entry->hash != 0xffff) + entry->hash = 0xffff; + continue; + } + + entry->hash = hash; + __mtk_foe_entry_commit(ppe, &entry->data, hash); + found = true; + } + + if (found) + goto out; + + eh = eth_hdr(skb); + ether_addr_copy(key.dest_mac, eh->h_dest); + ether_addr_copy(key.src_mac, eh->h_source); + tag = skb->data - 2; + key.vlan = 0; + switch (skb->protocol) { +#if IS_ENABLED(CONFIG_NET_DSA) + case htons(ETH_P_XDSA): + if (!netdev_uses_dsa(skb->dev) || + skb->dev->dsa_ptr->tag_ops->proto != DSA_TAG_PROTO_MTK) + goto out; + + tag += 4; + if (get_unaligned_be16(tag) != ETH_P_8021Q) + break; + + fallthrough; +#endif + case htons(ETH_P_8021Q): + key.vlan = get_unaligned_be16(tag + 2) & VLAN_VID_MASK; + break; + default: + break; + } + + entry = rhashtable_lookup_fast(&ppe->l2_flows, &key, mtk_flow_l2_ht_params); + if (!entry) + goto out; + + mtk_foe_entry_commit_subflow(ppe, entry, hash); + +out: + spin_unlock_bh(&ppe_lock); +} + +int mtk_foe_entry_idle_time(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) +{ + mtk_flow_entry_update(ppe, entry); + + return __mtk_foe_entry_idle_time(ppe, entry->data.ib1); } -int mtk_ppe_init(struct mtk_ppe *ppe, struct device *dev, void __iomem *base, - int version) +struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, + int version, int index) { - struct mtk_foe_entry *foe; + const struct mtk_soc_data *soc = eth->soc; + struct device *dev = eth->dev; + struct mtk_ppe *ppe; + u32 foe_flow_size; + void *foe; + + ppe = devm_kzalloc(dev, sizeof(*ppe), GFP_KERNEL); + if (!ppe) + return NULL; + + rhashtable_init(&ppe->l2_flows, &mtk_flow_l2_ht_params); /* need to allocate a separate device, since it PPE DMA access is * not coherent. */ ppe->base = base; + ppe->eth = eth; ppe->dev = dev; ppe->version = version; - foe = dmam_alloc_coherent(ppe->dev, MTK_PPE_ENTRIES * sizeof(*foe), + foe = dmam_alloc_coherent(ppe->dev, + MTK_PPE_ENTRIES * soc->foe_entry_size, &ppe->foe_phys, GFP_KERNEL); if (!foe) - return -ENOMEM; + return NULL; ppe->foe_table = foe; - mtk_ppe_debugfs_init(ppe); + foe_flow_size = (MTK_PPE_ENTRIES / soc->hash_offset) * + sizeof(*ppe->foe_flow); + ppe->foe_flow = devm_kzalloc(dev, foe_flow_size, GFP_KERNEL); + if (!ppe->foe_flow) + return NULL; - return 0; + mtk_ppe_debugfs_init(ppe, index); + + return ppe; } static void mtk_ppe_init_foe_table(struct mtk_ppe *ppe) @@ -395,21 +757,30 @@ static void mtk_ppe_init_foe_table(struct mtk_ppe *ppe) static const u8 skip[] = { 12, 25, 38, 51, 76, 89, 102 }; int i, k; - memset(ppe->foe_table, 0, MTK_PPE_ENTRIES * sizeof(ppe->foe_table)); + memset(ppe->foe_table, 0, + MTK_PPE_ENTRIES * ppe->eth->soc->foe_entry_size); if (!IS_ENABLED(CONFIG_SOC_MT7621)) return; /* skip all entries that cross the 1024 byte boundary */ - for (i = 0; i < MTK_PPE_ENTRIES; i += 128) - for (k = 0; k < ARRAY_SIZE(skip); k++) - ppe->foe_table[i + skip[k]].ib1 |= MTK_FOE_IB1_STATIC; + for (i = 0; i < MTK_PPE_ENTRIES; i += 128) { + for (k = 0; k < ARRAY_SIZE(skip); k++) { + struct mtk_foe_entry *hwe; + + hwe = mtk_foe_get_entry(ppe, i + skip[k]); + hwe->ib1 |= MTK_FOE_IB1_STATIC; + } + } } -int mtk_ppe_start(struct mtk_ppe *ppe) +void mtk_ppe_start(struct mtk_ppe *ppe) { u32 val; + if (!ppe) + return; + mtk_ppe_init_foe_table(ppe); ppe_w32(ppe, MTK_PPE_TB_BASE, ppe->foe_phys); @@ -428,6 +799,8 @@ int mtk_ppe_start(struct mtk_ppe *ppe) MTK_PPE_SCAN_MODE_KEEPALIVE_AGE) | FIELD_PREP(MTK_PPE_TB_CFG_ENTRY_NUM, MTK_PPE_ENTRIES_SHIFT); + if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) + val |= MTK_PPE_TB_CFG_INFO_SEL; ppe_w32(ppe, MTK_PPE_TB_CFG, val); ppe_w32(ppe, MTK_PPE_IP_PROTO_CHK, @@ -435,16 +808,21 @@ int mtk_ppe_start(struct mtk_ppe *ppe) mtk_ppe_cache_enable(ppe, true); - val = MTK_PPE_FLOW_CFG_IP4_TCP_FRAG | - MTK_PPE_FLOW_CFG_IP4_UDP_FRAG | - MTK_PPE_FLOW_CFG_IP6_3T_ROUTE | + val = MTK_PPE_FLOW_CFG_IP6_3T_ROUTE | MTK_PPE_FLOW_CFG_IP6_5T_ROUTE | MTK_PPE_FLOW_CFG_IP6_6RD | MTK_PPE_FLOW_CFG_IP4_NAT | MTK_PPE_FLOW_CFG_IP4_NAPT | MTK_PPE_FLOW_CFG_IP4_DSLITE | - MTK_PPE_FLOW_CFG_L2_BRIDGE | MTK_PPE_FLOW_CFG_IP4_NAT_FRAG; + if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) + val |= MTK_PPE_MD_TOAP_BYP_CRSN0 | + MTK_PPE_MD_TOAP_BYP_CRSN1 | + MTK_PPE_MD_TOAP_BYP_CRSN2 | + MTK_PPE_FLOW_CFG_IP4_HASH_GRE_KEY; + else + val |= MTK_PPE_FLOW_CFG_IP4_TCP_FRAG | + MTK_PPE_FLOW_CFG_IP4_UDP_FRAG; ppe_w32(ppe, MTK_PPE_FLOW_CFG, val); val = FIELD_PREP(MTK_PPE_UNBIND_AGE_MIN_PACKETS, 1000) | @@ -479,7 +857,10 @@ int mtk_ppe_start(struct mtk_ppe *ppe) ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT, 0); - return 0; + if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) { + ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT1, 0xcb777); + ppe_w32(ppe, MTK_PPE_SBW_CTRL, 0x7f); + } } int mtk_ppe_stop(struct mtk_ppe *ppe) @@ -487,9 +868,15 @@ int mtk_ppe_stop(struct mtk_ppe *ppe) u32 val; int i; - for (i = 0; i < MTK_PPE_ENTRIES; i++) - ppe->foe_table[i].ib1 = FIELD_PREP(MTK_FOE_IB1_STATE, - MTK_FOE_STATE_INVALID); + if (!ppe) + return 0; + + for (i = 0; i < MTK_PPE_ENTRIES; i++) { + struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, i); + + hwe->ib1 = FIELD_PREP(MTK_FOE_IB1_STATE, + MTK_FOE_STATE_INVALID); + } mtk_ppe_cache_enable(ppe, false); |