diff options
Diffstat (limited to 'drivers/net/ethernet/chelsio/cxgb4/sge.c')
-rw-r--r-- | drivers/net/ethernet/chelsio/cxgb4/sge.c | 323 |
1 files changed, 235 insertions, 88 deletions
diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c index 922f2f937789..6e310a0da7c9 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/sge.c +++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c @@ -41,6 +41,7 @@ #include <linux/jiffies.h> #include <linux/prefetch.h> #include <linux/export.h> +#include <net/xfrm.h> #include <net/ipv6.h> #include <net/tcp.h> #include <net/busy_poll.h> @@ -53,6 +54,7 @@ #include "t4_msg.h" #include "t4fw_api.h" #include "cxgb4_ptp.h" +#include "cxgb4_uld.h" /* * Rx buffer size. We use largish buffers if possible but settle for single @@ -110,14 +112,6 @@ #define NOMEM_TMR_IDX (SGE_NTIMERS - 1) /* - * Suspend an Ethernet Tx queue with fewer available descriptors than this. - * This is the same as calc_tx_descs() for a TSO packet with - * nr_frags == MAX_SKB_FRAGS. - */ -#define ETHTXQ_STOP_THRES \ - (1 + DIV_ROUND_UP((3 * MAX_SKB_FRAGS) / 2 + (MAX_SKB_FRAGS & 1), 8)) - -/* * Suspension threshold for non-Ethernet Tx queues. We require enough room * for a full sized WR. */ @@ -134,11 +128,6 @@ */ #define MAX_CTRL_WR_LEN SGE_MAX_WR_LEN -struct tx_sw_desc { /* SW state per Tx descriptor */ - struct sk_buff *skb; - struct ulptx_sgl *sgl; -}; - struct rx_sw_desc { /* SW state per Rx descriptor */ struct page *page; dma_addr_t dma_addr; @@ -248,8 +237,8 @@ static inline bool fl_starving(const struct adapter *adapter, return fl->avail - fl->pend_cred <= s->fl_starve_thres; } -static int map_skb(struct device *dev, const struct sk_buff *skb, - dma_addr_t *addr) +int cxgb4_map_skb(struct device *dev, const struct sk_buff *skb, + dma_addr_t *addr) { const skb_frag_t *fp, *end; const struct skb_shared_info *si; @@ -277,6 +266,7 @@ unwind: out_err: return -ENOMEM; } +EXPORT_SYMBOL(cxgb4_map_skb); #ifdef CONFIG_NEED_DMA_MAP_STATE static void unmap_skb(struct device *dev, const struct sk_buff *skb, @@ -411,7 +401,7 @@ static inline int reclaimable(const struct sge_txq *q) } /** - * reclaim_completed_tx - reclaims completed Tx descriptors + * cxgb4_reclaim_completed_tx - reclaims completed Tx descriptors * @adap: the adapter * @q: the Tx queue to reclaim completed descriptors from * @unmap: whether the buffers should be unmapped for DMA @@ -420,7 +410,7 @@ static inline int reclaimable(const struct sge_txq *q) * and frees the associated buffers if possible. Called with the Tx * queue locked. */ -static inline void reclaim_completed_tx(struct adapter *adap, struct sge_txq *q, +inline void cxgb4_reclaim_completed_tx(struct adapter *adap, struct sge_txq *q, bool unmap) { int avail = reclaimable(q); @@ -437,6 +427,7 @@ static inline void reclaim_completed_tx(struct adapter *adap, struct sge_txq *q, q->in_use -= avail; } } +EXPORT_SYMBOL(cxgb4_reclaim_completed_tx); static inline int get_buf_size(struct adapter *adapter, const struct rx_sw_desc *d) @@ -770,12 +761,19 @@ static inline unsigned int flits_to_desc(unsigned int n) * Returns whether an Ethernet packet is small enough to fit as * immediate data. Return value corresponds to headroom required. */ -static inline int is_eth_imm(const struct sk_buff *skb) +static inline int is_eth_imm(const struct sk_buff *skb, unsigned int chip_ver) { - int hdrlen = skb_shinfo(skb)->gso_size ? - sizeof(struct cpl_tx_pkt_lso_core) : 0; + int hdrlen = 0; - hdrlen += sizeof(struct cpl_tx_pkt); + if (skb->encapsulation && skb_shinfo(skb)->gso_size && + chip_ver > CHELSIO_T5) { + hdrlen = sizeof(struct cpl_tx_tnl_lso); + hdrlen += sizeof(struct cpl_tx_pkt_core); + } else { + hdrlen = skb_shinfo(skb)->gso_size ? + sizeof(struct cpl_tx_pkt_lso_core) : 0; + hdrlen += sizeof(struct cpl_tx_pkt); + } if (skb->len <= MAX_IMM_TX_PKT_LEN - hdrlen) return hdrlen; return 0; @@ -788,10 +786,11 @@ static inline int is_eth_imm(const struct sk_buff *skb) * Returns the number of flits needed for a Tx WR for the given Ethernet * packet, including the needed WR and CPL headers. */ -static inline unsigned int calc_tx_flits(const struct sk_buff *skb) +static inline unsigned int calc_tx_flits(const struct sk_buff *skb, + unsigned int chip_ver) { unsigned int flits; - int hdrlen = is_eth_imm(skb); + int hdrlen = is_eth_imm(skb, chip_ver); /* If the skb is small enough, we can pump it out as a work request * with only immediate data. In that case we just have to have the @@ -810,13 +809,20 @@ static inline unsigned int calc_tx_flits(const struct sk_buff *skb) * with an embedded TX Packet Write CPL message. */ flits = sgl_len(skb_shinfo(skb)->nr_frags + 1); - if (skb_shinfo(skb)->gso_size) - flits += (sizeof(struct fw_eth_tx_pkt_wr) + - sizeof(struct cpl_tx_pkt_lso_core) + - sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64); - else + if (skb_shinfo(skb)->gso_size) { + if (skb->encapsulation && chip_ver > CHELSIO_T5) + hdrlen = sizeof(struct fw_eth_tx_pkt_wr) + + sizeof(struct cpl_tx_tnl_lso); + else + hdrlen = sizeof(struct fw_eth_tx_pkt_wr) + + sizeof(struct cpl_tx_pkt_lso_core); + + hdrlen += sizeof(struct cpl_tx_pkt_core); + flits += (hdrlen / sizeof(__be64)); + } else { flits += (sizeof(struct fw_eth_tx_pkt_wr) + sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64); + } return flits; } @@ -827,13 +833,14 @@ static inline unsigned int calc_tx_flits(const struct sk_buff *skb) * Returns the number of Tx descriptors needed for the given Ethernet * packet, including the needed WR and CPL headers. */ -static inline unsigned int calc_tx_descs(const struct sk_buff *skb) +static inline unsigned int calc_tx_descs(const struct sk_buff *skb, + unsigned int chip_ver) { - return flits_to_desc(calc_tx_flits(skb)); + return flits_to_desc(calc_tx_flits(skb, chip_ver)); } /** - * write_sgl - populate a scatter/gather list for a packet + * cxgb4_write_sgl - populate a scatter/gather list for a packet * @skb: the packet * @q: the Tx queue we are writing into * @sgl: starting location for writing the SGL @@ -849,9 +856,9 @@ static inline unsigned int calc_tx_descs(const struct sk_buff *skb) * right after the end of the SGL but does not account for any potential * wrap around, i.e., @end > @sgl. */ -static void write_sgl(const struct sk_buff *skb, struct sge_txq *q, - struct ulptx_sgl *sgl, u64 *end, unsigned int start, - const dma_addr_t *addr) +void cxgb4_write_sgl(const struct sk_buff *skb, struct sge_txq *q, + struct ulptx_sgl *sgl, u64 *end, unsigned int start, + const dma_addr_t *addr) { unsigned int i, len; struct ulptx_sge_pair *to; @@ -903,6 +910,7 @@ static void write_sgl(const struct sk_buff *skb, struct sge_txq *q, if ((uintptr_t)end & 8) /* 0-pad to multiple of 16 */ *end = 0; } +EXPORT_SYMBOL(cxgb4_write_sgl); /* This function copies 64 byte coalesced work request to * memory mapped BAR2 space. For coalesced WR SGE fetches @@ -921,14 +929,14 @@ static void cxgb_pio_copy(u64 __iomem *dst, u64 *src) } /** - * ring_tx_db - check and potentially ring a Tx queue's doorbell + * cxgb4_ring_tx_db - check and potentially ring a Tx queue's doorbell * @adap: the adapter * @q: the Tx queue * @n: number of new descriptors to give to HW * * Ring the doorbel for a Tx queue. */ -static inline void ring_tx_db(struct adapter *adap, struct sge_txq *q, int n) +inline void cxgb4_ring_tx_db(struct adapter *adap, struct sge_txq *q, int n) { /* Make sure that all writes to the TX Descriptors are committed * before we tell the hardware about them. @@ -995,9 +1003,10 @@ static inline void ring_tx_db(struct adapter *adap, struct sge_txq *q, int n) wmb(); } } +EXPORT_SYMBOL(cxgb4_ring_tx_db); /** - * inline_tx_skb - inline a packet's data into Tx descriptors + * cxgb4_inline_tx_skb - inline a packet's data into Tx descriptors * @skb: the packet * @q: the Tx queue where the packet will be inlined * @pos: starting position in the Tx queue where to inline the packet @@ -1007,8 +1016,8 @@ static inline void ring_tx_db(struct adapter *adap, struct sge_txq *q, int n) * Most of the complexity of this operation is dealing with wrap arounds * in the middle of the packet we want to inline. */ -static void inline_tx_skb(const struct sk_buff *skb, const struct sge_txq *q, - void *pos) +void cxgb4_inline_tx_skb(const struct sk_buff *skb, + const struct sge_txq *q, void *pos) { u64 *p; int left = (void *)q->stat - pos; @@ -1030,6 +1039,7 @@ static void inline_tx_skb(const struct sk_buff *skb, const struct sge_txq *q, if ((uintptr_t)p & 8) *p = 0; } +EXPORT_SYMBOL(cxgb4_inline_tx_skb); static void *inline_tx_skb_header(const struct sk_buff *skb, const struct sge_txq *q, void *pos, @@ -1154,6 +1164,105 @@ cxgb_fcoe_offload(struct sk_buff *skb, struct adapter *adap, } #endif /* CONFIG_CHELSIO_T4_FCOE */ +/* Returns tunnel type if hardware supports offloading of the same. + * It is called only for T5 and onwards. + */ +enum cpl_tx_tnl_lso_type cxgb_encap_offload_supported(struct sk_buff *skb) +{ + u8 l4_hdr = 0; + enum cpl_tx_tnl_lso_type tnl_type = TX_TNL_TYPE_OPAQUE; + struct port_info *pi = netdev_priv(skb->dev); + struct adapter *adapter = pi->adapter; + + if (skb->inner_protocol_type != ENCAP_TYPE_ETHER || + skb->inner_protocol != htons(ETH_P_TEB)) + return tnl_type; + + switch (vlan_get_protocol(skb)) { + case htons(ETH_P_IP): + l4_hdr = ip_hdr(skb)->protocol; + break; + case htons(ETH_P_IPV6): + l4_hdr = ipv6_hdr(skb)->nexthdr; + break; + default: + return tnl_type; + } + + switch (l4_hdr) { + case IPPROTO_UDP: + if (adapter->vxlan_port == udp_hdr(skb)->dest) + tnl_type = TX_TNL_TYPE_VXLAN; + else if (adapter->geneve_port == udp_hdr(skb)->dest) + tnl_type = TX_TNL_TYPE_GENEVE; + break; + default: + return tnl_type; + } + + return tnl_type; +} + +static inline void t6_fill_tnl_lso(struct sk_buff *skb, + struct cpl_tx_tnl_lso *tnl_lso, + enum cpl_tx_tnl_lso_type tnl_type) +{ + u32 val; + int in_eth_xtra_len; + int l3hdr_len = skb_network_header_len(skb); + int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN; + const struct skb_shared_info *ssi = skb_shinfo(skb); + bool v6 = (ip_hdr(skb)->version == 6); + + val = CPL_TX_TNL_LSO_OPCODE_V(CPL_TX_TNL_LSO) | + CPL_TX_TNL_LSO_FIRST_F | + CPL_TX_TNL_LSO_LAST_F | + (v6 ? CPL_TX_TNL_LSO_IPV6OUT_F : 0) | + CPL_TX_TNL_LSO_ETHHDRLENOUT_V(eth_xtra_len / 4) | + CPL_TX_TNL_LSO_IPHDRLENOUT_V(l3hdr_len / 4) | + (v6 ? 0 : CPL_TX_TNL_LSO_IPHDRCHKOUT_F) | + CPL_TX_TNL_LSO_IPLENSETOUT_F | + (v6 ? 0 : CPL_TX_TNL_LSO_IPIDINCOUT_F); + tnl_lso->op_to_IpIdSplitOut = htonl(val); + + tnl_lso->IpIdOffsetOut = 0; + + /* Get the tunnel header length */ + val = skb_inner_mac_header(skb) - skb_mac_header(skb); + in_eth_xtra_len = skb_inner_network_header(skb) - + skb_inner_mac_header(skb) - ETH_HLEN; + + switch (tnl_type) { + case TX_TNL_TYPE_VXLAN: + case TX_TNL_TYPE_GENEVE: + tnl_lso->UdpLenSetOut_to_TnlHdrLen = + htons(CPL_TX_TNL_LSO_UDPCHKCLROUT_F | + CPL_TX_TNL_LSO_UDPLENSETOUT_F); + break; + default: + tnl_lso->UdpLenSetOut_to_TnlHdrLen = 0; + break; + } + + tnl_lso->UdpLenSetOut_to_TnlHdrLen |= + htons(CPL_TX_TNL_LSO_TNLHDRLEN_V(val) | + CPL_TX_TNL_LSO_TNLTYPE_V(tnl_type)); + + tnl_lso->r1 = 0; + + val = CPL_TX_TNL_LSO_ETHHDRLEN_V(in_eth_xtra_len / 4) | + CPL_TX_TNL_LSO_IPV6_V(inner_ip_hdr(skb)->version == 6) | + CPL_TX_TNL_LSO_IPHDRLEN_V(skb_inner_network_header_len(skb) / 4) | + CPL_TX_TNL_LSO_TCPHDRLEN_V(inner_tcp_hdrlen(skb) / 4); + tnl_lso->Flow_to_TcpHdrLen = htonl(val); + + tnl_lso->IpIdOffset = htons(0); + + tnl_lso->IpIdSplit_to_Mss = htons(CPL_TX_TNL_LSO_MSS_V(ssi->gso_size)); + tnl_lso->TCPSeqOffset = htonl(0); + tnl_lso->EthLenOffset_Size = htonl(CPL_TX_TNL_LSO_SIZE_V(skb->len)); +} + /** * t4_eth_xmit - add a packet to an Ethernet Tx queue * @skb: the packet @@ -1177,6 +1286,9 @@ netdev_tx_t t4_eth_xmit(struct sk_buff *skb, struct net_device *dev) bool immediate = false; int len, max_pkt_len; bool ptp_enabled = is_ptp_enabled(skb, dev); + unsigned int chip_ver; + enum cpl_tx_tnl_lso_type tnl_type = TX_TNL_TYPE_OPAQUE; + #ifdef CONFIG_CHELSIO_T4_FCOE int err; #endif /* CONFIG_CHELSIO_T4_FCOE */ @@ -1199,6 +1311,12 @@ out_free: dev_kfree_skb_any(skb); pi = netdev_priv(dev); adap = pi->adapter; + ssi = skb_shinfo(skb); +#ifdef CONFIG_CHELSIO_IPSEC_INLINE + if (xfrm_offload(skb) && !ssi->gso_size) + return adap->uld[CXGB4_ULD_CRYPTO].tx_handler(skb, dev); +#endif /* CHELSIO_IPSEC_INLINE */ + qidx = skb_get_queue_mapping(skb); if (ptp_enabled) { spin_lock(&adap->ptp_lock); @@ -1215,7 +1333,7 @@ out_free: dev_kfree_skb_any(skb); } skb_tx_timestamp(skb); - reclaim_completed_tx(adap, &q->q, true); + cxgb4_reclaim_completed_tx(adap, &q->q, true); cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F; #ifdef CONFIG_CHELSIO_T4_FCOE @@ -1227,7 +1345,8 @@ out_free: dev_kfree_skb_any(skb); } #endif /* CONFIG_CHELSIO_T4_FCOE */ - flits = calc_tx_flits(skb); + chip_ver = CHELSIO_CHIP_VERSION(adap->params.chip); + flits = calc_tx_flits(skb, chip_ver); ndesc = flits_to_desc(flits); credits = txq_avail(&q->q) - ndesc; @@ -1241,11 +1360,14 @@ out_free: dev_kfree_skb_any(skb); return NETDEV_TX_BUSY; } - if (is_eth_imm(skb)) + if (is_eth_imm(skb, chip_ver)) immediate = true; + if (skb->encapsulation && chip_ver > CHELSIO_T5) + tnl_type = cxgb_encap_offload_supported(skb); + if (!immediate && - unlikely(map_skb(adap->pdev_dev, skb, addr) < 0)) { + unlikely(cxgb4_map_skb(adap->pdev_dev, skb, addr) < 0)) { q->mapping_err++; if (ptp_enabled) spin_unlock(&adap->ptp_lock); @@ -1264,39 +1386,63 @@ out_free: dev_kfree_skb_any(skb); end = (u64 *)wr + flits; len = immediate ? skb->len : 0; - ssi = skb_shinfo(skb); if (ssi->gso_size) { struct cpl_tx_pkt_lso *lso = (void *)wr; bool v6 = (ssi->gso_type & SKB_GSO_TCPV6) != 0; int l3hdr_len = skb_network_header_len(skb); int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN; + struct cpl_tx_tnl_lso *tnl_lso = (void *)(wr + 1); + + if (tnl_type) + len += sizeof(*tnl_lso); + else + len += sizeof(*lso); - len += sizeof(*lso); wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) | FW_WR_IMMDLEN_V(len)); - lso->c.lso_ctrl = htonl(LSO_OPCODE_V(CPL_TX_PKT_LSO) | - LSO_FIRST_SLICE_F | LSO_LAST_SLICE_F | - LSO_IPV6_V(v6) | - LSO_ETHHDR_LEN_V(eth_xtra_len / 4) | - LSO_IPHDR_LEN_V(l3hdr_len / 4) | - LSO_TCPHDR_LEN_V(tcp_hdr(skb)->doff)); - lso->c.ipid_ofst = htons(0); - lso->c.mss = htons(ssi->gso_size); - lso->c.seqno_offset = htonl(0); - if (is_t4(adap->params.chip)) - lso->c.len = htonl(skb->len); - else - lso->c.len = htonl(LSO_T5_XFER_SIZE_V(skb->len)); - cpl = (void *)(lso + 1); + if (tnl_type) { + struct iphdr *iph = ip_hdr(skb); - if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) - cntrl = TXPKT_ETHHDR_LEN_V(eth_xtra_len); - else - cntrl = T6_TXPKT_ETHHDR_LEN_V(eth_xtra_len); + t6_fill_tnl_lso(skb, tnl_lso, tnl_type); + cpl = (void *)(tnl_lso + 1); + /* Driver is expected to compute partial checksum that + * does not include the IP Total Length. + */ + if (iph->version == 4) { + iph->check = 0; + iph->tot_len = 0; + iph->check = (u16)(~ip_fast_csum((u8 *)iph, + iph->ihl)); + } + if (skb->ip_summed == CHECKSUM_PARTIAL) + cntrl = hwcsum(adap->params.chip, skb); + } else { + lso->c.lso_ctrl = htonl(LSO_OPCODE_V(CPL_TX_PKT_LSO) | + LSO_FIRST_SLICE_F | LSO_LAST_SLICE_F | + LSO_IPV6_V(v6) | + LSO_ETHHDR_LEN_V(eth_xtra_len / 4) | + LSO_IPHDR_LEN_V(l3hdr_len / 4) | + LSO_TCPHDR_LEN_V(tcp_hdr(skb)->doff)); + lso->c.ipid_ofst = htons(0); + lso->c.mss = htons(ssi->gso_size); + lso->c.seqno_offset = htonl(0); + if (is_t4(adap->params.chip)) + lso->c.len = htonl(skb->len); + else + lso->c.len = + htonl(LSO_T5_XFER_SIZE_V(skb->len)); + cpl = (void *)(lso + 1); - cntrl |= TXPKT_CSUM_TYPE_V(v6 ? - TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) | - TXPKT_IPHDR_LEN_V(l3hdr_len); + if (CHELSIO_CHIP_VERSION(adap->params.chip) + <= CHELSIO_T5) + cntrl = TXPKT_ETHHDR_LEN_V(eth_xtra_len); + else + cntrl = T6_TXPKT_ETHHDR_LEN_V(eth_xtra_len); + + cntrl |= TXPKT_CSUM_TYPE_V(v6 ? + TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) | + TXPKT_IPHDR_LEN_V(l3hdr_len); + } q->tso++; q->tx_cso += ssi->gso_segs; } else { @@ -1341,13 +1487,13 @@ out_free: dev_kfree_skb_any(skb); cpl->ctrl1 = cpu_to_be64(cntrl); if (immediate) { - inline_tx_skb(skb, &q->q, cpl + 1); + cxgb4_inline_tx_skb(skb, &q->q, cpl + 1); dev_consume_skb_any(skb); } else { int last_desc; - write_sgl(skb, &q->q, (struct ulptx_sgl *)(cpl + 1), end, 0, - addr); + cxgb4_write_sgl(skb, &q->q, (struct ulptx_sgl *)(cpl + 1), + end, 0, addr); skb_orphan(skb); last_desc = q->q.pidx + ndesc - 1; @@ -1359,7 +1505,7 @@ out_free: dev_kfree_skb_any(skb); txq_advance(&q->q, ndesc); - ring_tx_db(adap, &q->q, ndesc); + cxgb4_ring_tx_db(adap, &q->q, ndesc); if (ptp_enabled) spin_unlock(&adap->ptp_lock); return NETDEV_TX_OK; @@ -1369,9 +1515,9 @@ out_free: dev_kfree_skb_any(skb); * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs * @q: the SGE control Tx queue * - * This is a variant of reclaim_completed_tx() that is used for Tx queues - * that send only immediate data (presently just the control queues) and - * thus do not have any sk_buffs to release. + * This is a variant of cxgb4_reclaim_completed_tx() that is used + * for Tx queues that send only immediate data (presently just + * the control queues) and thus do not have any sk_buffs to release. */ static inline void reclaim_completed_tx_imm(struct sge_txq *q) { @@ -1446,13 +1592,13 @@ static int ctrl_xmit(struct sge_ctrl_txq *q, struct sk_buff *skb) } wr = (struct fw_wr_hdr *)&q->q.desc[q->q.pidx]; - inline_tx_skb(skb, &q->q, wr); + cxgb4_inline_tx_skb(skb, &q->q, wr); txq_advance(&q->q, ndesc); if (unlikely(txq_avail(&q->q) < TXQ_STOP_THRES)) ctrlq_check_stop(q, wr); - ring_tx_db(q->adap, &q->q, ndesc); + cxgb4_ring_tx_db(q->adap, &q->q, ndesc); spin_unlock(&q->sendq.lock); kfree_skb(skb); @@ -1487,7 +1633,7 @@ static void restart_ctrlq(unsigned long data) txq_advance(&q->q, ndesc); spin_unlock(&q->sendq.lock); - inline_tx_skb(skb, &q->q, wr); + cxgb4_inline_tx_skb(skb, &q->q, wr); kfree_skb(skb); if (unlikely(txq_avail(&q->q) < TXQ_STOP_THRES)) { @@ -1500,14 +1646,15 @@ static void restart_ctrlq(unsigned long data) } } if (written > 16) { - ring_tx_db(q->adap, &q->q, written); + cxgb4_ring_tx_db(q->adap, &q->q, written); written = 0; } spin_lock(&q->sendq.lock); } q->full = 0; -ringdb: if (written) - ring_tx_db(q->adap, &q->q, written); +ringdb: + if (written) + cxgb4_ring_tx_db(q->adap, &q->q, written); spin_unlock(&q->sendq.lock); } @@ -1650,7 +1797,7 @@ static void service_ofldq(struct sge_uld_txq *q) */ spin_unlock(&q->sendq.lock); - reclaim_completed_tx(q->adap, &q->q, false); + cxgb4_reclaim_completed_tx(q->adap, &q->q, false); flits = skb->priority; /* previously saved */ ndesc = flits_to_desc(flits); @@ -1661,9 +1808,9 @@ static void service_ofldq(struct sge_uld_txq *q) pos = (u64 *)&q->q.desc[q->q.pidx]; if (is_ofld_imm(skb)) - inline_tx_skb(skb, &q->q, pos); - else if (map_skb(q->adap->pdev_dev, skb, - (dma_addr_t *)skb->head)) { + cxgb4_inline_tx_skb(skb, &q->q, pos); + else if (cxgb4_map_skb(q->adap->pdev_dev, skb, + (dma_addr_t *)skb->head)) { txq_stop_maperr(q); spin_lock(&q->sendq.lock); break; @@ -1694,9 +1841,9 @@ static void service_ofldq(struct sge_uld_txq *q) pos = (void *)txq->desc; } - write_sgl(skb, &q->q, (void *)pos, - end, hdr_len, - (dma_addr_t *)skb->head); + cxgb4_write_sgl(skb, &q->q, (void *)pos, + end, hdr_len, + (dma_addr_t *)skb->head); #ifdef CONFIG_NEED_DMA_MAP_STATE skb->dev = q->adap->port[0]; skb->destructor = deferred_unmap_destructor; @@ -1710,7 +1857,7 @@ static void service_ofldq(struct sge_uld_txq *q) txq_advance(&q->q, ndesc); written += ndesc; if (unlikely(written > 32)) { - ring_tx_db(q->adap, &q->q, written); + cxgb4_ring_tx_db(q->adap, &q->q, written); written = 0; } @@ -1725,7 +1872,7 @@ static void service_ofldq(struct sge_uld_txq *q) kfree_skb(skb); } if (likely(written)) - ring_tx_db(q->adap, &q->q, written); + cxgb4_ring_tx_db(q->adap, &q->q, written); /*Indicate that no thread is processing the Pending Send Queue * currently. |