diff options
Diffstat (limited to 'drivers/infiniband/hw/cxgb4')
| -rw-r--r-- | drivers/infiniband/hw/cxgb4/cm.c | 69 | ||||
| -rw-r--r-- | drivers/infiniband/hw/cxgb4/cq.c | 21 | ||||
| -rw-r--r-- | drivers/infiniband/hw/cxgb4/device.c | 142 | ||||
| -rw-r--r-- | drivers/infiniband/hw/cxgb4/iw_cxgb4.h | 30 | ||||
| -rw-r--r-- | drivers/infiniband/hw/cxgb4/provider.c | 33 | ||||
| -rw-r--r-- | drivers/infiniband/hw/cxgb4/qp.c | 149 | ||||
| -rw-r--r-- | drivers/infiniband/hw/cxgb4/t4.h | 2 |
7 files changed, 286 insertions, 160 deletions
diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c index f1510cc76d2d..03a1b0e64fc3 100644 --- a/drivers/infiniband/hw/cxgb4/cm.c +++ b/drivers/infiniband/hw/cxgb4/cm.c @@ -692,6 +692,10 @@ static int send_connect(struct c4iw_ep *ep) int ret; enum chip_type adapter_type = ep->com.dev->rdev.lldi.adapter_type; u32 isn = (prandom_u32() & ~7UL) - 1; + struct net_device *netdev; + u64 params; + + netdev = ep->com.dev->rdev.lldi.ports[0]; switch (CHELSIO_CHIP_VERSION(adapter_type)) { case CHELSIO_T4: @@ -768,6 +772,8 @@ static int send_connect(struct c4iw_ep *ep) opt2 |= T5_ISS_F; } + params = cxgb4_select_ntuple(netdev, ep->l2t); + if (ep->com.remote_addr.ss_family == AF_INET6) cxgb4_clip_get(ep->com.dev->rdev.lldi.ports[0], (const u32 *)&la6->sin6_addr.s6_addr, 1); @@ -809,18 +815,22 @@ static int send_connect(struct c4iw_ep *ep) req->opt0 = cpu_to_be64(opt0); if (is_t4(ep->com.dev->rdev.lldi.adapter_type)) { - req->params = cpu_to_be32(cxgb4_select_ntuple( - ep->com.dev->rdev.lldi.ports[0], - ep->l2t)); + req->params = cpu_to_be32(params); req->opt2 = cpu_to_be32(opt2); } else { - t5req->params = cpu_to_be64(FILTER_TUPLE_V( - cxgb4_select_ntuple( - ep->com.dev->rdev.lldi.ports[0], - ep->l2t))); - t5req->rsvd = cpu_to_be32(isn); - PDBG("%s snd_isn %u\n", __func__, t5req->rsvd); - t5req->opt2 = cpu_to_be32(opt2); + if (is_t5(ep->com.dev->rdev.lldi.adapter_type)) { + t5req->params = + cpu_to_be64(FILTER_TUPLE_V(params)); + t5req->rsvd = cpu_to_be32(isn); + PDBG("%s snd_isn %u\n", __func__, t5req->rsvd); + t5req->opt2 = cpu_to_be32(opt2); + } else { + t6req->params = + cpu_to_be64(FILTER_TUPLE_V(params)); + t6req->rsvd = cpu_to_be32(isn); + PDBG("%s snd_isn %u\n", __func__, t6req->rsvd); + t6req->opt2 = cpu_to_be32(opt2); + } } } else { switch (CHELSIO_CHIP_VERSION(adapter_type)) { @@ -859,18 +869,24 @@ static int send_connect(struct c4iw_ep *ep) req6->opt0 = cpu_to_be64(opt0); if (is_t4(ep->com.dev->rdev.lldi.adapter_type)) { - req6->params = cpu_to_be32(cxgb4_select_ntuple( - ep->com.dev->rdev.lldi.ports[0], - ep->l2t)); + req6->params = cpu_to_be32(cxgb4_select_ntuple(netdev, + ep->l2t)); req6->opt2 = cpu_to_be32(opt2); } else { - t5req6->params = cpu_to_be64(FILTER_TUPLE_V( - cxgb4_select_ntuple( - ep->com.dev->rdev.lldi.ports[0], - ep->l2t))); - t5req6->rsvd = cpu_to_be32(isn); - PDBG("%s snd_isn %u\n", __func__, t5req6->rsvd); - t5req6->opt2 = cpu_to_be32(opt2); + if (is_t5(ep->com.dev->rdev.lldi.adapter_type)) { + t5req6->params = + cpu_to_be64(FILTER_TUPLE_V(params)); + t5req6->rsvd = cpu_to_be32(isn); + PDBG("%s snd_isn %u\n", __func__, t5req6->rsvd); + t5req6->opt2 = cpu_to_be32(opt2); + } else { + t6req6->params = + cpu_to_be64(FILTER_TUPLE_V(params)); + t6req6->rsvd = cpu_to_be32(isn); + PDBG("%s snd_isn %u\n", __func__, t6req6->rsvd); + t6req6->opt2 = cpu_to_be32(opt2); + } + } } @@ -1804,20 +1820,21 @@ static int rx_data(struct c4iw_dev *dev, struct sk_buff *skb) skb_trim(skb, dlen); mutex_lock(&ep->com.mutex); - /* update RX credits */ - update_rx_credits(ep, dlen); - switch (ep->com.state) { case MPA_REQ_SENT: + update_rx_credits(ep, dlen); ep->rcv_seq += dlen; disconnect = process_mpa_reply(ep, skb); break; case MPA_REQ_WAIT: + update_rx_credits(ep, dlen); ep->rcv_seq += dlen; disconnect = process_mpa_request(ep, skb); break; case FPDU_MODE: { struct c4iw_qp_attributes attrs; + + update_rx_credits(ep, dlen); BUG_ON(!ep->com.qp); if (status) pr_err("%s Unexpected streaming data." \ @@ -2516,18 +2533,18 @@ static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb) struct sockaddr_in *sin = (struct sockaddr_in *) &child_ep->com.local_addr; - sin->sin_family = PF_INET; + sin->sin_family = AF_INET; sin->sin_port = local_port; sin->sin_addr.s_addr = *(__be32 *)local_ip; sin = (struct sockaddr_in *)&child_ep->com.local_addr; - sin->sin_family = PF_INET; + sin->sin_family = AF_INET; sin->sin_port = ((struct sockaddr_in *) &parent_ep->com.local_addr)->sin_port; sin->sin_addr.s_addr = *(__be32 *)local_ip; sin = (struct sockaddr_in *)&child_ep->com.remote_addr; - sin->sin_family = PF_INET; + sin->sin_family = AF_INET; sin->sin_port = peer_port; sin->sin_addr.s_addr = *(__be32 *)peer_ip; } else { diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c index 19c6477af19f..bec82a600d77 100644 --- a/drivers/infiniband/hw/cxgb4/cq.c +++ b/drivers/infiniband/hw/cxgb4/cq.c @@ -505,6 +505,15 @@ static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe, } /* + * Special cqe for drain WR completions... + */ + if (CQE_OPCODE(hw_cqe) == C4IW_DRAIN_OPCODE) { + *cookie = CQE_DRAIN_COOKIE(hw_cqe); + *cqe = *hw_cqe; + goto skip_cqe; + } + + /* * Gotta tweak READ completions: * 1) the cqe doesn't contain the sq_wptr from the wr. * 2) opcode not reflected from the wr. @@ -753,6 +762,9 @@ static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ib_wc *wc) c4iw_invalidate_mr(qhp->rhp, CQE_WRID_FR_STAG(&cqe)); break; + case C4IW_DRAIN_OPCODE: + wc->opcode = IB_WC_SEND; + break; default: printk(KERN_ERR MOD "Unexpected opcode %d " "in the CQE received for QPID=0x%0x\n", @@ -817,15 +829,8 @@ static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ib_wc *wc) } } out: - if (wq) { - if (unlikely(qhp->attr.state != C4IW_QP_STATE_RTS)) { - if (t4_sq_empty(wq)) - complete(&qhp->sq_drained); - if (t4_rq_empty(wq)) - complete(&qhp->rq_drained); - } + if (wq) spin_unlock(&qhp->lock); - } return ret; } diff --git a/drivers/infiniband/hw/cxgb4/device.c b/drivers/infiniband/hw/cxgb4/device.c index 516b0ae6dc3f..4e4f1a732b01 100644 --- a/drivers/infiniband/hw/cxgb4/device.c +++ b/drivers/infiniband/hw/cxgb4/device.c @@ -214,6 +214,52 @@ static const struct file_operations wr_log_debugfs_fops = { .write = wr_log_clear, }; +static struct sockaddr_in zero_sin = { + .sin_family = AF_INET, +}; + +static struct sockaddr_in6 zero_sin6 = { + .sin6_family = AF_INET6, +}; + +static void set_ep_sin_addrs(struct c4iw_ep *ep, + struct sockaddr_in **lsin, + struct sockaddr_in **rsin, + struct sockaddr_in **m_lsin, + struct sockaddr_in **m_rsin) +{ + struct iw_cm_id *id = ep->com.cm_id; + + *lsin = (struct sockaddr_in *)&ep->com.local_addr; + *rsin = (struct sockaddr_in *)&ep->com.remote_addr; + if (id) { + *m_lsin = (struct sockaddr_in *)&id->m_local_addr; + *m_rsin = (struct sockaddr_in *)&id->m_remote_addr; + } else { + *m_lsin = &zero_sin; + *m_rsin = &zero_sin; + } +} + +static void set_ep_sin6_addrs(struct c4iw_ep *ep, + struct sockaddr_in6 **lsin6, + struct sockaddr_in6 **rsin6, + struct sockaddr_in6 **m_lsin6, + struct sockaddr_in6 **m_rsin6) +{ + struct iw_cm_id *id = ep->com.cm_id; + + *lsin6 = (struct sockaddr_in6 *)&ep->com.local_addr; + *rsin6 = (struct sockaddr_in6 *)&ep->com.remote_addr; + if (id) { + *m_lsin6 = (struct sockaddr_in6 *)&id->m_local_addr; + *m_rsin6 = (struct sockaddr_in6 *)&id->m_remote_addr; + } else { + *m_lsin6 = &zero_sin6; + *m_rsin6 = &zero_sin6; + } +} + static int dump_qp(int id, void *p, void *data) { struct c4iw_qp *qp = p; @@ -229,16 +275,15 @@ static int dump_qp(int id, void *p, void *data) return 1; if (qp->ep) { - if (qp->ep->com.local_addr.ss_family == AF_INET) { - struct sockaddr_in *lsin = (struct sockaddr_in *) - &qp->ep->com.cm_id->local_addr; - struct sockaddr_in *rsin = (struct sockaddr_in *) - &qp->ep->com.cm_id->remote_addr; - struct sockaddr_in *mapped_lsin = (struct sockaddr_in *) - &qp->ep->com.cm_id->m_local_addr; - struct sockaddr_in *mapped_rsin = (struct sockaddr_in *) - &qp->ep->com.cm_id->m_remote_addr; + struct c4iw_ep *ep = qp->ep; + if (ep->com.local_addr.ss_family == AF_INET) { + struct sockaddr_in *lsin; + struct sockaddr_in *rsin; + struct sockaddr_in *m_lsin; + struct sockaddr_in *m_rsin; + + set_ep_sin_addrs(ep, &lsin, &rsin, &m_lsin, &m_rsin); cc = snprintf(qpd->buf + qpd->pos, space, "rc qp sq id %u rq id %u state %u " "onchip %u ep tid %u state %u " @@ -246,23 +291,19 @@ static int dump_qp(int id, void *p, void *data) qp->wq.sq.qid, qp->wq.rq.qid, (int)qp->attr.state, qp->wq.sq.flags & T4_SQ_ONCHIP, - qp->ep->hwtid, (int)qp->ep->com.state, + ep->hwtid, (int)ep->com.state, &lsin->sin_addr, ntohs(lsin->sin_port), - ntohs(mapped_lsin->sin_port), + ntohs(m_lsin->sin_port), &rsin->sin_addr, ntohs(rsin->sin_port), - ntohs(mapped_rsin->sin_port)); + ntohs(m_rsin->sin_port)); } else { - struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *) - &qp->ep->com.cm_id->local_addr; - struct sockaddr_in6 *rsin6 = (struct sockaddr_in6 *) - &qp->ep->com.cm_id->remote_addr; - struct sockaddr_in6 *mapped_lsin6 = - (struct sockaddr_in6 *) - &qp->ep->com.cm_id->m_local_addr; - struct sockaddr_in6 *mapped_rsin6 = - (struct sockaddr_in6 *) - &qp->ep->com.cm_id->m_remote_addr; + struct sockaddr_in6 *lsin6; + struct sockaddr_in6 *rsin6; + struct sockaddr_in6 *m_lsin6; + struct sockaddr_in6 *m_rsin6; + set_ep_sin6_addrs(ep, &lsin6, &rsin6, &m_lsin6, + &m_rsin6); cc = snprintf(qpd->buf + qpd->pos, space, "rc qp sq id %u rq id %u state %u " "onchip %u ep tid %u state %u " @@ -270,13 +311,13 @@ static int dump_qp(int id, void *p, void *data) qp->wq.sq.qid, qp->wq.rq.qid, (int)qp->attr.state, qp->wq.sq.flags & T4_SQ_ONCHIP, - qp->ep->hwtid, (int)qp->ep->com.state, + ep->hwtid, (int)ep->com.state, &lsin6->sin6_addr, ntohs(lsin6->sin6_port), - ntohs(mapped_lsin6->sin6_port), + ntohs(m_lsin6->sin6_port), &rsin6->sin6_addr, ntohs(rsin6->sin6_port), - ntohs(mapped_rsin6->sin6_port)); + ntohs(m_rsin6->sin6_port)); } } else cc = snprintf(qpd->buf + qpd->pos, space, @@ -533,15 +574,12 @@ static int dump_ep(int id, void *p, void *data) return 1; if (ep->com.local_addr.ss_family == AF_INET) { - struct sockaddr_in *lsin = (struct sockaddr_in *) - &ep->com.cm_id->local_addr; - struct sockaddr_in *rsin = (struct sockaddr_in *) - &ep->com.cm_id->remote_addr; - struct sockaddr_in *mapped_lsin = (struct sockaddr_in *) - &ep->com.cm_id->m_local_addr; - struct sockaddr_in *mapped_rsin = (struct sockaddr_in *) - &ep->com.cm_id->m_remote_addr; + struct sockaddr_in *lsin; + struct sockaddr_in *rsin; + struct sockaddr_in *m_lsin; + struct sockaddr_in *m_rsin; + set_ep_sin_addrs(ep, &lsin, &rsin, &m_lsin, &m_rsin); cc = snprintf(epd->buf + epd->pos, space, "ep %p cm_id %p qp %p state %d flags 0x%lx " "history 0x%lx hwtid %d atid %d " @@ -553,19 +591,16 @@ static int dump_ep(int id, void *p, void *data) ep->stats.connect_neg_adv, ep->stats.abort_neg_adv, &lsin->sin_addr, ntohs(lsin->sin_port), - ntohs(mapped_lsin->sin_port), + ntohs(m_lsin->sin_port), &rsin->sin_addr, ntohs(rsin->sin_port), - ntohs(mapped_rsin->sin_port)); + ntohs(m_rsin->sin_port)); } else { - struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *) - &ep->com.cm_id->local_addr; - struct sockaddr_in6 *rsin6 = (struct sockaddr_in6 *) - &ep->com.cm_id->remote_addr; - struct sockaddr_in6 *mapped_lsin6 = (struct sockaddr_in6 *) - &ep->com.cm_id->m_local_addr; - struct sockaddr_in6 *mapped_rsin6 = (struct sockaddr_in6 *) - &ep->com.cm_id->m_remote_addr; + struct sockaddr_in6 *lsin6; + struct sockaddr_in6 *rsin6; + struct sockaddr_in6 *m_lsin6; + struct sockaddr_in6 *m_rsin6; + set_ep_sin6_addrs(ep, &lsin6, &rsin6, &m_lsin6, &m_rsin6); cc = snprintf(epd->buf + epd->pos, space, "ep %p cm_id %p qp %p state %d flags 0x%lx " "history 0x%lx hwtid %d atid %d " @@ -577,9 +612,9 @@ static int dump_ep(int id, void *p, void *data) ep->stats.connect_neg_adv, ep->stats.abort_neg_adv, &lsin6->sin6_addr, ntohs(lsin6->sin6_port), - ntohs(mapped_lsin6->sin6_port), + ntohs(m_lsin6->sin6_port), &rsin6->sin6_addr, ntohs(rsin6->sin6_port), - ntohs(mapped_rsin6->sin6_port)); + ntohs(m_rsin6->sin6_port)); } if (cc < space) epd->pos += cc; @@ -600,7 +635,7 @@ static int dump_listen_ep(int id, void *p, void *data) if (ep->com.local_addr.ss_family == AF_INET) { struct sockaddr_in *lsin = (struct sockaddr_in *) &ep->com.cm_id->local_addr; - struct sockaddr_in *mapped_lsin = (struct sockaddr_in *) + struct sockaddr_in *m_lsin = (struct sockaddr_in *) &ep->com.cm_id->m_local_addr; cc = snprintf(epd->buf + epd->pos, space, @@ -609,11 +644,11 @@ static int dump_listen_ep(int id, void *p, void *data) ep, ep->com.cm_id, (int)ep->com.state, ep->com.flags, ep->stid, ep->backlog, &lsin->sin_addr, ntohs(lsin->sin_port), - ntohs(mapped_lsin->sin_port)); + ntohs(m_lsin->sin_port)); } else { struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *) &ep->com.cm_id->local_addr; - struct sockaddr_in6 *mapped_lsin6 = (struct sockaddr_in6 *) + struct sockaddr_in6 *m_lsin6 = (struct sockaddr_in6 *) &ep->com.cm_id->m_local_addr; cc = snprintf(epd->buf + epd->pos, space, @@ -622,7 +657,7 @@ static int dump_listen_ep(int id, void *p, void *data) ep, ep->com.cm_id, (int)ep->com.state, ep->com.flags, ep->stid, ep->backlog, &lsin6->sin6_addr, ntohs(lsin6->sin6_port), - ntohs(mapped_lsin6->sin6_port)); + ntohs(m_lsin6->sin6_port)); } if (cc < space) epd->pos += cc; @@ -846,9 +881,17 @@ static int c4iw_rdev_open(struct c4iw_rdev *rdev) } } + rdev->free_workq = create_singlethread_workqueue("iw_cxgb4_free"); + if (!rdev->free_workq) { + err = -ENOMEM; + goto err_free_status_page; + } + rdev->status_page->db_off = 0; return 0; +err_free_status_page: + free_page((unsigned long)rdev->status_page); destroy_ocqp_pool: c4iw_ocqp_pool_destroy(rdev); destroy_rqtpool: @@ -862,6 +905,7 @@ destroy_resource: static void c4iw_rdev_close(struct c4iw_rdev *rdev) { + destroy_workqueue(rdev->free_workq); kfree(rdev->wr_log); free_page((unsigned long)rdev->status_page); c4iw_pblpool_destroy(rdev); diff --git a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h index 4788e1a46fde..d19662f635b1 100644 --- a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h +++ b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h @@ -45,6 +45,7 @@ #include <linux/kref.h> #include <linux/timer.h> #include <linux/io.h> +#include <linux/workqueue.h> #include <asm/byteorder.h> @@ -107,6 +108,7 @@ struct c4iw_dev_ucontext { struct list_head qpids; struct list_head cqids; struct mutex lock; + struct kref kref; }; enum c4iw_rdev_flags { @@ -183,6 +185,7 @@ struct c4iw_rdev { atomic_t wr_log_idx; struct wr_log_entry *wr_log; int wr_log_size; + struct workqueue_struct *free_workq; }; static inline int c4iw_fatal_error(struct c4iw_rdev *rdev) @@ -480,8 +483,8 @@ struct c4iw_qp { wait_queue_head_t wait; struct timer_list timer; int sq_sig_all; - struct completion rq_drained; - struct completion sq_drained; + struct work_struct free_work; + struct c4iw_ucontext *ucontext; }; static inline struct c4iw_qp *to_c4iw_qp(struct ib_qp *ibqp) @@ -495,6 +498,7 @@ struct c4iw_ucontext { u32 key; spinlock_t mmap_lock; struct list_head mmaps; + struct kref kref; }; static inline struct c4iw_ucontext *to_c4iw_ucontext(struct ib_ucontext *c) @@ -502,6 +506,18 @@ static inline struct c4iw_ucontext *to_c4iw_ucontext(struct ib_ucontext *c) return container_of(c, struct c4iw_ucontext, ibucontext); } +void _c4iw_free_ucontext(struct kref *kref); + +static inline void c4iw_put_ucontext(struct c4iw_ucontext *ucontext) +{ + kref_put(&ucontext->kref, _c4iw_free_ucontext); +} + +static inline void c4iw_get_ucontext(struct c4iw_ucontext *ucontext) +{ + kref_get(&ucontext->kref); +} + struct c4iw_mm_entry { struct list_head entry; u64 addr; @@ -615,6 +631,8 @@ static inline int to_ib_qp_state(int c4iw_qp_state) return IB_QPS_ERR; } +#define C4IW_DRAIN_OPCODE FW_RI_SGE_EC_CR_RETURN + static inline u32 c4iw_ib_to_tpt_access(int a) { return (a & IB_ACCESS_REMOTE_WRITE ? FW_RI_MEM_ACCESS_REM_WRITE : 0) | @@ -654,14 +672,14 @@ enum c4iw_mmid_state { #define c4iw_put_ep(ep) { \ PDBG("put_ep (via %s:%u) ep %p refcnt %d\n", __func__, __LINE__, \ - ep, atomic_read(&((ep)->kref.refcount))); \ - WARN_ON(atomic_read(&((ep)->kref.refcount)) < 1); \ + ep, kref_read(&((ep)->kref))); \ + WARN_ON(kref_read(&((ep)->kref)) < 1); \ kref_put(&((ep)->kref), _c4iw_free_ep); \ } #define c4iw_get_ep(ep) { \ PDBG("get_ep (via %s:%u) ep %p, refcnt %d\n", __func__, __LINE__, \ - ep, atomic_read(&((ep)->kref.refcount))); \ + ep, kref_read(&((ep)->kref))); \ kref_get(&((ep)->kref)); \ } void _c4iw_free_ep(struct kref *kref); @@ -997,8 +1015,6 @@ extern int c4iw_wr_log; extern int db_fc_threshold; extern int db_coalescing_threshold; extern int use_dsgl; -void c4iw_drain_rq(struct ib_qp *qp); -void c4iw_drain_sq(struct ib_qp *qp); void c4iw_invalidate_mr(struct c4iw_dev *rhp, u32 rkey); #endif diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c index 5b00b50a484f..bdf7de571d83 100644 --- a/drivers/infiniband/hw/cxgb4/provider.c +++ b/drivers/infiniband/hw/cxgb4/provider.c @@ -93,17 +93,28 @@ static int c4iw_process_mad(struct ib_device *ibdev, int mad_flags, return -ENOSYS; } -static int c4iw_dealloc_ucontext(struct ib_ucontext *context) +void _c4iw_free_ucontext(struct kref *kref) { - struct c4iw_dev *rhp = to_c4iw_dev(context->device); - struct c4iw_ucontext *ucontext = to_c4iw_ucontext(context); + struct c4iw_ucontext *ucontext; + struct c4iw_dev *rhp; struct c4iw_mm_entry *mm, *tmp; - PDBG("%s context %p\n", __func__, context); + ucontext = container_of(kref, struct c4iw_ucontext, kref); + rhp = to_c4iw_dev(ucontext->ibucontext.device); + + PDBG("%s ucontext %p\n", __func__, ucontext); list_for_each_entry_safe(mm, tmp, &ucontext->mmaps, entry) kfree(mm); c4iw_release_dev_ucontext(&rhp->rdev, &ucontext->uctx); kfree(ucontext); +} + +static int c4iw_dealloc_ucontext(struct ib_ucontext *context) +{ + struct c4iw_ucontext *ucontext = to_c4iw_ucontext(context); + + PDBG("%s context %p\n", __func__, context); + c4iw_put_ucontext(ucontext); return 0; } @@ -127,6 +138,7 @@ static struct ib_ucontext *c4iw_alloc_ucontext(struct ib_device *ibdev, c4iw_init_dev_ucontext(&rhp->rdev, &context->uctx); INIT_LIST_HEAD(&context->mmaps); spin_lock_init(&context->mmap_lock); + kref_init(&context->kref); if (udata->outlen < sizeof(uresp) - sizeof(uresp.reserved)) { if (!warned++) @@ -360,16 +372,7 @@ static int c4iw_query_port(struct ib_device *ibdev, u8 port, netdev = dev->rdev.lldi.ports[port-1]; /* props being zeroed by the caller, avoid zeroing it here */ props->max_mtu = IB_MTU_4096; - if (netdev->mtu >= 4096) - props->active_mtu = IB_MTU_4096; - else if (netdev->mtu >= 2048) - props->active_mtu = IB_MTU_2048; - else if (netdev->mtu >= 1024) - props->active_mtu = IB_MTU_1024; - else if (netdev->mtu >= 512) - props->active_mtu = IB_MTU_512; - else - props->active_mtu = IB_MTU_256; + props->active_mtu = ib_mtu_int_to_enum(netdev->mtu); if (!netif_carrier_ok(netdev)) props->state = IB_PORT_DOWN; @@ -607,8 +610,6 @@ int c4iw_register_device(struct c4iw_dev *dev) dev->ibdev.uverbs_abi_ver = C4IW_UVERBS_ABI_VERSION; dev->ibdev.get_port_immutable = c4iw_port_immutable; dev->ibdev.get_dev_fw_str = get_dev_fw_str; - dev->ibdev.drain_sq = c4iw_drain_sq; - dev->ibdev.drain_rq = c4iw_drain_rq; dev->ibdev.iwcm = kmalloc(sizeof(struct iw_cm_verbs), GFP_KERNEL); if (!dev->ibdev.iwcm) diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c index cda5542e13a2..d4fd2f5c8326 100644 --- a/drivers/infiniband/hw/cxgb4/qp.c +++ b/drivers/infiniband/hw/cxgb4/qp.c @@ -715,13 +715,32 @@ static int build_inv_stag(union t4_wr *wqe, struct ib_send_wr *wr, u8 *len16) return 0; } -static void _free_qp(struct kref *kref) +static void free_qp_work(struct work_struct *work) +{ + struct c4iw_ucontext *ucontext; + struct c4iw_qp *qhp; + struct c4iw_dev *rhp; + + qhp = container_of(work, struct c4iw_qp, free_work); + ucontext = qhp->ucontext; + rhp = qhp->rhp; + + PDBG("%s qhp %p ucontext %p\n", __func__, qhp, ucontext); + destroy_qp(&rhp->rdev, &qhp->wq, + ucontext ? &ucontext->uctx : &rhp->rdev.uctx); + + if (ucontext) + c4iw_put_ucontext(ucontext); + kfree(qhp); +} + +static void queue_qp_free(struct kref *kref) { struct c4iw_qp *qhp; qhp = container_of(kref, struct c4iw_qp, kref); PDBG("%s qhp %p\n", __func__, qhp); - kfree(qhp); + queue_work(qhp->rhp->rdev.free_workq, &qhp->free_work); } void c4iw_qp_add_ref(struct ib_qp *qp) @@ -733,7 +752,7 @@ void c4iw_qp_add_ref(struct ib_qp *qp) void c4iw_qp_rem_ref(struct ib_qp *qp) { PDBG("%s ib_qp %p\n", __func__, qp); - kref_put(&to_c4iw_qp(qp)->kref, _free_qp); + kref_put(&to_c4iw_qp(qp)->kref, queue_qp_free); } static void add_to_fc_list(struct list_head *head, struct list_head *entry) @@ -776,6 +795,64 @@ static int ring_kernel_rq_db(struct c4iw_qp *qhp, u16 inc) return 0; } +static void complete_sq_drain_wr(struct c4iw_qp *qhp, struct ib_send_wr *wr) +{ + struct t4_cqe cqe = {}; + struct c4iw_cq *schp; + unsigned long flag; + struct t4_cq *cq; + + schp = to_c4iw_cq(qhp->ibqp.send_cq); + cq = &schp->cq; + + cqe.u.drain_cookie = wr->wr_id; + cqe.header = cpu_to_be32(CQE_STATUS_V(T4_ERR_SWFLUSH) | + CQE_OPCODE_V(C4IW_DRAIN_OPCODE) | + CQE_TYPE_V(1) | + CQE_SWCQE_V(1) | + CQE_QPID_V(qhp->wq.sq.qid)); + + spin_lock_irqsave(&schp->lock, flag); + cqe.bits_type_ts = cpu_to_be64(CQE_GENBIT_V((u64)cq->gen)); + cq->sw_queue[cq->sw_pidx] = cqe; + t4_swcq_produce(cq); + spin_unlock_irqrestore(&schp->lock, flag); + + spin_lock_irqsave(&schp->comp_handler_lock, flag); + (*schp->ibcq.comp_handler)(&schp->ibcq, + schp->ibcq.cq_context); + spin_unlock_irqrestore(&schp->comp_handler_lock, flag); +} + +static void complete_rq_drain_wr(struct c4iw_qp *qhp, struct ib_recv_wr *wr) +{ + struct t4_cqe cqe = {}; + struct c4iw_cq *rchp; + unsigned long flag; + struct t4_cq *cq; + + rchp = to_c4iw_cq(qhp->ibqp.recv_cq); + cq = &rchp->cq; + + cqe.u.drain_cookie = wr->wr_id; + cqe.header = cpu_to_be32(CQE_STATUS_V(T4_ERR_SWFLUSH) | + CQE_OPCODE_V(C4IW_DRAIN_OPCODE) | + CQE_TYPE_V(0) | + CQE_SWCQE_V(1) | + CQE_QPID_V(qhp->wq.sq.qid)); + + spin_lock_irqsave(&rchp->lock, flag); + cqe.bits_type_ts = cpu_to_be64(CQE_GENBIT_V((u64)cq->gen)); + cq->sw_queue[cq->sw_pidx] = cqe; + t4_swcq_produce(cq); + spin_unlock_irqrestore(&rchp->lock, flag); + + spin_lock_irqsave(&rchp->comp_handler_lock, flag); + (*rchp->ibcq.comp_handler)(&rchp->ibcq, + rchp->ibcq.cq_context); + spin_unlock_irqrestore(&rchp->comp_handler_lock, flag); +} + int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, struct ib_send_wr **bad_wr) { @@ -794,8 +871,8 @@ int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, spin_lock_irqsave(&qhp->lock, flag); if (t4_wq_in_error(&qhp->wq)) { spin_unlock_irqrestore(&qhp->lock, flag); - *bad_wr = wr; - return -EINVAL; + complete_sq_drain_wr(qhp, wr); + return err; } num_wrs = t4_sq_avail(&qhp->wq); if (num_wrs == 0) { @@ -937,8 +1014,8 @@ int c4iw_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, spin_lock_irqsave(&qhp->lock, flag); if (t4_wq_in_error(&qhp->wq)) { spin_unlock_irqrestore(&qhp->lock, flag); - *bad_wr = wr; - return -EINVAL; + complete_rq_drain_wr(qhp, wr); + return err; } num_wrs = t4_rq_avail(&qhp->wq); if (num_wrs == 0) { @@ -1503,7 +1580,7 @@ int c4iw_modify_qp(struct c4iw_dev *rhp, struct c4iw_qp *qhp, case C4IW_QP_STATE_RTS: switch (attrs->next_state) { case C4IW_QP_STATE_CLOSING: - BUG_ON(atomic_read(&qhp->ep->com.kref.refcount) < 2); + BUG_ON(kref_read(&qhp->ep->com.kref) < 2); t4_set_wq_in_error(&qhp->wq); set_state(qhp, C4IW_QP_STATE_CLOSING); ep = qhp->ep; @@ -1550,7 +1627,12 @@ int c4iw_modify_qp(struct c4iw_dev *rhp, struct c4iw_qp *qhp, } break; case C4IW_QP_STATE_CLOSING: - if (!internal) { + + /* + * Allow kernel users to move to ERROR for qp draining. + */ + if (!internal && (qhp->ibqp.uobject || attrs->next_state != + C4IW_QP_STATE_ERROR)) { ret = -EINVAL; goto out; } @@ -1643,7 +1725,6 @@ int c4iw_destroy_qp(struct ib_qp *ib_qp) struct c4iw_dev *rhp; struct c4iw_qp *qhp; struct c4iw_qp_attributes attrs; - struct c4iw_ucontext *ucontext; qhp = to_c4iw_qp(ib_qp); rhp = qhp->rhp; @@ -1663,11 +1744,6 @@ int c4iw_destroy_qp(struct ib_qp *ib_qp) spin_unlock_irq(&rhp->lock); free_ird(rhp, qhp->attr.max_ird); - ucontext = ib_qp->uobject ? - to_c4iw_ucontext(ib_qp->uobject->context) : NULL; - destroy_qp(&rhp->rdev, &qhp->wq, - ucontext ? &ucontext->uctx : &rhp->rdev.uctx); - c4iw_qp_rem_ref(ib_qp); PDBG("%s ib_qp %p qpid 0x%0x\n", __func__, ib_qp, qhp->wq.sq.qid); @@ -1763,11 +1839,10 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, qhp->attr.max_ird = 0; qhp->sq_sig_all = attrs->sq_sig_type == IB_SIGNAL_ALL_WR; spin_lock_init(&qhp->lock); - init_completion(&qhp->sq_drained); - init_completion(&qhp->rq_drained); mutex_init(&qhp->mutex); init_waitqueue_head(&qhp->wait); kref_init(&qhp->kref); + INIT_WORK(&qhp->free_work, free_qp_work); ret = insert_handle(rhp, &rhp->qpidr, qhp, qhp->wq.sq.qid); if (ret) @@ -1854,6 +1929,9 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, ma_sync_key_mm->len = PAGE_SIZE; insert_mmap(ucontext, ma_sync_key_mm); } + + c4iw_get_ucontext(ucontext); + qhp->ucontext = ucontext; } qhp->ibqp.qp_num = qhp->wq.sq.qid; init_timer(&(qhp->timer)); @@ -1958,40 +2036,3 @@ int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, init_attr->sq_sig_type = qhp->sq_sig_all ? IB_SIGNAL_ALL_WR : 0; return 0; } - -static void move_qp_to_err(struct c4iw_qp *qp) -{ - struct c4iw_qp_attributes attrs = { .next_state = C4IW_QP_STATE_ERROR }; - - (void)c4iw_modify_qp(qp->rhp, qp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); -} - -void c4iw_drain_sq(struct ib_qp *ibqp) -{ - struct c4iw_qp *qp = to_c4iw_qp(ibqp); - unsigned long flag; - bool need_to_wait; - - move_qp_to_err(qp); - spin_lock_irqsave(&qp->lock, flag); - need_to_wait = !t4_sq_empty(&qp->wq); - spin_unlock_irqrestore(&qp->lock, flag); - - if (need_to_wait) - wait_for_completion(&qp->sq_drained); -} - -void c4iw_drain_rq(struct ib_qp *ibqp) -{ - struct c4iw_qp *qp = to_c4iw_qp(ibqp); - unsigned long flag; - bool need_to_wait; - - move_qp_to_err(qp); - spin_lock_irqsave(&qp->lock, flag); - need_to_wait = !t4_rq_empty(&qp->wq); - spin_unlock_irqrestore(&qp->lock, flag); - - if (need_to_wait) - wait_for_completion(&qp->rq_drained); -} diff --git a/drivers/infiniband/hw/cxgb4/t4.h b/drivers/infiniband/hw/cxgb4/t4.h index 862381aa83c8..640d22148a3e 100644 --- a/drivers/infiniband/hw/cxgb4/t4.h +++ b/drivers/infiniband/hw/cxgb4/t4.h @@ -179,6 +179,7 @@ struct t4_cqe { __be32 wrid_hi; __be32 wrid_low; } gen; + u64 drain_cookie; } u; __be64 reserved; __be64 bits_type_ts; @@ -238,6 +239,7 @@ struct t4_cqe { /* generic accessor macros */ #define CQE_WRID_HI(x) (be32_to_cpu((x)->u.gen.wrid_hi)) #define CQE_WRID_LOW(x) (be32_to_cpu((x)->u.gen.wrid_low)) +#define CQE_DRAIN_COOKIE(x) ((x)->u.drain_cookie) /* macros for flit 3 of the cqe */ #define CQE_GENBIT_S 63 |