From 62aee0e3028614588a14e19979cd3f7b97948505 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 29 Nov 2016 10:52:08 -0500 Subject: xprtrdma: Cap size of callback buffer resources When the inline threshold size is set to large values (say, 32KB) any NFSv4.1 CB request from the server gets a reply with status NFS4ERR_RESOURCE. Looks like there are some upper layer assumptions about the maximum size of a reply (for example, in process_op). Cap the size of the NFSv4 client's reply resources at a page. Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- net/sunrpc/xprtrdma/backchannel.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'net/sunrpc') diff --git a/net/sunrpc/xprtrdma/backchannel.c b/net/sunrpc/xprtrdma/backchannel.c index 2c472e1b4827..24fedd4b117e 100644 --- a/net/sunrpc/xprtrdma/backchannel.c +++ b/net/sunrpc/xprtrdma/backchannel.c @@ -55,7 +55,8 @@ static int rpcrdma_bc_setup_rqst(struct rpcrdma_xprt *r_xprt, if (IS_ERR(rb)) goto out_fail; req->rl_sendbuf = rb; - xdr_buf_init(&rqst->rq_snd_buf, rb->rg_base, size); + xdr_buf_init(&rqst->rq_snd_buf, rb->rg_base, + min_t(size_t, size, PAGE_SIZE)); rpcrdma_set_xprtdata(rqst, req); return 0; @@ -191,6 +192,7 @@ size_t xprt_rdma_bc_maxpayload(struct rpc_xprt *xprt) size_t maxmsg; maxmsg = min_t(unsigned int, cdata->inline_rsize, cdata->inline_wsize); + maxmsg = min_t(unsigned int, maxmsg, PAGE_SIZE); return maxmsg - RPCRDMA_HDRLEN_MIN; } -- cgit v1.2.3-59-g8ed1b From 8d38de65644d900199f035277aa5f3da4aa9fc17 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 29 Nov 2016 10:52:16 -0500 Subject: xprtrdma: Make FRWR send queue entry accounting more accurate Verbs providers may perform house-keeping on the Send Queue during each signaled send completion. It is necessary therefore for a verbs consumer (like xprtrdma) to occasionally force a signaled send completion if it runs unsignaled most of the time. xprtrdma does not require signaled completions for Send or FastReg Work Requests, but does signal some LocalInv Work Requests. To ensure that Send Queue house-keeping can run before the Send Queue is more than half-consumed, xprtrdma forces a signaled completion on occasion by counting the number of Send Queue Entries it consumes. It currently does this by counting each ib_post_send as one Entry. Commit c9918ff56dfb ("xprtrdma: Add ro_unmap_sync method for FRWR") introduced the ability for frwr_op_unmap_sync to post more than one Work Request with a single post_send. Thus the underlying assumption of one Send Queue Entry per ib_post_send is no longer true. Also, FastReg Work Requests are currently never signaled. They should be signaled once in a while, just as Send is, to keep the accounting of consumed SQEs accurate. While we're here, convert the CQCOUNT macros to the currently preferred kernel coding style, which is inline functions. Fixes: c9918ff56dfb ("xprtrdma: Add ro_unmap_sync method for FRWR") Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- net/sunrpc/xprtrdma/frwr_ops.c | 13 ++++++++++--- net/sunrpc/xprtrdma/verbs.c | 10 ++-------- net/sunrpc/xprtrdma/xprt_rdma.h | 20 ++++++++++++++++++-- 3 files changed, 30 insertions(+), 13 deletions(-) (limited to 'net/sunrpc') diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c index 26b26beef2d4..adbf52c6df83 100644 --- a/net/sunrpc/xprtrdma/frwr_ops.c +++ b/net/sunrpc/xprtrdma/frwr_ops.c @@ -421,7 +421,7 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg, IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE : IB_ACCESS_REMOTE_READ; - DECR_CQCOUNT(&r_xprt->rx_ep); + rpcrdma_set_signaled(&r_xprt->rx_ep, ®_wr->wr); rc = ib_post_send(ia->ri_id->qp, ®_wr->wr, &bad_wr); if (rc) goto out_senderr; @@ -486,7 +486,7 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) struct rpcrdma_ia *ia = &r_xprt->rx_ia; struct rpcrdma_mw *mw, *tmp; struct rpcrdma_frmr *f; - int rc; + int count, rc; dprintk("RPC: %s: req %p\n", __func__, req); @@ -496,6 +496,7 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) * a single ib_post_send() call. */ f = NULL; + count = 0; invalidate_wrs = pos = prev = NULL; list_for_each_entry(mw, &req->rl_registered, mw_list) { if ((rep->rr_wc_flags & IB_WC_WITH_INVALIDATE) && @@ -505,6 +506,7 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) } pos = __frwr_prepare_linv_wr(mw); + count++; if (!invalidate_wrs) invalidate_wrs = pos; @@ -523,7 +525,12 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) f->fr_invwr.send_flags = IB_SEND_SIGNALED; f->fr_cqe.done = frwr_wc_localinv_wake; reinit_completion(&f->fr_linv_done); - INIT_CQCOUNT(&r_xprt->rx_ep); + + /* Initialize CQ count, since there is always a signaled + * WR being posted here. The new cqcount depends on how + * many SQEs are about to be consumed. + */ + rpcrdma_init_cqcount(&r_xprt->rx_ep, count); /* Transport disconnect drains the receive CQ before it * replaces the QP. The RPC reply handler won't call us diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index ec74289af7ec..451f5f27d8af 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -532,7 +532,7 @@ rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia, ep->rep_cqinit = ep->rep_attr.cap.max_send_wr/2 - 1; if (ep->rep_cqinit <= 2) ep->rep_cqinit = 0; /* always signal? */ - INIT_CQCOUNT(ep); + rpcrdma_init_cqcount(ep, 0); init_waitqueue_head(&ep->rep_connect_wait); INIT_DELAYED_WORK(&ep->rep_connect_worker, rpcrdma_connect_worker); @@ -1311,13 +1311,7 @@ rpcrdma_ep_post(struct rpcrdma_ia *ia, dprintk("RPC: %s: posting %d s/g entries\n", __func__, send_wr->num_sge); - if (DECR_CQCOUNT(ep) > 0) - send_wr->send_flags = 0; - else { /* Provider must take a send completion every now and then */ - INIT_CQCOUNT(ep); - send_wr->send_flags = IB_SEND_SIGNALED; - } - + rpcrdma_set_signaled(ep, send_wr); rc = ib_post_send(ia->ri_id->qp, send_wr, &send_wr_fail); if (rc) goto out_postsend_err; diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h index 6e1bba358203..f6ae1b22da47 100644 --- a/net/sunrpc/xprtrdma/xprt_rdma.h +++ b/net/sunrpc/xprtrdma/xprt_rdma.h @@ -95,8 +95,24 @@ struct rpcrdma_ep { struct delayed_work rep_connect_worker; }; -#define INIT_CQCOUNT(ep) atomic_set(&(ep)->rep_cqcount, (ep)->rep_cqinit) -#define DECR_CQCOUNT(ep) atomic_sub_return(1, &(ep)->rep_cqcount) +static inline void +rpcrdma_init_cqcount(struct rpcrdma_ep *ep, int count) +{ + atomic_set(&ep->rep_cqcount, ep->rep_cqinit - count); +} + +/* To update send queue accounting, provider must take a + * send completion every now and then. + */ +static inline void +rpcrdma_set_signaled(struct rpcrdma_ep *ep, struct ib_send_wr *send_wr) +{ + send_wr->send_flags = 0; + if (unlikely(atomic_sub_return(1, &ep->rep_cqcount) <= 0)) { + rpcrdma_init_cqcount(ep, 0); + send_wr->send_flags = IB_SEND_SIGNALED; + } +} /* Pre-allocate extra Work Requests for handling backward receives * and sends. This is a fixed value because the Work Queues are -- cgit v1.2.3-59-g8ed1b From 5e9fc6a06bba9e6821ce964067fcf4401496bc29 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 29 Nov 2016 10:52:24 -0500 Subject: xprtrdma: Support for SG_GAP devices Some devices (such as the Mellanox CX-4) can register, under a single R_key, a set of memory regions that are not contiguous. When this is done, all the segments in a Reply list, say, can then be invalidated in a single LocalInv Work Request (or via Remote Invalidation, which can invalidate exactly one R_key when completing a Receive). This means a single FastReg WR is used to register, and one or zero LocalInv WRs can invalidate, the memory involved with RDMA transfers on behalf of an RPC. In addition, xprtrdma constructs some Reply chunks from three or more segments. By registering them with SG_GAP, only one segment is needed for the Reply chunk, allowing the whole chunk to be invalidated remotely. Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- net/sunrpc/xprtrdma/frwr_ops.c | 20 +++++++++++++------- net/sunrpc/xprtrdma/xprt_rdma.h | 1 + 2 files changed, 14 insertions(+), 7 deletions(-) (limited to 'net/sunrpc') diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c index adbf52c6df83..e99bf6180136 100644 --- a/net/sunrpc/xprtrdma/frwr_ops.c +++ b/net/sunrpc/xprtrdma/frwr_ops.c @@ -101,7 +101,7 @@ frwr_op_init_mr(struct rpcrdma_ia *ia, struct rpcrdma_mw *r) struct rpcrdma_frmr *f = &r->frmr; int rc; - f->fr_mr = ib_alloc_mr(ia->ri_pd, IB_MR_TYPE_MEM_REG, depth); + f->fr_mr = ib_alloc_mr(ia->ri_pd, ia->ri_mrtype, depth); if (IS_ERR(f->fr_mr)) goto out_mr_err; @@ -157,7 +157,7 @@ __frwr_reset_mr(struct rpcrdma_ia *ia, struct rpcrdma_mw *r) return rc; } - f->fr_mr = ib_alloc_mr(ia->ri_pd, IB_MR_TYPE_MEM_REG, + f->fr_mr = ib_alloc_mr(ia->ri_pd, ia->ri_mrtype, ia->ri_max_frmr_depth); if (IS_ERR(f->fr_mr)) { pr_warn("rpcrdma: ib_alloc_mr status %ld, frwr %p orphaned\n", @@ -210,11 +210,16 @@ static int frwr_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep, struct rpcrdma_create_data_internal *cdata) { + struct ib_device_attr *attrs = &ia->ri_device->attrs; int depth, delta; + ia->ri_mrtype = IB_MR_TYPE_MEM_REG; + if (attrs->device_cap_flags & IB_DEVICE_SG_GAPS_REG) + ia->ri_mrtype = IB_MR_TYPE_SG_GAPS; + ia->ri_max_frmr_depth = min_t(unsigned int, RPCRDMA_MAX_DATA_SEGS, - ia->ri_device->attrs.max_fast_reg_page_list_len); + attrs->max_fast_reg_page_list_len); dprintk("RPC: %s: device's max FR page list len = %u\n", __func__, ia->ri_max_frmr_depth); @@ -241,8 +246,8 @@ frwr_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep, } ep->rep_attr.cap.max_send_wr *= depth; - if (ep->rep_attr.cap.max_send_wr > ia->ri_device->attrs.max_qp_wr) { - cdata->max_requests = ia->ri_device->attrs.max_qp_wr / depth; + if (ep->rep_attr.cap.max_send_wr > attrs->max_qp_wr) { + cdata->max_requests = attrs->max_qp_wr / depth; if (!cdata->max_requests) return -EINVAL; ep->rep_attr.cap.max_send_wr = cdata->max_requests * @@ -348,6 +353,7 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg, int nsegs, bool writing, struct rpcrdma_mw **out) { struct rpcrdma_ia *ia = &r_xprt->rx_ia; + bool holes_ok = ia->ri_mrtype == IB_MR_TYPE_SG_GAPS; struct rpcrdma_mw *mw; struct rpcrdma_frmr *frmr; struct ib_mr *mr; @@ -383,8 +389,8 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg, ++seg; ++i; - - /* Check for holes */ + if (holes_ok) + continue; if ((i < nsegs && offset_in_page(seg->mr_offset)) || offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len)) break; diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h index f6ae1b22da47..b8424fa47d29 100644 --- a/net/sunrpc/xprtrdma/xprt_rdma.h +++ b/net/sunrpc/xprtrdma/xprt_rdma.h @@ -75,6 +75,7 @@ struct rpcrdma_ia { unsigned int ri_max_inline_write; unsigned int ri_max_inline_read; bool ri_reminv_expected; + enum ib_mr_type ri_mrtype; struct ib_qp_attr ri_qp_attr; struct ib_qp_init_attr ri_qp_init_attr; }; -- cgit v1.2.3-59-g8ed1b From ae09531d3c1f657c1701fa7fd076b125b0a4d8cf Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 29 Nov 2016 10:52:32 -0500 Subject: SUNRPC: Proper metric accounting when RPC is not transmitted I noticed recently that during an xfstests on a krb5i mount, the retransmit count for certain operations had gone negative, and the backlog value became unreasonably large. I recall that Andy has pointed this out to me in the past. When call_refresh fails to find a valid credential for an RPC, the RPC exits immediately without sending anything on the wire. This leaves rq_ntrans, rq_xtime, and rq_rtt set to zero. The solution for om_queue is to not add the to RPC's running backlog queue total whenever rq_xtime is zero. For om_ntrans, it's a bit more difficult. A zero rq_ntrans causes om_ops to become larger than om_ntrans. The design of the RPC metrics API assumes that ntrans will always be equal to or larger than the ops count. The result is that when an RPC fails to find credentials, the RPC operation's reported retransmit count, which is computed in user space as the difference between ops and ntrans, goes negative. Ideally the kernel API should report a separate retransmit and "exited before initial transmission" metric, so that user space can sort out the difference properly. To avoid kernel API changes and changes to the way rq_ntrans is used when performing transport locking, account for untransmitted RPCs so that om_ntrans keeps up with om_ops: always add one or more to om_ntrans. Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- net/sunrpc/stats.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'net/sunrpc') diff --git a/net/sunrpc/stats.c b/net/sunrpc/stats.c index 2ecb994314c1..caeb01ad2b5a 100644 --- a/net/sunrpc/stats.c +++ b/net/sunrpc/stats.c @@ -157,15 +157,17 @@ void rpc_count_iostats_metrics(const struct rpc_task *task, spin_lock(&op_metrics->om_lock); op_metrics->om_ops++; - op_metrics->om_ntrans += req->rq_ntrans; + /* kernel API: om_ops must never become larger than om_ntrans */ + op_metrics->om_ntrans += max(req->rq_ntrans, 1); op_metrics->om_timeouts += task->tk_timeouts; op_metrics->om_bytes_sent += req->rq_xmit_bytes_sent; op_metrics->om_bytes_recv += req->rq_reply_bytes_recvd; - delta = ktime_sub(req->rq_xtime, task->tk_start); - op_metrics->om_queue = ktime_add(op_metrics->om_queue, delta); - + if (ktime_to_ns(req->rq_xtime)) { + delta = ktime_sub(req->rq_xtime, task->tk_start); + op_metrics->om_queue = ktime_add(op_metrics->om_queue, delta); + } op_metrics->om_rtt = ktime_add(op_metrics->om_rtt, req->rq_rtt); delta = ktime_sub(now, task->tk_start); -- cgit v1.2.3-59-g8ed1b From 109b88ab9d78f76b3f6f42155cac2241f2ad2e31 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 29 Nov 2016 10:52:40 -0500 Subject: xprtrdma: Address coverity complaint about wait_for_completion() > ** CID 114101: Error handling issues (CHECKED_RETURN) > /net/sunrpc/xprtrdma/verbs.c: 355 in rpcrdma_create_id() Commit 5675add36e76 ("RPC/RDMA: harden connection logic against missing/late rdma_cm upcalls.") replaced wait_for_completion() calls with these two call sites. The original wait_for_completion() calls were added in the initial commit of verbs.c, which was commit c56c65fb67d6 ("RPCRDMA: rpc rdma verbs interface implementation"), but these returned void. rpcrdma_create_id() is called by the RDMA connect worker, which probably won't ever be interrupted. It is also called by rpcrdma_ia_open which is in the synchronous mount path, and ^C is possible there. Add a bit of logic at those two call sites to return if the waits return ERESTARTSYS. Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- net/sunrpc/xprtrdma/verbs.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) (limited to 'net/sunrpc') diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index 451f5f27d8af..cbb18857841a 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -331,6 +331,7 @@ static struct rdma_cm_id * rpcrdma_create_id(struct rpcrdma_xprt *xprt, struct rpcrdma_ia *ia, struct sockaddr *addr) { + unsigned long wtimeout = msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1; struct rdma_cm_id *id; int rc; @@ -352,8 +353,12 @@ rpcrdma_create_id(struct rpcrdma_xprt *xprt, __func__, rc); goto out; } - wait_for_completion_interruptible_timeout(&ia->ri_done, - msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1); + rc = wait_for_completion_interruptible_timeout(&ia->ri_done, wtimeout); + if (rc < 0) { + dprintk("RPC: %s: wait() exited: %i\n", + __func__, rc); + goto out; + } /* FIXME: * Until xprtrdma supports DEVICE_REMOVAL, the provider must @@ -376,8 +381,12 @@ rpcrdma_create_id(struct rpcrdma_xprt *xprt, __func__, rc); goto put; } - wait_for_completion_interruptible_timeout(&ia->ri_done, - msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1); + rc = wait_for_completion_interruptible_timeout(&ia->ri_done, wtimeout); + if (rc < 0) { + dprintk("RPC: %s: wait() exited: %i\n", + __func__, rc); + goto put; + } rc = ia->ri_async_rc; if (rc) goto put; -- cgit v1.2.3-59-g8ed1b From 48016dce46ad504a378849490bfb99c98be5cfaa Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 29 Nov 2016 10:52:48 -0500 Subject: xprtrdma: Avoid calls to ro_unmap_safe() Micro-optimization: Most of the time, calls to ro_unmap_safe are expensive no-ops. Call only when there is work to do. Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- net/sunrpc/xprtrdma/transport.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'net/sunrpc') diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c index ed5e285fd2ea..545d3fc8de5a 100644 --- a/net/sunrpc/xprtrdma/transport.c +++ b/net/sunrpc/xprtrdma/transport.c @@ -621,7 +621,8 @@ xprt_rdma_free(struct rpc_task *task) dprintk("RPC: %s: called on 0x%p\n", __func__, req->rl_reply); - ia->ri_ops->ro_unmap_safe(r_xprt, req, !RPC_IS_ASYNC(task)); + if (unlikely(!list_empty(&req->rl_registered))) + ia->ri_ops->ro_unmap_safe(r_xprt, req, !RPC_IS_ASYNC(task)); rpcrdma_unmap_sges(ia, req); rpcrdma_buffer_put(req); } @@ -657,7 +658,8 @@ xprt_rdma_send_request(struct rpc_task *task) int rc = 0; /* On retransmit, remove any previously registered chunks */ - r_xprt->rx_ia.ri_ops->ro_unmap_safe(r_xprt, req, false); + if (unlikely(!list_empty(&req->rl_registered))) + r_xprt->rx_ia.ri_ops->ro_unmap_safe(r_xprt, req, false); rc = rpcrdma_marshal_req(rqst); if (rc < 0) -- cgit v1.2.3-59-g8ed1b From a100fda1a2e1fa6c52373b9c7985a0bd3459bf4c Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 29 Nov 2016 10:52:57 -0500 Subject: xprtrdma: Refactor FRMR invalidation Clean up: After some recent updates, clarifications can be made to the FRMR invalidation logic. - Both the remote and local invalidation case mark the frmr INVALID, so make that a common path. - Manage the WR list more "tastefully" by replacing the conditional that discriminates between the list head and ->next pointers. - Use mw->mw_handle in all cases, since that has the same value as f->fr_mr->rkey, and is already in cache. Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- net/sunrpc/xprtrdma/frwr_ops.c | 57 ++++++++++++++++-------------------------- 1 file changed, 21 insertions(+), 36 deletions(-) (limited to 'net/sunrpc') diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c index e99bf6180136..900dc4024d2c 100644 --- a/net/sunrpc/xprtrdma/frwr_ops.c +++ b/net/sunrpc/xprtrdma/frwr_ops.c @@ -457,26 +457,6 @@ out_senderr: return -ENOTCONN; } -static struct ib_send_wr * -__frwr_prepare_linv_wr(struct rpcrdma_mw *mw) -{ - struct rpcrdma_frmr *f = &mw->frmr; - struct ib_send_wr *invalidate_wr; - - dprintk("RPC: %s: invalidating frmr %p\n", __func__, f); - - f->fr_state = FRMR_IS_INVALID; - invalidate_wr = &f->fr_invwr; - - memset(invalidate_wr, 0, sizeof(*invalidate_wr)); - f->fr_cqe.done = frwr_wc_localinv; - invalidate_wr->wr_cqe = &f->fr_cqe; - invalidate_wr->opcode = IB_WR_LOCAL_INV; - invalidate_wr->ex.invalidate_rkey = f->fr_mr->rkey; - - return invalidate_wr; -} - /* Invalidate all memory regions that were registered for "req". * * Sleeps until it is safe for the host CPU to access the @@ -487,7 +467,7 @@ __frwr_prepare_linv_wr(struct rpcrdma_mw *mw) static void frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) { - struct ib_send_wr *invalidate_wrs, *pos, *prev, *bad_wr; + struct ib_send_wr *first, **prev, *last, *bad_wr; struct rpcrdma_rep *rep = req->rl_reply; struct rpcrdma_ia *ia = &r_xprt->rx_ia; struct rpcrdma_mw *mw, *tmp; @@ -503,23 +483,28 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) */ f = NULL; count = 0; - invalidate_wrs = pos = prev = NULL; + prev = &first; list_for_each_entry(mw, &req->rl_registered, mw_list) { + mw->frmr.fr_state = FRMR_IS_INVALID; + if ((rep->rr_wc_flags & IB_WC_WITH_INVALIDATE) && - (mw->mw_handle == rep->rr_inv_rkey)) { - mw->frmr.fr_state = FRMR_IS_INVALID; + (mw->mw_handle == rep->rr_inv_rkey)) continue; - } - pos = __frwr_prepare_linv_wr(mw); + f = &mw->frmr; + dprintk("RPC: %s: invalidating frmr %p\n", + __func__, f); + + f->fr_cqe.done = frwr_wc_localinv; + last = &f->fr_invwr; + memset(last, 0, sizeof(*last)); + last->wr_cqe = &f->fr_cqe; + last->opcode = IB_WR_LOCAL_INV; + last->ex.invalidate_rkey = mw->mw_handle; count++; - if (!invalidate_wrs) - invalidate_wrs = pos; - else - prev->next = pos; - prev = pos; - f = &mw->frmr; + *prev = last; + prev = &last->next; } if (!f) goto unmap; @@ -528,7 +513,7 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) * last WR in the chain completes, all WRs in the chain * are complete. */ - f->fr_invwr.send_flags = IB_SEND_SIGNALED; + last->send_flags = IB_SEND_SIGNALED; f->fr_cqe.done = frwr_wc_localinv_wake; reinit_completion(&f->fr_linv_done); @@ -543,7 +528,7 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) * unless ri_id->qp is a valid pointer. */ r_xprt->rx_stats.local_inv_needed++; - rc = ib_post_send(ia->ri_id->qp, invalidate_wrs, &bad_wr); + rc = ib_post_send(ia->ri_id->qp, first, &bad_wr); if (rc) goto reset_mrs; @@ -554,7 +539,7 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) */ unmap: list_for_each_entry_safe(mw, tmp, &req->rl_registered, mw_list) { - dprintk("RPC: %s: unmapping frmr %p\n", + dprintk("RPC: %s: DMA unmapping frmr %p\n", __func__, &mw->frmr); list_del_init(&mw->mw_list); ib_dma_unmap_sg(ia->ri_device, @@ -572,7 +557,7 @@ reset_mrs: */ list_for_each_entry(mw, &req->rl_registered, mw_list) { f = &mw->frmr; - if (mw->frmr.fr_mr->rkey == bad_wr->ex.invalidate_rkey) { + if (mw->mw_handle == bad_wr->ex.invalidate_rkey) { __frwr_reset_mr(ia, mw); bad_wr = bad_wr->next; } -- cgit v1.2.3-59-g8ed1b From 289400af2b8783b3b01aee7ec9dba5b476bb3450 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 29 Nov 2016 10:53:05 -0500 Subject: xprtrdma: Update documenting comment Clean up: If reset fails, FRMRs are no longer abandoned, rather they are released immediately. Update the comment to reflect this. Fixes: 2ffc871a574d ('xprtrdma: Release orphaned MRs immediately') Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- net/sunrpc/xprtrdma/frwr_ops.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'net/sunrpc') diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c index 900dc4024d2c..47bed5333c7f 100644 --- a/net/sunrpc/xprtrdma/frwr_ops.c +++ b/net/sunrpc/xprtrdma/frwr_ops.c @@ -171,10 +171,6 @@ __frwr_reset_mr(struct rpcrdma_ia *ia, struct rpcrdma_mw *r) } /* Reset of a single FRMR. Generate a fresh rkey by replacing the MR. - * - * There's no recovery if this fails. The FRMR is abandoned, but - * remains in rb_all. It will be cleaned up when the transport is - * destroyed. */ static void frwr_op_recover_mr(struct rpcrdma_mw *mw) -- cgit v1.2.3-59-g8ed1b From 6d6bf72de914059b304f7b99530a7856e5c846aa Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 29 Nov 2016 10:53:13 -0500 Subject: xprtrdma: Squelch "max send, max recv" messages at connect time Clean up: This message was intended to be a dprintk, as it is on the server-side. Fixes: 87cfb9a0c85c ('xprtrdma: Client-side support for ...') Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- net/sunrpc/xprtrdma/verbs.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'net/sunrpc') diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index cbb18857841a..9f66fd862fd9 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -223,8 +223,8 @@ rpcrdma_update_connect_private(struct rpcrdma_xprt *r_xprt, cdata->inline_rsize = rsize; if (wsize < cdata->inline_wsize) cdata->inline_wsize = wsize; - pr_info("rpcrdma: max send %u, max recv %u\n", - cdata->inline_wsize, cdata->inline_rsize); + dprintk("RPC: %s: max send %u, max recv %u\n", + __func__, cdata->inline_wsize, cdata->inline_rsize); rpcrdma_set_max_header_sizes(r_xprt); } -- cgit v1.2.3-59-g8ed1b From 2f6922ca334e27155253e290416068ec30d391fa Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 29 Nov 2016 10:53:21 -0500 Subject: xprtrdma: Shorten QP access error message Clean up: The convention for this type of warning message is not to show the function name or "RPC: ". Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- net/sunrpc/xprtrdma/verbs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'net/sunrpc') diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index 9f66fd862fd9..11d07748f699 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -103,9 +103,9 @@ rpcrdma_qp_async_error_upcall(struct ib_event *event, void *context) { struct rpcrdma_ep *ep = context; - pr_err("RPC: %s: %s on device %s ep %p\n", - __func__, ib_event_msg(event->event), - event->device->name, context); + pr_err("rpcrdma: %s on device %s ep %p\n", + ib_event_msg(event->event), event->device->name, context); + if (ep->rep_connected == 1) { ep->rep_connected = -EIO; rpcrdma_conn_func(ep); -- cgit v1.2.3-59-g8ed1b From c351f943875499a950146afae136e8d0c2959453 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 29 Nov 2016 10:53:29 -0500 Subject: xprtrdma: Update dprintk in rpcrdma_count_chunks Clean up: offset and handle should be zero-filled, just like in the chunk encoders. Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- net/sunrpc/xprtrdma/rpc_rdma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'net/sunrpc') diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c index d987c2d3dd6e..01f5cbc4b21c 100644 --- a/net/sunrpc/xprtrdma/rpc_rdma.c +++ b/net/sunrpc/xprtrdma/rpc_rdma.c @@ -786,7 +786,7 @@ rpcrdma_count_chunks(struct rpcrdma_rep *rep, int wrchunk, __be32 **iptrp) ifdebug(FACILITY) { u64 off; xdr_decode_hyper((__be32 *)&seg->rs_offset, &off); - dprintk("RPC: %s: chunk %d@0x%llx:0x%x\n", + dprintk("RPC: %s: chunk %d@0x%016llx:0x%08x\n", __func__, be32_to_cpu(seg->rs_length), (unsigned long long)off, -- cgit v1.2.3-59-g8ed1b From 3a72dc771cc38e4d6e441a86256a3d7788a84c01 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 29 Nov 2016 10:53:37 -0500 Subject: xprtrdma: Relocate connection helper functions Clean up: Disentangle connection helpers from RPC-over-RDMA reply decoding functions. Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- net/sunrpc/xprtrdma/rpc_rdma.c | 34 ---------------------------------- net/sunrpc/xprtrdma/transport.c | 28 ++++++++++++++++++++++++++++ net/sunrpc/xprtrdma/xprt_rdma.h | 10 +++------- 3 files changed, 31 insertions(+), 41 deletions(-) (limited to 'net/sunrpc') diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c index 01f5cbc4b21c..c52e0f2ffe52 100644 --- a/net/sunrpc/xprtrdma/rpc_rdma.c +++ b/net/sunrpc/xprtrdma/rpc_rdma.c @@ -906,28 +906,6 @@ rpcrdma_inline_fixup(struct rpc_rqst *rqst, char *srcp, int copy_len, int pad) return fixup_copy_count; } -void -rpcrdma_connect_worker(struct work_struct *work) -{ - struct rpcrdma_ep *ep = - container_of(work, struct rpcrdma_ep, rep_connect_worker.work); - struct rpcrdma_xprt *r_xprt = - container_of(ep, struct rpcrdma_xprt, rx_ep); - struct rpc_xprt *xprt = &r_xprt->rx_xprt; - - spin_lock_bh(&xprt->transport_lock); - if (++xprt->connect_cookie == 0) /* maintain a reserved value */ - ++xprt->connect_cookie; - if (ep->rep_connected > 0) { - if (!xprt_test_and_set_connected(xprt)) - xprt_wake_pending_tasks(xprt, 0); - } else { - if (xprt_test_and_clear_connected(xprt)) - xprt_wake_pending_tasks(xprt, -ENOTCONN); - } - spin_unlock_bh(&xprt->transport_lock); -} - #if defined(CONFIG_SUNRPC_BACKCHANNEL) /* By convention, backchannel calls arrive via rdma_msg type * messages, and never populate the chunk lists. This makes @@ -959,18 +937,6 @@ rpcrdma_is_bcall(struct rpcrdma_msg *headerp) } #endif /* CONFIG_SUNRPC_BACKCHANNEL */ -/* - * This function is called when an async event is posted to - * the connection which changes the connection state. All it - * does at this point is mark the connection up/down, the rpc - * timers do the rest. - */ -void -rpcrdma_conn_func(struct rpcrdma_ep *ep) -{ - schedule_delayed_work(&ep->rep_connect_worker, 0); -} - /* Process received RPC/RDMA messages. * * Errors must result in the RPC task either being awakened, or diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c index 545d3fc8de5a..534c178d2a7e 100644 --- a/net/sunrpc/xprtrdma/transport.c +++ b/net/sunrpc/xprtrdma/transport.c @@ -219,6 +219,34 @@ xprt_rdma_free_addresses(struct rpc_xprt *xprt) } } +void +rpcrdma_conn_func(struct rpcrdma_ep *ep) +{ + schedule_delayed_work(&ep->rep_connect_worker, 0); +} + +void +rpcrdma_connect_worker(struct work_struct *work) +{ + struct rpcrdma_ep *ep = + container_of(work, struct rpcrdma_ep, rep_connect_worker.work); + struct rpcrdma_xprt *r_xprt = + container_of(ep, struct rpcrdma_xprt, rx_ep); + struct rpc_xprt *xprt = &r_xprt->rx_xprt; + + spin_lock_bh(&xprt->transport_lock); + if (++xprt->connect_cookie == 0) /* maintain a reserved value */ + ++xprt->connect_cookie; + if (ep->rep_connected > 0) { + if (!xprt_test_and_set_connected(xprt)) + xprt_wake_pending_tasks(xprt, 0); + } else { + if (xprt_test_and_clear_connected(xprt)) + xprt_wake_pending_tasks(xprt, -ENOTCONN); + } + spin_unlock_bh(&xprt->transport_lock); +} + static void xprt_rdma_connect_worker(struct work_struct *work) { diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h index b8424fa47d29..e35efd4ac1e4 100644 --- a/net/sunrpc/xprtrdma/xprt_rdma.h +++ b/net/sunrpc/xprtrdma/xprt_rdma.h @@ -490,6 +490,7 @@ int rpcrdma_ep_create(struct rpcrdma_ep *, struct rpcrdma_ia *, struct rpcrdma_create_data_internal *); void rpcrdma_ep_destroy(struct rpcrdma_ep *, struct rpcrdma_ia *); int rpcrdma_ep_connect(struct rpcrdma_ep *, struct rpcrdma_ia *); +void rpcrdma_conn_func(struct rpcrdma_ep *ep); void rpcrdma_ep_disconnect(struct rpcrdma_ep *, struct rpcrdma_ia *); int rpcrdma_ep_post(struct rpcrdma_ia *, struct rpcrdma_ep *, @@ -548,13 +549,6 @@ rpcrdma_data_dir(bool writing) return writing ? DMA_FROM_DEVICE : DMA_TO_DEVICE; } -/* - * RPC/RDMA connection management calls - xprtrdma/rpc_rdma.c - */ -void rpcrdma_connect_worker(struct work_struct *); -void rpcrdma_conn_func(struct rpcrdma_ep *); -void rpcrdma_reply_handler(struct work_struct *); - /* * RPC/RDMA protocol calls - xprtrdma/rpc_rdma.c */ @@ -572,12 +566,14 @@ bool rpcrdma_prepare_send_sges(struct rpcrdma_ia *, struct rpcrdma_req *, void rpcrdma_unmap_sges(struct rpcrdma_ia *, struct rpcrdma_req *); int rpcrdma_marshal_req(struct rpc_rqst *); void rpcrdma_set_max_header_sizes(struct rpcrdma_xprt *); +void rpcrdma_reply_handler(struct work_struct *work); /* RPC/RDMA module init - xprtrdma/transport.c */ extern unsigned int xprt_rdma_max_inline_read; void xprt_rdma_format_addresses(struct rpc_xprt *xprt, struct sockaddr *sap); void xprt_rdma_free_addresses(struct rpc_xprt *xprt); +void rpcrdma_connect_worker(struct work_struct *work); void xprt_rdma_print_stats(struct rpc_xprt *xprt, struct seq_file *seq); int xprt_rdma_init(void); void xprt_rdma_cleanup(void); -- cgit v1.2.3-59-g8ed1b