/* * Copyright(c) 2016 Intel Corporation. * * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. * * GPL LICENSE SUMMARY * * This program is free software; you can redistribute it and/or modify * it under the terms of version 2 of the GNU General Public License as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * BSD LICENSE * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * - Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * - Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * - Neither the name of Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ #include #include /* * Convert the AETH credit code into the number of credits. */ static const u16 credit_table[31] = { 0, /* 0 */ 1, /* 1 */ 2, /* 2 */ 3, /* 3 */ 4, /* 4 */ 6, /* 5 */ 8, /* 6 */ 12, /* 7 */ 16, /* 8 */ 24, /* 9 */ 32, /* A */ 48, /* B */ 64, /* C */ 96, /* D */ 128, /* E */ 192, /* F */ 256, /* 10 */ 384, /* 11 */ 512, /* 12 */ 768, /* 13 */ 1024, /* 14 */ 1536, /* 15 */ 2048, /* 16 */ 3072, /* 17 */ 4096, /* 18 */ 6144, /* 19 */ 8192, /* 1A */ 12288, /* 1B */ 16384, /* 1C */ 24576, /* 1D */ 32768 /* 1E */ }; /** * rvt_compute_aeth - compute the AETH (syndrome + MSN) * @qp: the queue pair to compute the AETH for * * Returns the AETH. */ __be32 rvt_compute_aeth(struct rvt_qp *qp) { u32 aeth = qp->r_msn & IB_MSN_MASK; if (qp->ibqp.srq) { /* * Shared receive queues don't generate credits. * Set the credit field to the invalid value. */ aeth |= IB_AETH_CREDIT_INVAL << IB_AETH_CREDIT_SHIFT; } else { u32 min, max, x; u32 credits; u32 head; u32 tail; credits = READ_ONCE(qp->r_rq.kwq->count); if (credits == 0) { /* sanity check pointers before trusting them */ if (qp->ip) { head = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->head); tail = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->tail); } else { head = READ_ONCE(qp->r_rq.kwq->head); tail = READ_ONCE(qp->r_rq.kwq->tail); } if (head >= qp->r_rq.size) head = 0; if (tail >= qp->r_rq.size) tail = 0; /* * Compute the number of credits available (RWQEs). * There is a small chance that the pair of reads are * not atomic, which is OK, since the fuzziness is * resolved as further ACKs go out. */ credits = head - tail; if ((int)credits < 0) credits += qp->r_rq.size; } /* * Binary search the credit table to find the code to * use. */ min = 0; max = 31; for (;;) { x = (min + max) / 2; if (credit_table[x] == credits) break; if (credit_table[x] > credits) { max = x; } else { if (min == x) break; min = x; } } aeth |= x << IB_AETH_CREDIT_SHIFT; } return cpu_to_be32(aeth); } EXPORT_SYMBOL(rvt_compute_aeth); /** * rvt_get_credit - flush the send work queue of a QP * @qp: the qp who's send work queue to flush * @aeth: the Acknowledge Extended Transport Header * * The QP s_lock should be held. */ void rvt_get_credit(struct rvt_qp *qp, u32 aeth) { struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); u32 credit = (aeth >> IB_AETH_CREDIT_SHIFT) & IB_AETH_CREDIT_MASK; lockdep_assert_held(&qp->s_lock); /* * If the credit is invalid, we can send * as many packets as we like. Otherwise, we have to * honor the credit field. */ if (credit == IB_AETH_CREDIT_INVAL) { if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) { qp->s_flags |= RVT_S_UNLIMITED_CREDIT; if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) { qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT; rdi->driver_f.schedule_send(qp); } } } else if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) { /* Compute new LSN (i.e., MSN + credit) */ credit = (aeth + credit_table[credit]) & IB_MSN_MASK; if (rvt_cmp_msn(credit, qp->s_lsn) > 0) { qp->s_lsn = credit; if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) { qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT; rdi->driver_f.schedule_send(qp); } } } } EXPORT_SYMBOL(rvt_get_credit); /* rvt_restart_sge - rewind the sge state for a wqe */ u32 rvt_restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, u32 len) { ss->sge = wqe->sg_list[0]; ss->sg_list = wqe->sg_list + 1; ss->num_sge = wqe->wr.num_sge; ss->total_len = wqe->length; rvt_skip_sge(ss, len, false); return wqe->length - len; } EXPORT_SYMBOL(rvt_restart_sge);