// SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause /* Authors: Bernard Metzler */ /* Copyright (c) 2008-2019, IBM Corporation */ #include #include #include #include #include #include #include #include "siw.h" #include "siw_verbs.h" #include "siw_mem.h" static char siw_qp_state_to_string[SIW_QP_STATE_COUNT][sizeof "TERMINATE"] = { [SIW_QP_STATE_IDLE] = "IDLE", [SIW_QP_STATE_RTR] = "RTR", [SIW_QP_STATE_RTS] = "RTS", [SIW_QP_STATE_CLOSING] = "CLOSING", [SIW_QP_STATE_TERMINATE] = "TERMINATE", [SIW_QP_STATE_ERROR] = "ERROR" }; /* * iWARP (RDMAP, DDP and MPA) parameters as well as Softiwarp settings on a * per-RDMAP message basis. Please keep order of initializer. All MPA len * is initialized to minimum packet size. */ struct iwarp_msg_info iwarp_pktinfo[RDMAP_TERMINATE + 1] = { { /* RDMAP_RDMA_WRITE */ .hdr_len = sizeof(struct iwarp_rdma_write), .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_write) - 2), .ctrl.ddp_rdmap_ctrl = DDP_FLAG_TAGGED | DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | cpu_to_be16(RDMAP_VERSION << 6) | cpu_to_be16(RDMAP_RDMA_WRITE), .rx_data = siw_proc_write }, { /* RDMAP_RDMA_READ_REQ */ .hdr_len = sizeof(struct iwarp_rdma_rreq), .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rreq) - 2), .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | cpu_to_be16(RDMAP_VERSION << 6) | cpu_to_be16(RDMAP_RDMA_READ_REQ), .rx_data = siw_proc_rreq }, { /* RDMAP_RDMA_READ_RESP */ .hdr_len = sizeof(struct iwarp_rdma_rresp), .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rresp) - 2), .ctrl.ddp_rdmap_ctrl = DDP_FLAG_TAGGED | DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | cpu_to_be16(RDMAP_VERSION << 6) | cpu_to_be16(RDMAP_RDMA_READ_RESP), .rx_data = siw_proc_rresp }, { /* RDMAP_SEND */ .hdr_len = sizeof(struct iwarp_send), .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2), .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | cpu_to_be16(RDMAP_VERSION << 6) | cpu_to_be16(RDMAP_SEND), .rx_data = siw_proc_send }, { /* RDMAP_SEND_INVAL */ .hdr_len = sizeof(struct iwarp_send_inv), .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2), .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | cpu_to_be16(RDMAP_VERSION << 6) | cpu_to_be16(RDMAP_SEND_INVAL), .rx_data = siw_proc_send }, { /* RDMAP_SEND_SE */ .hdr_len = sizeof(struct iwarp_send), .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2), .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | cpu_to_be16(RDMAP_VERSION << 6) | cpu_to_be16(RDMAP_SEND_SE), .rx_data = siw_proc_send }, { /* RDMAP_SEND_SE_INVAL */ .hdr_len = sizeof(struct iwarp_send_inv), .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2), .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | cpu_to_be16(RDMAP_VERSION << 6) | cpu_to_be16(RDMAP_SEND_SE_INVAL), .rx_data = siw_proc_send }, { /* RDMAP_TERMINATE */ .hdr_len = sizeof(struct iwarp_terminate), .ctrl.mpa_len = htons(sizeof(struct iwarp_terminate) - 2), .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | cpu_to_be16(RDMAP_VERSION << 6) | cpu_to_be16(RDMAP_TERMINATE), .rx_data = siw_proc_terminate } }; void siw_qp_llp_data_ready(struct sock *sk) { struct siw_qp *qp; read_lock(&sk->sk_callback_lock); if (unlikely(!sk->sk_user_data || !sk_to_qp(sk))) goto done; qp = sk_to_qp(sk); if (likely(!qp->rx_stream.rx_suspend && down_read_trylock(&qp->state_lock))) { read_descriptor_t rd_desc = { .arg.data = qp, .count = 1 }; if (likely(qp->attrs.state == SIW_QP_STATE_RTS)) /* * Implements data receive operation during * socket callback. TCP gracefully catches * the case where there is nothing to receive * (not calling siw_tcp_rx_data() then). */ tcp_read_sock(sk, &rd_desc, siw_tcp_rx_data); up_read(&qp->state_lock); } else { siw_dbg_qp(qp, "unable to process RX, suspend: %d\n", qp->rx_stream.rx_suspend); } done: read_unlock(&sk->sk_callback_lock); } void siw_qp_llp_close(struct siw_qp *qp) { siw_dbg_qp(qp, "enter llp close, state = %s\n", siw_qp_state_to_string[qp->attrs.state]); down_write(&qp->state_lock); qp->rx_stream.rx_suspend = 1; qp->tx_ctx.tx_suspend = 1; qp->attrs.sk = NULL; switch (qp->attrs.state) { case SIW_QP_STATE_RTS: case SIW_QP_STATE_RTR: case SIW_QP_STATE_IDLE: case SIW_QP_STATE_TERMINATE: qp->attrs.state = SIW_QP_STATE_ERROR; break; /* * SIW_QP_STATE_CLOSING: * * This is a forced close. shall the QP be moved to * ERROR or IDLE ? */ case SIW_QP_STATE_CLOSING: if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) qp->attrs.state = SIW_QP_STATE_ERROR; else qp->attrs.state = SIW_QP_STATE_IDLE; break; default: siw_dbg_qp(qp, "llp close: no state transition needed: %s\n", siw_qp_state_to_string[qp->attrs.state]); break; } siw_sq_flush(qp); siw_rq_flush(qp); /* * Dereference closing CEP */ if (qp->cep) { siw_cep_put(qp->cep); qp->cep = NULL; } up_write(&qp->state_lock); siw_dbg_qp(qp, "llp close exit: state %s\n", siw_qp_state_to_string[qp->attrs.state]); } /* * socket callback routine informing about newly available send space. * Function schedules SQ work for processing SQ items. */ void siw_qp_llp_write_space(struct sock *sk) { struct siw_cep *cep; read_lock(&sk->sk_callback_lock); cep = sk_to_cep(sk); if (cep) { cep->sk_write_space(sk); if (!test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) (void)siw_sq_start(cep->qp); } read_unlock(&sk->sk_callback_lock); } static int siw_qp_readq_init(struct siw_qp *qp, int irq_size, int orq_size) { irq_size = roundup_pow_of_two(irq_size); orq_size = roundup_pow_of_two(orq_size); qp->attrs.irq_size = irq_size; qp->attrs.orq_size = orq_size; qp->irq = vzalloc(irq_size * sizeof(struct siw_sqe)); if (!qp->irq) { siw_dbg_qp(qp, "irq malloc for %d failed\n", irq_size); qp->attrs.irq_size = 0; return -ENOMEM; } qp->orq = vzalloc(orq_size * sizeof(struct siw_sqe)); if (!qp->orq) { siw_dbg_qp(qp, "orq malloc for %d failed\n", orq_size); qp->attrs.orq_size = 0; qp->attrs.irq_size = 0; vfree(qp->irq); return -ENOMEM; } siw_dbg_qp(qp, "ORD %d, IRD %d\n", orq_size, irq_size); return 0; } static int siw_qp_enable_crc(struct siw_qp *qp) { struct siw_rx_stream *c_rx = &qp->rx_stream; struct siw_iwarp_tx *c_tx = &qp->tx_ctx; int size; if (siw_crypto_shash == NULL) return -ENOENT; size = crypto_shash_descsize(siw_crypto_shash) + sizeof(struct shash_desc); c_tx->mpa_crc_hd = kzalloc(size, GFP_KERNEL); c_rx->mpa_crc_hd = kzalloc(size, GFP_KERNEL); if (!c_tx->mpa_crc_hd || !c_rx->mpa_crc_hd) { kfree(c_tx->mpa_crc_hd); kfree(c_rx->mpa_crc_hd); c_tx->mpa_crc_hd = NULL; c_rx->mpa_crc_hd = NULL; return -ENOMEM; } c_tx->mpa_crc_hd->tfm = siw_crypto_shash; c_rx->mpa_crc_hd->tfm = siw_crypto_shash; return 0; } /* * Send a non signalled READ or WRITE to peer side as negotiated * with MPAv2 P2P setup protocol. The work request is only created * as a current active WR and does not consume Send Queue space. * * Caller must hold QP state lock. */ int siw_qp_mpa_rts(struct siw_qp *qp, enum mpa_v2_ctrl ctrl) { struct siw_wqe *wqe = tx_wqe(qp); unsigned long flags; int rv = 0; spin_lock_irqsave(&qp->sq_lock, flags); if (unlikely(wqe->wr_status != SIW_WR_IDLE)) { spin_unlock_irqrestore(&qp->sq_lock, flags); return -EIO; } memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE); wqe->wr_status = SIW_WR_QUEUED; wqe->sqe.flags = 0; wqe->sqe.num_sge = 1; wqe->sqe.sge[0].length = 0; wqe->sqe.sge[0].laddr = 0; wqe->sqe.sge[0].lkey = 0; /* * While it must not be checked for inbound zero length * READ/WRITE, some HW may treat STag 0 special. */ wqe->sqe.rkey = 1; wqe->sqe.raddr = 0; wqe->processed = 0; if (ctrl & MPA_V2_RDMA_WRITE_RTR) wqe->sqe.opcode = SIW_OP_WRITE; else if (ctrl & MPA_V2_RDMA_READ_RTR) { struct siw_sqe *rreq; wqe->sqe.opcode = SIW_OP_READ; spin_lock(&qp->orq_lock); rreq = orq_get_free(qp); if (rreq) { siw_read_to_orq(rreq, &wqe->sqe); qp->orq_put++; } else rv = -EIO; spin_unlock(&qp->orq_lock); } else rv = -EINVAL; if (rv) wqe->wr_status = SIW_WR_IDLE; spin_unlock_irqrestore(&qp->sq_lock, flags); if (!rv) rv = siw_sq_start(qp); return rv; } /* * Map memory access error to DDP tagged error */ enum ddp_ecode siw_tagged_error(enum siw_access_state state) { switch (state) { case E_STAG_INVALID: return DDP_ECODE_T_INVALID_STAG; case E_BASE_BOUNDS: return DDP_ECODE_T_BASE_BOUNDS; case E_PD_MISMATCH: return DDP_ECODE_T_STAG_NOT_ASSOC; case E_ACCESS_PERM: /* * RFC 5041 (DDP) lacks an ecode for insufficient access * permissions. 'Invalid STag' seem to be the closest * match though. */ return DDP_ECODE_T_INVALID_STAG; default: WARN_ON(1); return DDP_ECODE_T_INVALID_STAG; } } /* * Map memory access error to RDMAP protection error */ enum rdmap_ecode siw_rdmap_error(enum siw_access_state state) { switch (state) { case E_STAG_INVALID: return RDMAP_ECODE_INVALID_STAG; case E_BASE_BOUNDS: return RDMAP_ECODE_BASE_BOUNDS; case E_PD_MISMATCH: return RDMAP_ECODE_STAG_NOT_ASSOC; case E_ACCESS_PERM: return RDMAP_ECODE_ACCESS_RIGHTS; default: return RDMAP_ECODE_UNSPECIFIED; } } void siw_init_terminate(struct siw_qp *qp, enum term_elayer layer, u8 etype, u8 ecode, int in_tx) { if (!qp->term_info.valid) { memset(&qp->term_info, 0, sizeof(qp->term_info)); qp->term_info.layer = layer; qp->term_info.etype = etype; qp->term_info.ecode = ecode; qp->term_info.in_tx = in_tx; qp->term_info.valid = 1; } siw_dbg_qp(qp, "init TERM: layer %d, type %d, code %d, in tx %s\n", layer, etype, ecode, in_tx ? "yes" : "no"); } /* * Send a TERMINATE message, as defined in RFC's 5040/5041/5044/6581. * Sending TERMINATE messages is best effort - such messages * can only be send if the QP is still connected and it does * not have another outbound message in-progress, i.e. the * TERMINATE message must not interfer with an incomplete current * transmit operation. */ void siw_send_terminate(struct siw_qp *qp) { struct kvec iov[3]; struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_EOR }; struct iwarp_terminate *term = NULL; union iwarp_hdr *err_hdr = NULL; struct socket *s = qp->attrs.sk; struct siw_rx_stream *srx = &qp->rx_stream; union iwarp_hdr *rx_hdr = &srx->hdr; u32 crc = 0; int num_frags, len_terminate, rv; if (!qp->term_info.valid) return; qp->term_info.valid = 0; if (tx_wqe(qp)->wr_status == SIW_WR_INPROGRESS) { siw_dbg_qp(qp, "cannot send TERMINATE: op %d in progress\n", tx_type(tx_wqe(qp))); return; } if (!s && qp->cep) /* QP not yet in RTS. Take socket from connection end point */ s = qp->cep->sock; if (!s) { siw_dbg_qp(qp, "cannot send TERMINATE: not connected\n"); return; } term = kzalloc(sizeof(*term), GFP_KERNEL); if (!term) return; term->ddp_qn = cpu_to_be32(RDMAP_UNTAGGED_QN_TERMINATE); term->ddp_mo = 0; term->ddp_msn = cpu_to_be32(1); iov[0].iov_base = term; iov[0].iov_len = sizeof(*term); if ((qp->term_info.layer == TERM_ERROR_LAYER_DDP) || ((qp->term_info.layer == TERM_ERROR_LAYER_RDMAP) && (qp->term_info.etype != RDMAP_ETYPE_CATASTROPHIC))) { err_hdr = kzalloc(sizeof(*err_hdr), GFP_KERNEL); if (!err_hdr) { kfree(term); return; } } memcpy(&term->ctrl, &iwarp_pktinfo[RDMAP_TERMINATE].ctrl, sizeof(struct iwarp_ctrl)); __rdmap_term_set_layer(term, qp->term_info.layer); __rdmap_term_set_etype(term, qp->term_info.etype); __rdmap_term_set_ecode(term, qp->term_info.ecode); switch (qp->term_info.layer) { case TERM_ERROR_LAYER_RDMAP: if (qp->term_info.etype == RDMAP_ETYPE_CATASTROPHIC) /* No additional DDP/RDMAP header to be included */ break; if (qp->term_info.etype == RDMAP_ETYPE_REMOTE_PROTECTION) { /* * Complete RDMAP frame will get attached, and * DDP segment length is valid */ term->flag_m = 1; term->flag_d = 1; term->flag_r = 1; if (qp->term_info.in_tx) { struct iwarp_rdma_rreq *rreq; struct siw_wqe *wqe = tx_wqe(qp); /* Inbound RREQ error, detected during * RRESP creation. Take state from * current TX work queue element to * reconstruct peers RREQ. */ rreq = (struct iwarp_rdma_rreq *)err_hdr; memcpy(&rreq->ctrl, &iwarp_pktinfo[RDMAP_RDMA_READ_REQ].ctrl, sizeof(struct iwarp_ctrl)); rreq->rsvd = 0; rreq->ddp_qn = htonl(RDMAP_UNTAGGED_QN_RDMA_READ); /* Provide RREQ's MSN as kept aside */ rreq->ddp_msn = htonl(wqe->sqe.sge[0].length); rreq->ddp_mo = htonl(wqe->processed); rreq->sink_stag = htonl(wqe->sqe.rkey); rreq->sink_to = cpu_to_be64(wqe->sqe.raddr); rreq->read_size = htonl(wqe->sqe.sge[0].length); rreq->source_stag = htonl(wqe->sqe.sge[0].lkey); rreq->source_to = cpu_to_be64(wqe->sqe.sge[0].laddr); iov[1].iov_base = rreq; iov[1].iov_len = sizeof(*rreq); rx_hdr = (union iwarp_hdr *)rreq; } else { /* Take RDMAP/DDP information from * current (failed) inbound frame. */ iov[1].iov_base = rx_hdr; if (__rdmap_get_opcode(&rx_hdr->ctrl) == RDMAP_RDMA_READ_REQ) iov[1].iov_len = sizeof(struct iwarp_rdma_rreq); else /* SEND type */ iov[1].iov_len = sizeof(struct iwarp_send); } } else { /* Do not report DDP hdr information if packet * layout is unknown */ if ((qp->term_info.ecode == RDMAP_ECODE_VERSION) || (qp->term_info.ecode == RDMAP_ECODE_OPCODE)) break; iov[1].iov_base = rx_hdr; /* Only DDP frame will get attached */ if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED) iov[1].iov_len = sizeof(struct iwarp_rdma_write); else iov[1].iov_len = sizeof(struct iwarp_send); term->flag_m = 1; term->flag_d = 1; } term->ctrl.mpa_len = cpu_to_be16(iov[1].iov_len); break; case TERM_ERROR_LAYER_DDP: /* Report error encountered while DDP processing. * This can only happen as a result of inbound * DDP processing */ /* Do not report DDP hdr information if packet * layout is unknown */ if (((qp->term_info.etype == DDP_ETYPE_TAGGED_BUF) && (qp->term_info.ecode == DDP_ECODE_T_VERSION)) || ((qp->term_info.etype == DDP_ETYPE_UNTAGGED_BUF) && (qp->term_info.ecode == DDP_ECODE_UT_VERSION))) break; iov[1].iov_base = rx_hdr; if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED) iov[1].iov_len = sizeof(struct iwarp_ctrl_tagged); else iov[1].iov_len = sizeof(struct iwarp_ctrl_untagged); term->flag_m = 1; term->flag_d = 1; break; default: break; } if (term->flag_m || term->flag_d || term->flag_r) { iov[2].iov_base = &crc; iov[2].iov_len = sizeof(crc); len_terminate = sizeof(*term) + iov[1].iov_len + MPA_CRC_SIZE; num_frags = 3; } else { iov[1].iov_base = &crc; iov[1].iov_len = sizeof(crc); len_terminate = sizeof(*term) + MPA_CRC_SIZE; num_frags = 2; } /* Adjust DDP Segment Length parameter, if valid */ if (term->flag_m) { u32 real_ddp_len = be16_to_cpu(rx_hdr->ctrl.mpa_len); enum rdma_opcode op = __rdmap_get_opcode(&rx_hdr->ctrl); real_ddp_len -= iwarp_pktinfo[op].hdr_len - MPA_HDR_SIZE; rx_hdr->ctrl.mpa_len = cpu_to_be16(real_ddp_len); } term->ctrl.mpa_len = cpu_to_be16(len_terminate - (MPA_HDR_SIZE + MPA_CRC_SIZE)); if (qp->tx_ctx.mpa_crc_hd) { crypto_shash_init(qp->tx_ctx.mpa_crc_hd); if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd, (u8 *)iov[0].iov_base, iov[0].iov_len)) goto out; if (num_frags == 3) { if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd, (u8 *)iov[1].iov_base, iov[1].iov_len)) goto out; } crypto_shash_final(qp->tx_ctx.mpa_crc_hd, (u8 *)&crc); } rv = kernel_sendmsg(s, &msg, iov, num_frags, len_terminate); siw_dbg_qp(qp, "sent TERM: %s, layer %d, type %d, code %d (%d bytes)\n", rv == len_terminate ? "success" : "failure", __rdmap_term_layer(term), __rdmap_term_etype(term), __rdmap_term_ecode(term), rv); out: kfree(term); kfree(err_hdr); } /* * Handle all attrs other than state */ static void siw_qp_modify_nonstate(struct siw_qp *qp, struct siw_qp_attrs *attrs, enum siw_qp_attr_mask mask) { if (mask & SIW_QP_ATTR_ACCESS_FLAGS) { if (attrs->flags & SIW_RDMA_BIND_ENABLED) qp->attrs.flags |= SIW_RDMA_BIND_ENABLED; else qp->attrs.flags &= ~SIW_RDMA_BIND_ENABLED; if (attrs->flags & SIW_RDMA_WRITE_ENABLED) qp->attrs.flags |= SIW_RDMA_WRITE_ENABLED; else qp->attrs.flags &= ~SIW_RDMA_WRITE_ENABLED; if (attrs->flags & SIW_RDMA_READ_ENABLED) qp->attrs.flags |= SIW_RDMA_READ_ENABLED; else qp->attrs.flags &= ~SIW_RDMA_READ_ENABLED; } } static int siw_qp_nextstate_from_idle(struct siw_qp *qp, struct siw_qp_attrs *attrs, enum siw_qp_attr_mask mask) { int rv = 0; switch (attrs->state) { case SIW_QP_STATE_RTS: if (attrs->flags & SIW_MPA_CRC) { rv = siw_qp_enable_crc(qp); if (rv) break; } if (!(mask & SIW_QP_ATTR_LLP_HANDLE)) { siw_dbg_qp(qp, "no socket\n"); rv = -EINVAL; break; } if (!(mask & SIW_QP_ATTR_MPA)) { siw_dbg_qp(qp, "no MPA\n"); rv = -EINVAL; break; } /* * Initialize iWARP TX state */ qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 0; qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 0; qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 0; /* * Initialize iWARP RX state */ qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 1; qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 1; qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 1; /* * init IRD free queue, caller has already checked * limits. */ rv = siw_qp_readq_init(qp, attrs->irq_size, attrs->orq_size); if (rv) break; qp->attrs.sk = attrs->sk; qp->attrs.state = SIW_QP_STATE_RTS; siw_dbg_qp(qp, "enter RTS: crc=%s, ord=%u, ird=%u\n", attrs->flags & SIW_MPA_CRC ? "y" : "n", qp->attrs.orq_size, qp->attrs.irq_size); break; case SIW_QP_STATE_ERROR: siw_rq_flush(qp); qp->attrs.state = SIW_QP_STATE_ERROR; if (qp->cep) { siw_cep_put(qp->cep); qp->cep = NULL; } break; default: break; } return rv; } static int siw_qp_nextstate_from_rts(struct siw_qp *qp, struct siw_qp_attrs *attrs) { int drop_conn = 0; switch (attrs->state) { case SIW_QP_STATE_CLOSING: /* * Verbs: move to IDLE if SQ and ORQ are empty. * Move to ERROR otherwise. But first of all we must * close the connection. So we keep CLOSING or ERROR * as a transient state, schedule connection drop work * and wait for the socket state change upcall to * come back closed. */ if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) { qp->attrs.state = SIW_QP_STATE_CLOSING; } else { qp->attrs.state = SIW_QP_STATE_ERROR; siw_sq_flush(qp); } siw_rq_flush(qp); drop_conn = 1; break; case SIW_QP_STATE_TERMINATE: qp->attrs.state = SIW_QP_STATE_TERMINATE; siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP, RDMAP_ETYPE_CATASTROPHIC, RDMAP_ECODE_UNSPECIFIED, 1); drop_conn = 1; break; case SIW_QP_STATE_ERROR: /* * This is an emergency close. * * Any in progress transmit operation will get * cancelled. * This will likely result in a protocol failure, * if a TX operation is in transit. The caller * could unconditional wait to give the current * operation a chance to complete. * Esp., how to handle the non-empty IRQ case? * The peer was asking for data transfer at a valid * point in time. */ siw_sq_flush(qp); siw_rq_flush(qp); qp->attrs.state = SIW_QP_STATE_ERROR; drop_conn = 1; break; default: break; } return drop_conn; } static void siw_qp_nextstate_from_term(struct siw_qp *qp, struct siw_qp_attrs *attrs) { switch (attrs->state) { case SIW_QP_STATE_ERROR: siw_rq_flush(qp); qp->attrs.state = SIW_QP_STATE_ERROR; if (tx_wqe(qp)->wr_status != SIW_WR_IDLE) siw_sq_flush(qp); break; default: break; } } static int siw_qp_nextstate_from_close(struct siw_qp *qp, struct siw_qp_attrs *attrs) { int rv = 0; switch (attrs->state) { case SIW_QP_STATE_IDLE: WARN_ON(tx_wqe(qp)->wr_status != SIW_WR_IDLE); qp->attrs.state = SIW_QP_STATE_IDLE; break; case SIW_QP_STATE_CLOSING: /* * The LLP may already moved the QP to closing * due to graceful peer close init */ break; case SIW_QP_STATE_ERROR: /* * QP was moved to CLOSING by LLP event * not yet seen by user. */ qp->attrs.state = SIW_QP_STATE_ERROR; if (tx_wqe(qp)->wr_status != SIW_WR_IDLE) siw_sq_flush(qp); siw_rq_flush(qp); break; default: siw_dbg_qp(qp, "state transition undefined: %s => %s\n", siw_qp_state_to_string[qp->attrs.state], siw_qp_state_to_string[attrs->state]); rv = -ECONNABORTED; } return rv; } /* * Caller must hold qp->state_lock */ int siw_qp_modify(struct siw_qp *qp, struct siw_qp_attrs *attrs, enum siw_qp_attr_mask mask) { int drop_conn = 0, rv = 0; if (!mask) return 0; siw_dbg_qp(qp, "state: %s => %s\n", siw_qp_state_to_string[qp->attrs.state], siw_qp_state_to_string[attrs->state]); if (mask != SIW_QP_ATTR_STATE) siw_qp_modify_nonstate(qp, attrs, mask); if (!(mask & SIW_QP_ATTR_STATE)) return 0; switch (qp->attrs.state) { case SIW_QP_STATE_IDLE: case SIW_QP_STATE_RTR: rv = siw_qp_nextstate_from_idle(qp, attrs, mask); break; case SIW_QP_STATE_RTS: drop_conn = siw_qp_nextstate_from_rts(qp, attrs); break; case SIW_QP_STATE_TERMINATE: siw_qp_nextstate_from_term(qp, attrs); break; case SIW_QP_STATE_CLOSING: siw_qp_nextstate_from_close(qp, attrs); break; default: break; } if (drop_conn) siw_qp_cm_drop(qp, 0); return rv; } void siw_read_to_orq(struct siw_sqe *rreq, struct siw_sqe *sqe) { rreq->id = sqe->id; rreq->opcode = sqe->opcode; rreq->sge[0].laddr = sqe->sge[0].laddr; rreq->sge[0].length = sqe->sge[0].length; rreq->sge[0].lkey = sqe->sge[0].lkey; rreq->sge[1].lkey = sqe->sge[1].lkey; rreq->flags = sqe->flags | SIW_WQE_VALID; rreq->num_sge = 1; } /* * Must be called with SQ locked. * To avoid complete SQ starvation by constant inbound READ requests, * the active IRQ will not be served after qp->irq_burst, if the * SQ has pending work. */ int siw_activate_tx(struct siw_qp *qp) { struct siw_sqe *irqe, *sqe; struct siw_wqe *wqe = tx_wqe(qp); int rv = 1; irqe = &qp->irq[qp->irq_get % qp->attrs.irq_size]; if (irqe->flags & SIW_WQE_VALID) { sqe = sq_get_next(qp); /* * Avoid local WQE processing starvation in case * of constant inbound READ request stream */ if (sqe && ++qp->irq_burst >= SIW_IRQ_MAXBURST_SQ_ACTIVE) { qp->irq_burst = 0; goto skip_irq; } memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE); wqe->wr_status = SIW_WR_QUEUED; /* start READ RESPONSE */ wqe->sqe.opcode = SIW_OP_READ_RESPONSE; wqe->sqe.flags = 0; if (irqe->num_sge) { wqe->sqe.num_sge = 1; wqe->sqe.sge[0].length = irqe->sge[0].length; wqe->sqe.sge[0].laddr = irqe->sge[0].laddr; wqe->sqe.sge[0].lkey = irqe->sge[0].lkey; } else { wqe->sqe.num_sge = 0; } /* Retain original RREQ's message sequence number for * potential error reporting cases. */ wqe->sqe.sge[1].length = irqe->sge[1].length; wqe->sqe.rkey = irqe->rkey; wqe->sqe.raddr = irqe->raddr; wqe->processed = 0; qp->irq_get++; /* mark current IRQ entry free */ smp_store_mb(irqe->flags, 0); goto out; } sqe = sq_get_next(qp); if (sqe) { skip_irq: memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE); wqe->wr_status = SIW_WR_QUEUED; /* First copy SQE to kernel private memory */ memcpy(&wqe->sqe, sqe, sizeof(*sqe)); if (wqe->sqe.opcode >= SIW_NUM_OPCODES) { rv = -EINVAL; goto out; } if (wqe->sqe.flags & SIW_WQE_INLINE) { if (wqe->sqe.opcode != SIW_OP_SEND && wqe->sqe.opcode != SIW_OP_WRITE) { rv = -EINVAL; goto out; } if (wqe->sqe.sge[0].length > SIW_MAX_INLINE) { rv = -EINVAL; goto out; } wqe->sqe.sge[0].laddr = (uintptr_t)&wqe->sqe.sge[1]; wqe->sqe.sge[0].lkey = 0; wqe->sqe.num_sge = 1; } if (wqe->sqe.flags & SIW_WQE_READ_FENCE) { /* A READ cannot be fenced */ if (unlikely(wqe->sqe.opcode == SIW_OP_READ || wqe->sqe.opcode == SIW_OP_READ_LOCAL_INV)) { siw_dbg_qp(qp, "cannot fence read\n"); rv = -EINVAL; goto out; } spin_lock(&qp->orq_lock); if (!siw_orq_empty(qp)) { qp->tx_ctx.orq_fence = 1; rv = 0; } spin_unlock(&qp->orq_lock); } else if (wqe->sqe.opcode == SIW_OP_READ || wqe->sqe.opcode == SIW_OP_READ_LOCAL_INV) { struct siw_sqe *rreq; wqe->sqe.num_sge = 1; spin_lock(&qp->orq_lock); rreq = orq_get_free(qp); if (rreq) { /* * Make an immediate copy in ORQ to be ready * to process loopback READ reply */ siw_read_to_orq(rreq, &wqe->sqe); qp->orq_put++; } else { qp->tx_ctx.orq_fence = 1; rv = 0; } spin_unlock(&qp->orq_lock); } /* Clear SQE, can be re-used by application */ smp_store_mb(sqe->flags, 0); qp->sq_get++; } else { rv = 0; } out: if (unlikely(rv < 0)) { siw_dbg_qp(qp, "error %d\n", rv); wqe->wr_status = SIW_WR_IDLE; } return rv; } /* * Check if current CQ state qualifies for calling CQ completion * handler. Must be called with CQ lock held. */ static bool siw_cq_notify_now(struct siw_cq *cq, u32 flags) { u32 cq_notify; if (!cq->base_cq.comp_handler) return false; /* Read application shared notification state */ cq_notify = READ_ONCE(cq->notify->flags); if ((cq_notify & SIW_NOTIFY_NEXT_COMPLETION) || ((cq_notify & SIW_NOTIFY_SOLICITED) && (flags & SIW_WQE_SOLICITED))) { /* * CQ notification is one-shot: Since the * current CQE causes user notification, * the CQ gets dis-aremd and must be re-aremd * by the user for a new notification. */ WRITE_ONCE(cq->notify->flags, SIW_NOTIFY_NOT); return true; } return false; } int siw_sqe_complete(struct siw_qp *qp, struct siw_sqe *sqe, u32 bytes, enum siw_wc_status status) { struct siw_cq *cq = qp->scq; int rv = 0; if (cq) { u32 sqe_flags = sqe->flags; struct siw_cqe *cqe; u32 idx; unsigned long flags; spin_lock_irqsave(&cq->lock, flags); idx = cq->cq_put % cq->num_cqe; cqe = &cq->queue[idx]; if (!READ_ONCE(cqe->flags)) { bool notify; cqe->id = sqe->id; cqe->opcode = sqe->opcode; cqe->status = status; cqe->imm_data = 0; cqe->bytes = bytes; if (rdma_is_kernel_res(&cq->base_cq.res)) cqe->base_qp = &qp->base_qp; else cqe->qp_id = qp_id(qp); /* mark CQE valid for application */ WRITE_ONCE(cqe->flags, SIW_WQE_VALID); /* recycle SQE */ smp_store_mb(sqe->flags, 0); cq->cq_put++; notify = siw_cq_notify_now(cq, sqe_flags); spin_unlock_irqrestore(&cq->lock, flags); if (notify) { siw_dbg_cq(cq, "Call completion handler\n"); cq->base_cq.comp_handler(&cq->base_cq, cq->base_cq.cq_context); } } else { spin_unlock_irqrestore(&cq->lock, flags); rv = -ENOMEM; siw_cq_event(cq, IB_EVENT_CQ_ERR); } } else { /* recycle SQE */ smp_store_mb(sqe->flags, 0); } return rv; } int siw_rqe_complete(struct siw_qp *qp, struct siw_rqe *rqe, u32 bytes, u32 inval_stag, enum siw_wc_status status) { struct siw_cq *cq = qp->rcq; int rv = 0; if (cq) { struct siw_cqe *cqe; u32 idx; unsigned long flags; spin_lock_irqsave(&cq->lock, flags); idx = cq->cq_put % cq->num_cqe; cqe = &cq->queue[idx]; if (!READ_ONCE(cqe->flags)) { bool notify; u8 cqe_flags = SIW_WQE_VALID; cqe->id = rqe->id; cqe->opcode = SIW_OP_RECEIVE; cqe->status = status; cqe->imm_data = 0; cqe->bytes = bytes; if (rdma_is_kernel_res(&cq->base_cq.res)) { cqe->base_qp = &qp->base_qp; if (inval_stag) { cqe_flags |= SIW_WQE_REM_INVAL; cqe->inval_stag = inval_stag; } } else { cqe->qp_id = qp_id(qp); } /* mark CQE valid for application */ WRITE_ONCE(cqe->flags, cqe_flags); /* recycle RQE */ smp_store_mb(rqe->flags, 0); cq->cq_put++; notify = siw_cq_notify_now(cq, SIW_WQE_SIGNALLED); spin_unlock_irqrestore(&cq->lock, flags); if (notify) { siw_dbg_cq(cq, "Call completion handler\n"); cq->base_cq.comp_handler(&cq->base_cq, cq->base_cq.cq_context); } } else { spin_unlock_irqrestore(&cq->lock, flags); rv = -ENOMEM; siw_cq_event(cq, IB_EVENT_CQ_ERR); } } else { /* recycle RQE */ smp_store_mb(rqe->flags, 0); } return rv; } /* * siw_sq_flush() * * Flush SQ and ORRQ entries to CQ. * * Must be called with QP state write lock held. * Therefore, SQ and ORQ lock must not be taken. */ void siw_sq_flush(struct siw_qp *qp) { struct siw_sqe *sqe; struct siw_wqe *wqe = tx_wqe(qp); int async_event = 0; /* * Start with completing any work currently on the ORQ */ while (qp->attrs.orq_size) { sqe = &qp->orq[qp->orq_get % qp->attrs.orq_size]; if (!READ_ONCE(sqe->flags)) break; if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0) break; WRITE_ONCE(sqe->flags, 0); qp->orq_get++; } /* * Flush an in-progress WQE if present */ if (wqe->wr_status != SIW_WR_IDLE) { siw_dbg_qp(qp, "flush current SQE, type %d, status %d\n", tx_type(wqe), wqe->wr_status); siw_wqe_put_mem(wqe, tx_type(wqe)); if (tx_type(wqe) != SIW_OP_READ_RESPONSE && ((tx_type(wqe) != SIW_OP_READ && tx_type(wqe) != SIW_OP_READ_LOCAL_INV) || wqe->wr_status == SIW_WR_QUEUED)) /* * An in-progress Read Request is already in * the ORQ */ siw_sqe_complete(qp, &wqe->sqe, wqe->bytes, SIW_WC_WR_FLUSH_ERR); wqe->wr_status = SIW_WR_IDLE; } /* * Flush the Send Queue */ while (qp->attrs.sq_size) { sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size]; if (!READ_ONCE(sqe->flags)) break; async_event = 1; if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0) /* * Shall IB_EVENT_SQ_DRAINED be supressed if work * completion fails? */ break; WRITE_ONCE(sqe->flags, 0); qp->sq_get++; } if (async_event) siw_qp_event(qp, IB_EVENT_SQ_DRAINED); } /* * siw_rq_flush() * * Flush recv queue entries to CQ. Also * takes care of pending active tagged and untagged * inbound transfers, which have target memory * referenced. * * Must be called with QP state write lock held. * Therefore, RQ lock must not be taken. */ void siw_rq_flush(struct siw_qp *qp) { struct siw_wqe *wqe = &qp->rx_untagged.wqe_active; /* * Flush an in-progress untagged operation if present */ if (wqe->wr_status != SIW_WR_IDLE) { siw_dbg_qp(qp, "flush current rqe, type %d, status %d\n", rx_type(wqe), wqe->wr_status); siw_wqe_put_mem(wqe, rx_type(wqe)); if (rx_type(wqe) == SIW_OP_RECEIVE) { siw_rqe_complete(qp, &wqe->rqe, wqe->bytes, 0, SIW_WC_WR_FLUSH_ERR); } else if (rx_type(wqe) != SIW_OP_READ && rx_type(wqe) != SIW_OP_READ_RESPONSE && rx_type(wqe) != SIW_OP_WRITE) { siw_sqe_complete(qp, &wqe->sqe, 0, SIW_WC_WR_FLUSH_ERR); } wqe->wr_status = SIW_WR_IDLE; } wqe = &qp->rx_tagged.wqe_active; if (wqe->wr_status != SIW_WR_IDLE) { siw_wqe_put_mem(wqe, rx_type(wqe)); wqe->wr_status = SIW_WR_IDLE; } /* * Flush the Receive Queue */ while (qp->attrs.rq_size) { struct siw_rqe *rqe = &qp->recvq[qp->rq_get % qp->attrs.rq_size]; if (!READ_ONCE(rqe->flags)) break; if (siw_rqe_complete(qp, rqe, 0, 0, SIW_WC_WR_FLUSH_ERR) != 0) break; WRITE_ONCE(rqe->flags, 0); qp->rq_get++; } } int siw_qp_add(struct siw_device *sdev, struct siw_qp *qp) { int rv = xa_alloc(&sdev->qp_xa, &qp->base_qp.qp_num, qp, xa_limit_32b, GFP_KERNEL); if (!rv) { kref_init(&qp->ref); qp->sdev = sdev; siw_dbg_qp(qp, "new QP\n"); } return rv; } void siw_free_qp(struct kref *ref) { struct siw_qp *found, *qp = container_of(ref, struct siw_qp, ref); struct siw_device *sdev = qp->sdev; unsigned long flags; if (qp->cep) siw_cep_put(qp->cep); found = xa_erase(&sdev->qp_xa, qp_id(qp)); WARN_ON(found != qp); spin_lock_irqsave(&sdev->lock, flags); list_del(&qp->devq); spin_unlock_irqrestore(&sdev->lock, flags); vfree(qp->sendq); vfree(qp->recvq); vfree(qp->irq); vfree(qp->orq); siw_put_tx_cpu(qp->tx_cpu); atomic_dec(&sdev->num_qp); siw_dbg_qp(qp, "free QP\n"); kfree_rcu(qp, rcu); }