From a242c36951ecd24bc16086940dbe6b522205c461 Mon Sep 17 00:00:00 2001 From: Håkon Bugge Date: Mon, 16 Dec 2019 13:04:36 +0100 Subject: RDMA/netlink: Do not always generate an ACK for some netlink operations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In rdma_nl_rcv_skb(), the local variable err is assigned the return value of the supplied callback function, which could be one of ib_nl_handle_resolve_resp(), ib_nl_handle_set_timeout(), or ib_nl_handle_ip_res_resp(). These three functions all return skb->len on success. rdma_nl_rcv_skb() is merely a copy of netlink_rcv_skb(). The callback functions used by the latter have the convention: "Returns 0 on success or a negative error code". In particular, the statement (equal for both functions): if (nlh->nlmsg_flags & NLM_F_ACK || err) implies that rdma_nl_rcv_skb() always will ack a message, independent of the NLM_F_ACK being set in nlmsg_flags or not. The fix could be to change the above statement, but it is better to keep the two *_rcv_skb() functions equal in this respect and instead change the three callback functions in the rdma subsystem to the correct convention. Fixes: 2ca546b92a02 ("IB/sa: Route SA pathrecord query through netlink") Fixes: ae43f8286730 ("IB/core: Add IP to GID netlink offload") Link: https://lore.kernel.org/r/20191216120436.3204814-1-haakon.bugge@oracle.com Suggested-by: Mark Haywood Signed-off-by: Håkon Bugge Tested-by: Mark Haywood Reviewed-by: Leon Romanovsky Reviewed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/addr.c | 2 +- drivers/infiniband/core/sa_query.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c index 606fa6d86685..1753a9801b70 100644 --- a/drivers/infiniband/core/addr.c +++ b/drivers/infiniband/core/addr.c @@ -139,7 +139,7 @@ int ib_nl_handle_ip_res_resp(struct sk_buff *skb, if (ib_nl_is_good_ip_resp(nlh)) ib_nl_process_good_ip_rsep(nlh); - return skb->len; + return 0; } static int ib_nl_ip_send_msg(struct rdma_dev_addr *dev_addr, diff --git a/drivers/infiniband/core/sa_query.c b/drivers/infiniband/core/sa_query.c index 8917125ea16d..30d4c126a2db 100644 --- a/drivers/infiniband/core/sa_query.c +++ b/drivers/infiniband/core/sa_query.c @@ -1068,7 +1068,7 @@ int ib_nl_handle_set_timeout(struct sk_buff *skb, } settimeout_out: - return skb->len; + return 0; } static inline int ib_nl_is_good_resolve_resp(const struct nlmsghdr *nlh) @@ -1139,7 +1139,7 @@ int ib_nl_handle_resolve_resp(struct sk_buff *skb, } resp_out: - return skb->len; + return 0; } static void free_sm_ah(struct kref *kref) -- cgit v1.2.3-59-g8ed1b From 4d6e8a033f7c6665c7ca5c5cdcf66019baa4b7a2 Mon Sep 17 00:00:00 2001 From: Danit Goldberg Date: Thu, 19 Dec 2019 15:47:50 +0200 Subject: RDMA/cm: Use RCU synchronization mechanism to protect cm_id_private xa_load() The RCU mechanism is optimized for read-mostly scenarios and therefore more suitable to protect the cm_id_private to decrease "cm.lock" congestion. This patch replaces the existing spinlock locking mechanism and kfree with RCU mechanism in places where spinlock(cm.lock) protected xa_load returning the cm_id_priv In addition, delete the cm_get_id() function as there is no longer a distinction if the caller already holds the cm_lock. Remove an open coded version of cm_get_id(). Link: https://lore.kernel.org/r/20191219134750.413429-1-leon@kernel.org Signed-off-by: Danit Goldberg Signed-off-by: Leon Romanovsky Reviewed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/cm.c | 43 +++++++++++++------------------------------ 1 file changed, 13 insertions(+), 30 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c index 455b3659d84b..2945031dd8a7 100644 --- a/drivers/infiniband/core/cm.c +++ b/drivers/infiniband/core/cm.c @@ -241,6 +241,7 @@ struct cm_id_private { /* Number of clients sharing this ib_cm_id. Only valid for listeners. * Protected by the cm.lock spinlock. */ int listen_sharecount; + struct rcu_head rcu; struct ib_mad_send_buf *msg; struct cm_timewait_info *timewait_info; @@ -593,28 +594,16 @@ static void cm_free_id(__be32 local_id) xa_erase_irq(&cm.local_id_table, cm_local_id(local_id)); } -static struct cm_id_private * cm_get_id(__be32 local_id, __be32 remote_id) +static struct cm_id_private *cm_acquire_id(__be32 local_id, __be32 remote_id) { struct cm_id_private *cm_id_priv; + rcu_read_lock(); cm_id_priv = xa_load(&cm.local_id_table, cm_local_id(local_id)); - if (cm_id_priv) { - if (cm_id_priv->id.remote_id == remote_id) - refcount_inc(&cm_id_priv->refcount); - else - cm_id_priv = NULL; - } - - return cm_id_priv; -} - -static struct cm_id_private * cm_acquire_id(__be32 local_id, __be32 remote_id) -{ - struct cm_id_private *cm_id_priv; - - spin_lock_irq(&cm.lock); - cm_id_priv = cm_get_id(local_id, remote_id); - spin_unlock_irq(&cm.lock); + if (!cm_id_priv || cm_id_priv->id.remote_id != remote_id || + !refcount_inc_not_zero(&cm_id_priv->refcount)) + cm_id_priv = NULL; + rcu_read_unlock(); return cm_id_priv; } @@ -1089,7 +1078,7 @@ retest: rdma_destroy_ah_attr(&cm_id_priv->av.ah_attr); rdma_destroy_ah_attr(&cm_id_priv->alt_av.ah_attr); kfree(cm_id_priv->private_data); - kfree(cm_id_priv); + kfree_rcu(cm_id_priv, rcu); } void ib_destroy_cm_id(struct ib_cm_id *cm_id) @@ -1821,7 +1810,7 @@ static struct cm_id_private * cm_match_req(struct cm_work *work, spin_lock_irq(&cm.lock); timewait_info = cm_insert_remote_id(cm_id_priv->timewait_info); if (timewait_info) { - cur_cm_id_priv = cm_get_id(timewait_info->work.local_id, + cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id, timewait_info->work.remote_id); spin_unlock_irq(&cm.lock); if (cur_cm_id_priv) { @@ -1835,7 +1824,7 @@ static struct cm_id_private * cm_match_req(struct cm_work *work, timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info); if (timewait_info) { cm_cleanup_timewait(cm_id_priv->timewait_info); - cur_cm_id_priv = cm_get_id(timewait_info->work.local_id, + cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id, timewait_info->work.remote_id); spin_unlock_irq(&cm.lock); @@ -2293,7 +2282,7 @@ static int cm_rep_handler(struct cm_work *work) rb_erase(&cm_id_priv->timewait_info->remote_id_node, &cm.remote_id_table); cm_id_priv->timewait_info->inserted_remote_id = 0; - cur_cm_id_priv = cm_get_id(timewait_info->work.local_id, + cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id, timewait_info->work.remote_id); spin_unlock(&cm.lock); @@ -2788,14 +2777,8 @@ static struct cm_id_private * cm_acquire_rejected_id(struct cm_rej_msg *rej_msg) spin_unlock_irq(&cm.lock); return NULL; } - cm_id_priv = xa_load(&cm.local_id_table, - cm_local_id(timewait_info->work.local_id)); - if (cm_id_priv) { - if (cm_id_priv->id.remote_id == remote_id) - refcount_inc(&cm_id_priv->refcount); - else - cm_id_priv = NULL; - } + cm_id_priv = + cm_acquire_id(timewait_info->work.local_id, remote_id); spin_unlock_irq(&cm.lock); } else if (cm_rej_get_msg_rejected(rej_msg) == CM_MSG_RESPONSE_REQ) cm_id_priv = cm_acquire_id(rej_msg->remote_comm_id, 0); -- cgit v1.2.3-59-g8ed1b From d07de8bd1709a80a282963ad7b2535148678a9e4 Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Sun, 22 Dec 2019 14:46:48 +0200 Subject: IB/core: Fix ODP get user pages flow The nr_pages argument of get_user_pages_remote() should always be in terms of the system page size, not the MR page size. Use PAGE_SIZE instead of umem_odp->page_shift. Fixes: 403cd12e2cf7 ("IB/umem: Add contiguous ODP support") Link: https://lore.kernel.org/r/20191222124649.52300-3-leon@kernel.org Signed-off-by: Yishai Hadas Reviewed-by: Artemy Kovalyov Reviewed-by: Jason Gunthorpe Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/umem_odp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c index e42d44e501fd..2e9ee7adab13 100644 --- a/drivers/infiniband/core/umem_odp.c +++ b/drivers/infiniband/core/umem_odp.c @@ -440,7 +440,7 @@ int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt, while (bcnt > 0) { const size_t gup_num_pages = min_t(size_t, - (bcnt + BIT(page_shift) - 1) >> page_shift, + ALIGN(bcnt, PAGE_SIZE) / PAGE_SIZE, PAGE_SIZE / sizeof(struct page *)); down_read(&owning_mm->mmap_sem); -- cgit v1.2.3-59-g8ed1b From 9ff1b6466a291a33389c4a9c7f3f9b64d62df40a Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Sun, 22 Dec 2019 14:46:49 +0200 Subject: IB/core: Fix ODP with IB_ACCESS_HUGETLB handling As VMAs for a given range might not be available as part of the registration phase in ODP. ib_init_umem_odp() considered the expected page shift value that was previously set and initializes its internals accordingly. If memory isn't backed by physical contiguous pages aligned to a hugepage boundary an error will be set as part of the page fault flow and come back to the user as some failed RDMA operation. Fixes: 0008b84ea9af ("IB/umem: Add support to huge ODP") Link: https://lore.kernel.org/r/20191222124649.52300-4-leon@kernel.org Signed-off-by: Yishai Hadas Reviewed-by: Artemy Kovalyov Signed-off-by: Leon Romanovsky Reviewed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/umem_odp.c | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c index 2e9ee7adab13..f42fa31c24a2 100644 --- a/drivers/infiniband/core/umem_odp.c +++ b/drivers/infiniband/core/umem_odp.c @@ -241,22 +241,10 @@ struct ib_umem_odp *ib_umem_odp_get(struct ib_udata *udata, unsigned long addr, umem_odp->umem.owning_mm = mm = current->mm; umem_odp->notifier.ops = ops; - umem_odp->page_shift = PAGE_SHIFT; - if (access & IB_ACCESS_HUGETLB) { - struct vm_area_struct *vma; - struct hstate *h; - - down_read(&mm->mmap_sem); - vma = find_vma(mm, ib_umem_start(umem_odp)); - if (!vma || !is_vm_hugetlb_page(vma)) { - up_read(&mm->mmap_sem); - ret = -EINVAL; - goto err_free; - } - h = hstate_vma(vma); - umem_odp->page_shift = huge_page_shift(h); - up_read(&mm->mmap_sem); - } + if (access & IB_ACCESS_HUGETLB) + umem_odp->page_shift = HPAGE_SHIFT; + else + umem_odp->page_shift = PAGE_SHIFT; umem_odp->tgid = get_task_pid(current->group_leader, PIDTYPE_PID); ret = ib_init_umem_odp(umem_odp, ops); @@ -266,7 +254,6 @@ struct ib_umem_odp *ib_umem_odp_get(struct ib_udata *udata, unsigned long addr, err_put_pid: put_pid(umem_odp->tgid); -err_free: kfree(umem_odp); return ERR_PTR(ret); } -- cgit v1.2.3-59-g8ed1b From 017d8eada807c6b80905445c0defa5d3b1d108bf Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Thu, 12 Dec 2019 11:38:24 +0200 Subject: RDMA/cm: Delete unused CM LAP functions Clean the code by deleting LAP functions, which are not called anyway. Link: https://lore.kernel.org/r/20191212093830.316934-43-leon@kernel.org Signed-off-by: Leon Romanovsky Reviewed-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/cm.c | 98 --------------------------------------- drivers/infiniband/core/cm_msgs.h | 70 ---------------------------- 2 files changed, 168 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c index 2945031dd8a7..fae788e38c87 100644 --- a/drivers/infiniband/core/cm.c +++ b/drivers/infiniband/core/cm.c @@ -3029,104 +3029,6 @@ out: return -EINVAL; } -static void cm_format_lap(struct cm_lap_msg *lap_msg, - struct cm_id_private *cm_id_priv, - struct sa_path_rec *alternate_path, - const void *private_data, - u8 private_data_len) -{ - bool alt_ext = false; - - if (alternate_path->rec_type == SA_PATH_REC_TYPE_OPA) - alt_ext = opa_is_extended_lid(alternate_path->opa.dlid, - alternate_path->opa.slid); - cm_format_mad_hdr(&lap_msg->hdr, CM_LAP_ATTR_ID, - cm_form_tid(cm_id_priv)); - lap_msg->local_comm_id = cm_id_priv->id.local_id; - lap_msg->remote_comm_id = cm_id_priv->id.remote_id; - cm_lap_set_remote_qpn(lap_msg, cm_id_priv->remote_qpn); - /* todo: need remote CM response timeout */ - cm_lap_set_remote_resp_timeout(lap_msg, 0x1F); - lap_msg->alt_local_lid = - htons(ntohl(sa_path_get_slid(alternate_path))); - lap_msg->alt_remote_lid = - htons(ntohl(sa_path_get_dlid(alternate_path))); - lap_msg->alt_local_gid = alternate_path->sgid; - lap_msg->alt_remote_gid = alternate_path->dgid; - if (alt_ext) { - lap_msg->alt_local_gid.global.interface_id - = OPA_MAKE_ID(be32_to_cpu(alternate_path->opa.slid)); - lap_msg->alt_remote_gid.global.interface_id - = OPA_MAKE_ID(be32_to_cpu(alternate_path->opa.dlid)); - } - cm_lap_set_flow_label(lap_msg, alternate_path->flow_label); - cm_lap_set_traffic_class(lap_msg, alternate_path->traffic_class); - lap_msg->alt_hop_limit = alternate_path->hop_limit; - cm_lap_set_packet_rate(lap_msg, alternate_path->rate); - cm_lap_set_sl(lap_msg, alternate_path->sl); - cm_lap_set_subnet_local(lap_msg, 1); /* local only... */ - cm_lap_set_local_ack_timeout(lap_msg, - cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay, - alternate_path->packet_life_time)); - - if (private_data && private_data_len) - memcpy(lap_msg->private_data, private_data, private_data_len); -} - -int ib_send_cm_lap(struct ib_cm_id *cm_id, - struct sa_path_rec *alternate_path, - const void *private_data, - u8 private_data_len) -{ - struct cm_id_private *cm_id_priv; - struct ib_mad_send_buf *msg; - unsigned long flags; - int ret; - - if (private_data && private_data_len > IB_CM_LAP_PRIVATE_DATA_SIZE) - return -EINVAL; - - cm_id_priv = container_of(cm_id, struct cm_id_private, id); - spin_lock_irqsave(&cm_id_priv->lock, flags); - if (cm_id->state != IB_CM_ESTABLISHED || - (cm_id->lap_state != IB_CM_LAP_UNINIT && - cm_id->lap_state != IB_CM_LAP_IDLE)) { - ret = -EINVAL; - goto out; - } - - ret = cm_init_av_by_path(alternate_path, NULL, &cm_id_priv->alt_av, - cm_id_priv); - if (ret) - goto out; - cm_id_priv->alt_av.timeout = - cm_ack_timeout(cm_id_priv->target_ack_delay, - cm_id_priv->alt_av.timeout - 1); - - ret = cm_alloc_msg(cm_id_priv, &msg); - if (ret) - goto out; - - cm_format_lap((struct cm_lap_msg *) msg->mad, cm_id_priv, - alternate_path, private_data, private_data_len); - msg->timeout_ms = cm_id_priv->timeout_ms; - msg->context[1] = (void *) (unsigned long) IB_CM_ESTABLISHED; - - ret = ib_post_send_mad(msg, NULL); - if (ret) { - spin_unlock_irqrestore(&cm_id_priv->lock, flags); - cm_free_msg(msg); - return ret; - } - - cm_id->lap_state = IB_CM_LAP_SENT; - cm_id_priv->msg = msg; - -out: spin_unlock_irqrestore(&cm_id_priv->lock, flags); - return ret; -} -EXPORT_SYMBOL(ib_send_cm_lap); - static void cm_format_path_lid_from_lap(struct cm_lap_msg *lap_msg, struct sa_path_rec *path) { diff --git a/drivers/infiniband/core/cm_msgs.h b/drivers/infiniband/core/cm_msgs.h index 92d7260ac913..23e3d0c6a67c 100644 --- a/drivers/infiniband/core/cm_msgs.h +++ b/drivers/infiniband/core/cm_msgs.h @@ -651,101 +651,31 @@ struct cm_lap_msg { u8 private_data[IB_CM_LAP_PRIVATE_DATA_SIZE]; } __packed; -static inline __be32 cm_lap_get_remote_qpn(struct cm_lap_msg *lap_msg) -{ - return cpu_to_be32(be32_to_cpu(lap_msg->offset12) >> 8); -} - -static inline void cm_lap_set_remote_qpn(struct cm_lap_msg *lap_msg, __be32 qpn) -{ - lap_msg->offset12 = cpu_to_be32((be32_to_cpu(qpn) << 8) | - (be32_to_cpu(lap_msg->offset12) & - 0x000000FF)); -} - -static inline u8 cm_lap_get_remote_resp_timeout(struct cm_lap_msg *lap_msg) -{ - return (u8) ((be32_to_cpu(lap_msg->offset12) & 0xF8) >> 3); -} - -static inline void cm_lap_set_remote_resp_timeout(struct cm_lap_msg *lap_msg, - u8 resp_timeout) -{ - lap_msg->offset12 = cpu_to_be32((resp_timeout << 3) | - (be32_to_cpu(lap_msg->offset12) & - 0xFFFFFF07)); -} - static inline __be32 cm_lap_get_flow_label(struct cm_lap_msg *lap_msg) { return cpu_to_be32(be32_to_cpu(lap_msg->offset56) >> 12); } -static inline void cm_lap_set_flow_label(struct cm_lap_msg *lap_msg, - __be32 flow_label) -{ - lap_msg->offset56 = cpu_to_be32( - (be32_to_cpu(lap_msg->offset56) & 0x00000FFF) | - (be32_to_cpu(flow_label) << 12)); -} - static inline u8 cm_lap_get_traffic_class(struct cm_lap_msg *lap_msg) { return (u8) be32_to_cpu(lap_msg->offset56); } -static inline void cm_lap_set_traffic_class(struct cm_lap_msg *lap_msg, - u8 traffic_class) -{ - lap_msg->offset56 = cpu_to_be32(traffic_class | - (be32_to_cpu(lap_msg->offset56) & - 0xFFFFFF00)); -} - static inline u8 cm_lap_get_packet_rate(struct cm_lap_msg *lap_msg) { return lap_msg->offset61 & 0x3F; } -static inline void cm_lap_set_packet_rate(struct cm_lap_msg *lap_msg, - u8 packet_rate) -{ - lap_msg->offset61 = (packet_rate & 0x3F) | (lap_msg->offset61 & 0xC0); -} - static inline u8 cm_lap_get_sl(struct cm_lap_msg *lap_msg) { return lap_msg->offset62 >> 4; } -static inline void cm_lap_set_sl(struct cm_lap_msg *lap_msg, u8 sl) -{ - lap_msg->offset62 = (sl << 4) | (lap_msg->offset62 & 0x0F); -} - -static inline u8 cm_lap_get_subnet_local(struct cm_lap_msg *lap_msg) -{ - return (lap_msg->offset62 >> 3) & 0x1; -} - -static inline void cm_lap_set_subnet_local(struct cm_lap_msg *lap_msg, - u8 subnet_local) -{ - lap_msg->offset62 = ((subnet_local & 0x1) << 3) | - (lap_msg->offset61 & 0xF7); -} static inline u8 cm_lap_get_local_ack_timeout(struct cm_lap_msg *lap_msg) { return lap_msg->offset63 >> 3; } -static inline void cm_lap_set_local_ack_timeout(struct cm_lap_msg *lap_msg, - u8 local_ack_timeout) -{ - lap_msg->offset63 = (local_ack_timeout << 3) | - (lap_msg->offset63 & 0x07); -} - struct cm_apr_msg { struct ib_mad_hdr hdr; -- cgit v1.2.3-59-g8ed1b From ad9efa05a0ea2fd9960d4e163fad56279cde45b2 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Thu, 12 Dec 2019 11:38:27 +0200 Subject: RDMA/cm: Delete unused CM ARP functions Clean the code by deleting ARP functions, which are not called anyway. Link: https://lore.kernel.org/r/20191212093830.316934-46-leon@kernel.org Signed-off-by: Leon Romanovsky Reviewed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/cm.c | 66 -------------------------------------------- include/rdma/ib_cm.h | 34 ----------------------- 2 files changed, 100 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c index fae788e38c87..d396b987b0f2 100644 --- a/drivers/infiniband/core/cm.c +++ b/drivers/infiniband/core/cm.c @@ -3163,72 +3163,6 @@ deref: cm_deref_id(cm_id_priv); return -EINVAL; } -static void cm_format_apr(struct cm_apr_msg *apr_msg, - struct cm_id_private *cm_id_priv, - enum ib_cm_apr_status status, - void *info, - u8 info_length, - const void *private_data, - u8 private_data_len) -{ - cm_format_mad_hdr(&apr_msg->hdr, CM_APR_ATTR_ID, cm_id_priv->tid); - apr_msg->local_comm_id = cm_id_priv->id.local_id; - apr_msg->remote_comm_id = cm_id_priv->id.remote_id; - apr_msg->ap_status = (u8) status; - - if (info && info_length) { - apr_msg->info_length = info_length; - memcpy(apr_msg->info, info, info_length); - } - - if (private_data && private_data_len) - memcpy(apr_msg->private_data, private_data, private_data_len); -} - -int ib_send_cm_apr(struct ib_cm_id *cm_id, - enum ib_cm_apr_status status, - void *info, - u8 info_length, - const void *private_data, - u8 private_data_len) -{ - struct cm_id_private *cm_id_priv; - struct ib_mad_send_buf *msg; - unsigned long flags; - int ret; - - if ((private_data && private_data_len > IB_CM_APR_PRIVATE_DATA_SIZE) || - (info && info_length > IB_CM_APR_INFO_LENGTH)) - return -EINVAL; - - cm_id_priv = container_of(cm_id, struct cm_id_private, id); - spin_lock_irqsave(&cm_id_priv->lock, flags); - if (cm_id->state != IB_CM_ESTABLISHED || - (cm_id->lap_state != IB_CM_LAP_RCVD && - cm_id->lap_state != IB_CM_MRA_LAP_SENT)) { - ret = -EINVAL; - goto out; - } - - ret = cm_alloc_msg(cm_id_priv, &msg); - if (ret) - goto out; - - cm_format_apr((struct cm_apr_msg *) msg->mad, cm_id_priv, status, - info, info_length, private_data, private_data_len); - ret = ib_post_send_mad(msg, NULL); - if (ret) { - spin_unlock_irqrestore(&cm_id_priv->lock, flags); - cm_free_msg(msg); - return ret; - } - - cm_id->lap_state = IB_CM_LAP_IDLE; -out: spin_unlock_irqrestore(&cm_id_priv->lock, flags); - return ret; -} -EXPORT_SYMBOL(ib_send_cm_apr); - static int cm_apr_handler(struct cm_work *work) { struct cm_id_private *cm_id_priv; diff --git a/include/rdma/ib_cm.h b/include/rdma/ib_cm.h index b01a8a8d4de9..8ec482e391aa 100644 --- a/include/rdma/ib_cm.h +++ b/include/rdma/ib_cm.h @@ -499,21 +499,6 @@ int ib_send_cm_mra(struct ib_cm_id *cm_id, const void *private_data, u8 private_data_len); -/** - * ib_send_cm_lap - Sends a load alternate path request. - * @cm_id: Connection identifier associated with the load alternate path - * message. - * @alternate_path: A path record that identifies the alternate path to - * load. - * @private_data: Optional user-defined private data sent with the - * load alternate path message. - * @private_data_len: Size of the private data buffer, in bytes. - */ -int ib_send_cm_lap(struct ib_cm_id *cm_id, - struct sa_path_rec *alternate_path, - const void *private_data, - u8 private_data_len); - /** * ib_cm_init_qp_attr - Initializes the QP attributes for use in transitioning * to a specified QP state. @@ -534,25 +519,6 @@ int ib_cm_init_qp_attr(struct ib_cm_id *cm_id, struct ib_qp_attr *qp_attr, int *qp_attr_mask); -/** - * ib_send_cm_apr - Sends an alternate path response message in response to - * a load alternate path request. - * @cm_id: Connection identifier associated with the alternate path response. - * @status: Reply status sent with the alternate path response. - * @info: Optional additional information sent with the alternate path - * response. - * @info_length: Size of the additional information, in bytes. - * @private_data: Optional user-defined private data sent with the - * alternate path response message. - * @private_data_len: Size of the private data buffer, in bytes. - */ -int ib_send_cm_apr(struct ib_cm_id *cm_id, - enum ib_cm_apr_status status, - void *info, - u8 info_length, - const void *private_data, - u8 private_data_len); - struct ib_cm_sidr_req_param { struct sa_path_rec *path; const struct ib_gid_attr *sgid_attr; -- cgit v1.2.3-59-g8ed1b From ed999f820a6c579298b53270656a0a26c8a6cb87 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Wed, 18 Dec 2019 15:18:10 -0500 Subject: RDMA/cma: Add trace points in RDMA Connection Manager Record state transitions as each connection is established. The IP address of both peers and the Type of Service is reported. These trace points are not in performance hot paths. Also, record each cm_event_handler call to ULPs. This eliminates the need for each ULP to add its own similar trace point in its CM event handler function. These new trace points appear in a new trace subsystem called "rdma_cma". Sample events: <...>-220 [004] 121.430733: cm_id_create: cm.id=0 <...>-472 [003] 121.430991: cm_event_handler: cm.id=0 src=192.168.2.51:35090 dst=192.168.2.55:20049 tos=0 ADDR_RESOLVED (0/0) <...>-472 [003] 121.430995: cm_event_done: cm.id=0 src=192.168.2.51:35090 dst=192.168.2.55:20049 tos=0 result=0 <...>-472 [003] 121.431172: cm_event_handler: cm.id=0 src=192.168.2.51:35090 dst=192.168.2.55:20049 tos=0 ROUTE_RESOLVED (2/0) <...>-472 [003] 121.431174: cm_event_done: cm.id=0 src=192.168.2.51:35090 dst=192.168.2.55:20049 tos=0 result=0 <...>-220 [004] 121.433480: cm_qp_create: cm.id=0 src=192.168.2.51:35090 dst=192.168.2.55:20049 tos=0 pd.id=2 qp_type=RC send_wr=4091 recv_wr=256 qp_num=521 rc=0 <...>-220 [004] 121.433577: cm_send_req: cm.id=0 src=192.168.2.51:35090 dst=192.168.2.55:20049 tos=0 qp_num=521 kworker/1:2-973 [001] 121.436190: cm_send_mra: cm.id=0 src=192.168.2.51:35090 dst=192.168.2.55:20049 tos=0 kworker/1:2-973 [001] 121.436340: cm_send_rtu: cm.id=0 src=192.168.2.51:35090 dst=192.168.2.55:20049 tos=0 kworker/1:2-973 [001] 121.436359: cm_event_handler: cm.id=0 src=192.168.2.51:35090 dst=192.168.2.55:20049 tos=0 ESTABLISHED (9/0) kworker/1:2-973 [001] 121.436365: cm_event_done: cm.id=0 src=192.168.2.51:35090 dst=192.168.2.55:20049 tos=0 result=0 <...>-1975 [005] 123.161954: cm_disconnect: cm.id=0 src=192.168.2.51:35090 dst=192.168.2.55:20049 tos=0 <...>-1975 [005] 123.161974: cm_sent_dreq: cm.id=0 src=192.168.2.51:35090 dst=192.168.2.55:20049 tos=0 <...>-220 [004] 123.162102: cm_disconnect: cm.id=0 src=192.168.2.51:35090 dst=192.168.2.55:20049 tos=0 kworker/0:1-13 [000] 123.162391: cm_event_handler: cm.id=0 src=192.168.2.51:35090 dst=192.168.2.55:20049 tos=0 DISCONNECTED (10/0) kworker/0:1-13 [000] 123.162393: cm_event_done: cm.id=0 src=192.168.2.51:35090 dst=192.168.2.55:20049 tos=0 result=0 <...>-220 [004] 123.164456: cm_qp_destroy: cm.id=0 src=192.168.2.51:35090 dst=192.168.2.55:20049 tos=0 qp_num=521 <...>-220 [004] 123.165290: cm_id_destroy: cm.id=0 src=192.168.2.51:35090 dst=192.168.2.55:20049 tos=0 Some features to note: - restracker ID of the rdma_cm_id is tagged on each trace event - The source and destination IP addresses and TOS are reported - CM event upcalls are shown with decoded event and status - CM state transitions are reported - rdma_cm_id lifetime events are captured - The latency of ULP CM event handlers is reported - Lifetime events of associated QPs are reported - Device removal and insertion is reported This patch is based on previous work by: Saeed Mahameed Mukesh Kacker Ajaykumar Hotchandani Aron Silverton Avinash Repaka Somasundaram Krishnasamy Link: https://lore.kernel.org/r/20191218201810.30584.3052.stgit@manet.1015granger.net Signed-off-by: Chuck Lever Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/Makefile | 3 +- drivers/infiniband/core/cma.c | 88 ++++++-- drivers/infiniband/core/cma_trace.c | 16 ++ drivers/infiniband/core/cma_trace.h | 391 ++++++++++++++++++++++++++++++++++++ 4 files changed, 475 insertions(+), 23 deletions(-) create mode 100644 drivers/infiniband/core/cma_trace.c create mode 100644 drivers/infiniband/core/cma_trace.h (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile index 9a8871e21545..f22555f982e2 100644 --- a/drivers/infiniband/core/Makefile +++ b/drivers/infiniband/core/Makefile @@ -20,7 +20,8 @@ ib_cm-y := cm.o iw_cm-y := iwcm.o iwpm_util.o iwpm_msg.o -rdma_cm-y := cma.o +CFLAGS_cma_trace.o += -I$(src) +rdma_cm-y := cma.o cma_trace.o rdma_cm-$(CONFIG_INFINIBAND_ADDR_TRANS_CONFIGFS) += cma_configfs.o diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index 43a6f07e0afe..55a9afacfedd 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -36,6 +36,7 @@ #include "core_priv.h" #include "cma_priv.h" +#include "cma_trace.h" MODULE_AUTHOR("Sean Hefty"); MODULE_DESCRIPTION("Generic RDMA CM Agent"); @@ -877,6 +878,7 @@ struct rdma_cm_id *__rdma_create_id(struct net *net, id_priv->id.route.addr.dev_addr.net = get_net(net); id_priv->seq_num &= 0x00ffffff; + trace_cm_id_create(id_priv); return &id_priv->id; } EXPORT_SYMBOL(__rdma_create_id); @@ -928,27 +930,34 @@ int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd, int ret; id_priv = container_of(id, struct rdma_id_private, id); - if (id->device != pd->device) - return -EINVAL; + if (id->device != pd->device) { + ret = -EINVAL; + goto out_err; + } qp_init_attr->port_num = id->port_num; qp = ib_create_qp(pd, qp_init_attr); - if (IS_ERR(qp)) - return PTR_ERR(qp); + if (IS_ERR(qp)) { + ret = PTR_ERR(qp); + goto out_err; + } if (id->qp_type == IB_QPT_UD) ret = cma_init_ud_qp(id_priv, qp); else ret = cma_init_conn_qp(id_priv, qp); if (ret) - goto err; + goto out_destroy; id->qp = qp; id_priv->qp_num = qp->qp_num; id_priv->srq = (qp->srq != NULL); + trace_cm_qp_create(id_priv, pd, qp_init_attr, 0); return 0; -err: +out_destroy: ib_destroy_qp(qp); +out_err: + trace_cm_qp_create(id_priv, pd, qp_init_attr, ret); return ret; } EXPORT_SYMBOL(rdma_create_qp); @@ -958,6 +967,7 @@ void rdma_destroy_qp(struct rdma_cm_id *id) struct rdma_id_private *id_priv; id_priv = container_of(id, struct rdma_id_private, id); + trace_cm_qp_destroy(id_priv); mutex_lock(&id_priv->qp_mutex); ib_destroy_qp(id_priv->id.qp); id_priv->id.qp = NULL; @@ -1811,6 +1821,7 @@ void rdma_destroy_id(struct rdma_cm_id *id) enum rdma_cm_state state; id_priv = container_of(id, struct rdma_id_private, id); + trace_cm_id_destroy(id_priv); state = cma_exch(id_priv, RDMA_CM_DESTROYING); cma_cancel_operation(id_priv, state); @@ -1863,6 +1874,7 @@ static int cma_rep_recv(struct rdma_id_private *id_priv) if (ret) goto reject; + trace_cm_send_rtu(id_priv); ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0); if (ret) goto reject; @@ -1871,6 +1883,7 @@ static int cma_rep_recv(struct rdma_id_private *id_priv) reject: pr_debug_ratelimited("RDMA CM: CONNECT_ERROR: failed to handle reply. status %d\n", ret); cma_modify_qp_err(id_priv); + trace_cm_send_rej(id_priv); ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0, NULL, 0); return ret; @@ -1890,6 +1903,17 @@ static void cma_set_rep_event_data(struct rdma_cm_event *event, event->param.conn.qp_num = rep_data->remote_qpn; } +static int cma_cm_event_handler(struct rdma_id_private *id_priv, + struct rdma_cm_event *event) +{ + int ret; + + trace_cm_event_handler(id_priv, event); + ret = id_priv->id.event_handler(&id_priv->id, event); + trace_cm_event_done(id_priv, event, ret); + return ret; +} + static int cma_ib_handler(struct ib_cm_id *cm_id, const struct ib_cm_event *ib_event) { @@ -1912,8 +1936,10 @@ static int cma_ib_handler(struct ib_cm_id *cm_id, break; case IB_CM_REP_RECEIVED: if (cma_comp(id_priv, RDMA_CM_CONNECT) && - (id_priv->id.qp_type != IB_QPT_UD)) + (id_priv->id.qp_type != IB_QPT_UD)) { + trace_cm_send_mra(id_priv); ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0); + } if (id_priv->id.qp) { event.status = cma_rep_recv(id_priv); event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR : @@ -1958,7 +1984,7 @@ static int cma_ib_handler(struct ib_cm_id *cm_id, goto out; } - ret = id_priv->id.event_handler(&id_priv->id, &event); + ret = cma_cm_event_handler(id_priv, &event); if (ret) { /* Destroy the CM ID by returning a non-zero value. */ id_priv->cm_id.ib = NULL; @@ -2119,6 +2145,7 @@ static int cma_ib_req_handler(struct ib_cm_id *cm_id, if (IS_ERR(listen_id)) return PTR_ERR(listen_id); + trace_cm_req_handler(listen_id, ib_event->event); if (!cma_ib_check_req_qp_type(&listen_id->id, ib_event)) { ret = -EINVAL; goto net_dev_put; @@ -2161,7 +2188,7 @@ static int cma_ib_req_handler(struct ib_cm_id *cm_id, * until we're done accessing it. */ atomic_inc(&conn_id->refcount); - ret = conn_id->id.event_handler(&conn_id->id, &event); + ret = cma_cm_event_handler(conn_id, &event); if (ret) goto err3; /* @@ -2170,8 +2197,10 @@ static int cma_ib_req_handler(struct ib_cm_id *cm_id, */ mutex_lock(&lock); if (cma_comp(conn_id, RDMA_CM_CONNECT) && - (conn_id->id.qp_type != IB_QPT_UD)) + (conn_id->id.qp_type != IB_QPT_UD)) { + trace_cm_send_mra(cm_id->context); ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0); + } mutex_unlock(&lock); mutex_unlock(&conn_id->handler_mutex); mutex_unlock(&listen_id->handler_mutex); @@ -2286,7 +2315,7 @@ static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event) event.status = iw_event->status; event.param.conn.private_data = iw_event->private_data; event.param.conn.private_data_len = iw_event->private_data_len; - ret = id_priv->id.event_handler(&id_priv->id, &event); + ret = cma_cm_event_handler(id_priv, &event); if (ret) { /* Destroy the CM ID by returning a non-zero value. */ id_priv->cm_id.iw = NULL; @@ -2363,7 +2392,7 @@ static int iw_conn_req_handler(struct iw_cm_id *cm_id, * until we're done accessing it. */ atomic_inc(&conn_id->refcount); - ret = conn_id->id.event_handler(&conn_id->id, &event); + ret = cma_cm_event_handler(conn_id, &event); if (ret) { /* User wants to destroy the CM ID */ conn_id->cm_id.iw = NULL; @@ -2435,6 +2464,7 @@ static int cma_listen_handler(struct rdma_cm_id *id, id->context = id_priv->id.context; id->event_handler = id_priv->id.event_handler; + trace_cm_event_handler(id_priv, event); return id_priv->id.event_handler(id, event); } @@ -2611,7 +2641,7 @@ static void cma_work_handler(struct work_struct *_work) if (!cma_comp_exch(id_priv, work->old_state, work->new_state)) goto out; - if (id_priv->id.event_handler(&id_priv->id, &work->event)) { + if (cma_cm_event_handler(id_priv, &work->event)) { cma_exch(id_priv, RDMA_CM_DESTROYING); destroy = 1; } @@ -2634,7 +2664,7 @@ static void cma_ndev_work_handler(struct work_struct *_work) id_priv->state == RDMA_CM_DEVICE_REMOVAL) goto out; - if (id_priv->id.event_handler(&id_priv->id, &work->event)) { + if (cma_cm_event_handler(id_priv, &work->event)) { cma_exch(id_priv, RDMA_CM_DESTROYING); destroy = 1; } @@ -3089,7 +3119,7 @@ static void addr_handler(int status, struct sockaddr *src_addr, } else event.event = RDMA_CM_EVENT_ADDR_RESOLVED; - if (id_priv->id.event_handler(&id_priv->id, &event)) { + if (cma_cm_event_handler(id_priv, &event)) { cma_exch(id_priv, RDMA_CM_DESTROYING); mutex_unlock(&id_priv->handler_mutex); rdma_destroy_id(&id_priv->id); @@ -3736,7 +3766,7 @@ static int cma_sidr_rep_handler(struct ib_cm_id *cm_id, goto out; } - ret = id_priv->id.event_handler(&id_priv->id, &event); + ret = cma_cm_event_handler(id_priv, &event); rdma_destroy_ah_attr(&event.param.ud.ah_attr); if (ret) { @@ -3800,6 +3830,7 @@ static int cma_resolve_ib_udp(struct rdma_id_private *id_priv, req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8); req.max_cm_retries = CMA_MAX_CM_RETRIES; + trace_cm_send_sidr_req(id_priv); ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req); if (ret) { ib_destroy_cm_id(id_priv->cm_id.ib); @@ -3873,6 +3904,7 @@ static int cma_connect_ib(struct rdma_id_private *id_priv, req.max_cm_retries = CMA_MAX_CM_RETRIES; req.srq = id_priv->srq ? 1 : 0; + trace_cm_send_req(id_priv); ret = ib_send_cm_req(id_priv->cm_id.ib, &req); out: if (ret && !IS_ERR(id)) { @@ -3986,6 +4018,7 @@ static int cma_accept_ib(struct rdma_id_private *id_priv, rep.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count); rep.srq = id_priv->srq ? 1 : 0; + trace_cm_send_rep(id_priv); ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep); out: return ret; @@ -4035,6 +4068,7 @@ static int cma_send_sidr_rep(struct rdma_id_private *id_priv, rep.private_data = private_data; rep.private_data_len = private_data_len; + trace_cm_send_sidr_rep(id_priv); return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep); } @@ -4120,13 +4154,15 @@ int rdma_reject(struct rdma_cm_id *id, const void *private_data, return -EINVAL; if (rdma_cap_ib_cm(id->device, id->port_num)) { - if (id->qp_type == IB_QPT_UD) + if (id->qp_type == IB_QPT_UD) { ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT, 0, private_data, private_data_len); - else + } else { + trace_cm_send_rej(id_priv); ret = ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0, private_data, private_data_len); + } } else if (rdma_cap_iw_cm(id->device, id->port_num)) { ret = iw_cm_reject(id_priv->cm_id.iw, private_data, private_data_len); @@ -4151,8 +4187,13 @@ int rdma_disconnect(struct rdma_cm_id *id) if (ret) goto out; /* Initiate or respond to a disconnect. */ - if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) - ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0); + trace_cm_disconnect(id_priv); + if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) { + if (!ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0)) + trace_cm_sent_drep(id_priv); + } else { + trace_cm_sent_dreq(id_priv); + } } else if (rdma_cap_iw_cm(id->device, id->port_num)) { ret = iw_cm_disconnect(id_priv->cm_id.iw, 0); } else @@ -4218,7 +4259,7 @@ static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast) } else event.event = RDMA_CM_EVENT_MULTICAST_ERROR; - ret = id_priv->id.event_handler(&id_priv->id, &event); + ret = cma_cm_event_handler(id_priv, &event); rdma_destroy_ah_attr(&event.param.ud.ah_attr); if (ret) { @@ -4623,6 +4664,7 @@ static void cma_add_one(struct ib_device *device) cma_listen_on_dev(id_priv, cma_dev); mutex_unlock(&lock); + trace_cm_add_one(device); return; free_gid_type: @@ -4653,7 +4695,7 @@ static int cma_remove_id_dev(struct rdma_id_private *id_priv) goto out; event.event = RDMA_CM_EVENT_DEVICE_REMOVAL; - ret = id_priv->id.event_handler(&id_priv->id, &event); + ret = cma_cm_event_handler(id_priv, &event); out: mutex_unlock(&id_priv->handler_mutex); return ret; @@ -4691,6 +4733,8 @@ static void cma_remove_one(struct ib_device *device, void *client_data) { struct cma_device *cma_dev = client_data; + trace_cm_remove_one(device); + if (!cma_dev) return; diff --git a/drivers/infiniband/core/cma_trace.c b/drivers/infiniband/core/cma_trace.c new file mode 100644 index 000000000000..b314a281e10e --- /dev/null +++ b/drivers/infiniband/core/cma_trace.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Trace points for the RDMA Connection Manager. + * + * Author: Chuck Lever + * + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + */ + +#define CREATE_TRACE_POINTS + +#include +#include +#include "cma_priv.h" + +#include "cma_trace.h" diff --git a/drivers/infiniband/core/cma_trace.h b/drivers/infiniband/core/cma_trace.h new file mode 100644 index 000000000000..81e36bf13159 --- /dev/null +++ b/drivers/infiniband/core/cma_trace.h @@ -0,0 +1,391 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Trace point definitions for the RDMA Connect Manager. + * + * Author: Chuck Lever + * + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM rdma_cma + +#if !defined(_TRACE_RDMA_CMA_H) || defined(TRACE_HEADER_MULTI_READ) + +#define _TRACE_RDMA_CMA_H + +#include +#include + +/* + * enum ib_cm_event_type, from include/rdma/ib_cm.h + */ +#define IB_CM_EVENT_LIST \ + ib_cm_event(REQ_ERROR) \ + ib_cm_event(REQ_RECEIVED) \ + ib_cm_event(REP_ERROR) \ + ib_cm_event(REP_RECEIVED) \ + ib_cm_event(RTU_RECEIVED) \ + ib_cm_event(USER_ESTABLISHED) \ + ib_cm_event(DREQ_ERROR) \ + ib_cm_event(DREQ_RECEIVED) \ + ib_cm_event(DREP_RECEIVED) \ + ib_cm_event(TIMEWAIT_EXIT) \ + ib_cm_event(MRA_RECEIVED) \ + ib_cm_event(REJ_RECEIVED) \ + ib_cm_event(LAP_ERROR) \ + ib_cm_event(LAP_RECEIVED) \ + ib_cm_event(APR_RECEIVED) \ + ib_cm_event(SIDR_REQ_ERROR) \ + ib_cm_event(SIDR_REQ_RECEIVED) \ + ib_cm_event_end(SIDR_REP_RECEIVED) + +#undef ib_cm_event +#undef ib_cm_event_end + +#define ib_cm_event(x) TRACE_DEFINE_ENUM(IB_CM_##x); +#define ib_cm_event_end(x) TRACE_DEFINE_ENUM(IB_CM_##x); + +IB_CM_EVENT_LIST + +#undef ib_cm_event +#undef ib_cm_event_end + +#define ib_cm_event(x) { IB_CM_##x, #x }, +#define ib_cm_event_end(x) { IB_CM_##x, #x } + +#define rdma_show_ib_cm_event(x) \ + __print_symbolic(x, IB_CM_EVENT_LIST) + + +DECLARE_EVENT_CLASS(cma_fsm_class, + TP_PROTO( + const struct rdma_id_private *id_priv + ), + + TP_ARGS(id_priv), + + TP_STRUCT__entry( + __field(u32, cm_id) + __field(u32, tos) + __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6)) + __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6)) + ), + + TP_fast_assign( + __entry->cm_id = id_priv->res.id; + __entry->tos = id_priv->tos; + memcpy(__entry->srcaddr, &id_priv->id.route.addr.src_addr, + sizeof(struct sockaddr_in6)); + memcpy(__entry->dstaddr, &id_priv->id.route.addr.dst_addr, + sizeof(struct sockaddr_in6)); + ), + + TP_printk("cm.id=%u src=%pISpc dst=%pISpc tos=%u", + __entry->cm_id, __entry->srcaddr, __entry->dstaddr, __entry->tos + ) +); + +#define DEFINE_CMA_FSM_EVENT(name) \ + DEFINE_EVENT(cma_fsm_class, cm_##name, \ + TP_PROTO( \ + const struct rdma_id_private *id_priv \ + ), \ + TP_ARGS(id_priv)) + +DEFINE_CMA_FSM_EVENT(send_rtu); +DEFINE_CMA_FSM_EVENT(send_rej); +DEFINE_CMA_FSM_EVENT(send_mra); +DEFINE_CMA_FSM_EVENT(send_sidr_req); +DEFINE_CMA_FSM_EVENT(send_sidr_rep); +DEFINE_CMA_FSM_EVENT(disconnect); +DEFINE_CMA_FSM_EVENT(sent_drep); +DEFINE_CMA_FSM_EVENT(sent_dreq); +DEFINE_CMA_FSM_EVENT(id_destroy); + +TRACE_EVENT(cm_id_create, + TP_PROTO( + const struct rdma_id_private *id_priv + ), + + TP_ARGS(id_priv), + + TP_STRUCT__entry( + __field(u32, cm_id) + ), + + TP_fast_assign( + __entry->cm_id = id_priv->res.id; + ), + + TP_printk("cm.id=%u", + __entry->cm_id + ) +); + +DECLARE_EVENT_CLASS(cma_qp_class, + TP_PROTO( + const struct rdma_id_private *id_priv + ), + + TP_ARGS(id_priv), + + TP_STRUCT__entry( + __field(u32, cm_id) + __field(u32, tos) + __field(u32, qp_num) + __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6)) + __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6)) + ), + + TP_fast_assign( + __entry->cm_id = id_priv->res.id; + __entry->tos = id_priv->tos; + __entry->qp_num = id_priv->qp_num; + memcpy(__entry->srcaddr, &id_priv->id.route.addr.src_addr, + sizeof(struct sockaddr_in6)); + memcpy(__entry->dstaddr, &id_priv->id.route.addr.dst_addr, + sizeof(struct sockaddr_in6)); + ), + + TP_printk("cm.id=%u src=%pISpc dst=%pISpc tos=%u qp_num=%u", + __entry->cm_id, __entry->srcaddr, __entry->dstaddr, __entry->tos, + __entry->qp_num + ) +); + +#define DEFINE_CMA_QP_EVENT(name) \ + DEFINE_EVENT(cma_qp_class, cm_##name, \ + TP_PROTO( \ + const struct rdma_id_private *id_priv \ + ), \ + TP_ARGS(id_priv)) + +DEFINE_CMA_QP_EVENT(send_req); +DEFINE_CMA_QP_EVENT(send_rep); +DEFINE_CMA_QP_EVENT(qp_destroy); + +/* + * enum ib_wp_type, from include/rdma/ib_verbs.h + */ +#define IB_QP_TYPE_LIST \ + ib_qp_type(SMI) \ + ib_qp_type(GSI) \ + ib_qp_type(RC) \ + ib_qp_type(UC) \ + ib_qp_type(UD) \ + ib_qp_type(RAW_IPV6) \ + ib_qp_type(RAW_ETHERTYPE) \ + ib_qp_type(RAW_PACKET) \ + ib_qp_type(XRC_INI) \ + ib_qp_type_end(XRC_TGT) + +#undef ib_qp_type +#undef ib_qp_type_end + +#define ib_qp_type(x) TRACE_DEFINE_ENUM(IB_QPT_##x); +#define ib_qp_type_end(x) TRACE_DEFINE_ENUM(IB_QPT_##x); + +IB_QP_TYPE_LIST + +#undef ib_qp_type +#undef ib_qp_type_end + +#define ib_qp_type(x) { IB_QPT_##x, #x }, +#define ib_qp_type_end(x) { IB_QPT_##x, #x } + +#define rdma_show_qp_type(x) \ + __print_symbolic(x, IB_QP_TYPE_LIST) + + +TRACE_EVENT(cm_qp_create, + TP_PROTO( + const struct rdma_id_private *id_priv, + const struct ib_pd *pd, + const struct ib_qp_init_attr *qp_init_attr, + int rc + ), + + TP_ARGS(id_priv, pd, qp_init_attr, rc), + + TP_STRUCT__entry( + __field(u32, cm_id) + __field(u32, pd_id) + __field(u32, tos) + __field(u32, qp_num) + __field(u32, send_wr) + __field(u32, recv_wr) + __field(int, rc) + __field(unsigned long, qp_type) + __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6)) + __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6)) + ), + + TP_fast_assign( + __entry->cm_id = id_priv->res.id; + __entry->pd_id = pd->res.id; + __entry->tos = id_priv->tos; + __entry->send_wr = qp_init_attr->cap.max_send_wr; + __entry->recv_wr = qp_init_attr->cap.max_recv_wr; + __entry->rc = rc; + if (!rc) { + __entry->qp_num = id_priv->qp_num; + __entry->qp_type = id_priv->id.qp_type; + } else { + __entry->qp_num = 0; + __entry->qp_type = 0; + } + memcpy(__entry->srcaddr, &id_priv->id.route.addr.src_addr, + sizeof(struct sockaddr_in6)); + memcpy(__entry->dstaddr, &id_priv->id.route.addr.dst_addr, + sizeof(struct sockaddr_in6)); + ), + + TP_printk("cm.id=%u src=%pISpc dst=%pISpc tos=%u pd.id=%u qp_type=%s" + " send_wr=%u recv_wr=%u qp_num=%u rc=%d", + __entry->cm_id, __entry->srcaddr, __entry->dstaddr, + __entry->tos, __entry->pd_id, + rdma_show_qp_type(__entry->qp_type), __entry->send_wr, + __entry->recv_wr, __entry->qp_num, __entry->rc + ) +); + +TRACE_EVENT(cm_req_handler, + TP_PROTO( + const struct rdma_id_private *id_priv, + int event + ), + + TP_ARGS(id_priv, event), + + TP_STRUCT__entry( + __field(u32, cm_id) + __field(u32, tos) + __field(unsigned long, event) + __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6)) + __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6)) + ), + + TP_fast_assign( + __entry->cm_id = id_priv->res.id; + __entry->tos = id_priv->tos; + __entry->event = event; + memcpy(__entry->srcaddr, &id_priv->id.route.addr.src_addr, + sizeof(struct sockaddr_in6)); + memcpy(__entry->dstaddr, &id_priv->id.route.addr.dst_addr, + sizeof(struct sockaddr_in6)); + ), + + TP_printk("cm.id=%u src=%pISpc dst=%pISpc tos=%u %s (%lu)", + __entry->cm_id, __entry->srcaddr, __entry->dstaddr, __entry->tos, + rdma_show_ib_cm_event(__entry->event), __entry->event + ) +); + +TRACE_EVENT(cm_event_handler, + TP_PROTO( + const struct rdma_id_private *id_priv, + const struct rdma_cm_event *event + ), + + TP_ARGS(id_priv, event), + + TP_STRUCT__entry( + __field(u32, cm_id) + __field(u32, tos) + __field(unsigned long, event) + __field(int, status) + __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6)) + __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6)) + ), + + TP_fast_assign( + __entry->cm_id = id_priv->res.id; + __entry->tos = id_priv->tos; + __entry->event = event->event; + __entry->status = event->status; + memcpy(__entry->srcaddr, &id_priv->id.route.addr.src_addr, + sizeof(struct sockaddr_in6)); + memcpy(__entry->dstaddr, &id_priv->id.route.addr.dst_addr, + sizeof(struct sockaddr_in6)); + ), + + TP_printk("cm.id=%u src=%pISpc dst=%pISpc tos=%u %s (%lu/%d)", + __entry->cm_id, __entry->srcaddr, __entry->dstaddr, __entry->tos, + rdma_show_cm_event(__entry->event), __entry->event, + __entry->status + ) +); + +TRACE_EVENT(cm_event_done, + TP_PROTO( + const struct rdma_id_private *id_priv, + const struct rdma_cm_event *event, + int result + ), + + TP_ARGS(id_priv, event, result), + + TP_STRUCT__entry( + __field(u32, cm_id) + __field(u32, tos) + __field(unsigned long, event) + __field(int, result) + __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6)) + __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6)) + ), + + TP_fast_assign( + __entry->cm_id = id_priv->res.id; + __entry->tos = id_priv->tos; + __entry->event = event->event; + __entry->result = result; + memcpy(__entry->srcaddr, &id_priv->id.route.addr.src_addr, + sizeof(struct sockaddr_in6)); + memcpy(__entry->dstaddr, &id_priv->id.route.addr.dst_addr, + sizeof(struct sockaddr_in6)); + ), + + TP_printk("cm.id=%u src=%pISpc dst=%pISpc tos=%u %s consumer returns %d", + __entry->cm_id, __entry->srcaddr, __entry->dstaddr, __entry->tos, + rdma_show_cm_event(__entry->event), __entry->result + ) +); + +DECLARE_EVENT_CLASS(cma_client_class, + TP_PROTO( + const struct ib_device *device + ), + + TP_ARGS(device), + + TP_STRUCT__entry( + __string(name, device->name) + ), + + TP_fast_assign( + __assign_str(name, device->name); + ), + + TP_printk("device name=%s", + __get_str(name) + ) +); + +#define DEFINE_CMA_CLIENT_EVENT(name) \ + DEFINE_EVENT(cma_client_class, cm_##name, \ + TP_PROTO( \ + const struct ib_device *device \ + ), \ + TP_ARGS(device)) + +DEFINE_CMA_CLIENT_EVENT(add_one); +DEFINE_CMA_CLIENT_EVENT(remove_one); + +#endif /* _TRACE_RDMA_CMA_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE cma_trace + +#include -- cgit v1.2.3-59-g8ed1b From 3e5901cbfcc15da54f6ad148add754e7a2b2a558 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Wed, 18 Dec 2019 15:18:15 -0500 Subject: RDMA/core: Trace points for diagnosing completion queue issues Sample trace events: kworker/u29:0-300 [007] 120.042217: cq_alloc: cq.id=4 nr_cqe=161 comp_vector=2 poll_ctx=WORKQUEUE -0 [002] 120.056292: cq_schedule: cq.id=4 kworker/2:1H-482 [002] 120.056402: cq_process: cq.id=4 wake-up took 109 [us] from interrupt kworker/2:1H-482 [002] 120.056407: cq_poll: cq.id=4 requested 16, returned 1 -0 [002] 120.067503: cq_schedule: cq.id=4 kworker/2:1H-482 [002] 120.067537: cq_process: cq.id=4 wake-up took 34 [us] from interrupt kworker/2:1H-482 [002] 120.067541: cq_poll: cq.id=4 requested 16, returned 1 -0 [002] 120.067657: cq_schedule: cq.id=4 kworker/2:1H-482 [002] 120.067672: cq_process: cq.id=4 wake-up took 15 [us] from interrupt kworker/2:1H-482 [002] 120.067674: cq_poll: cq.id=4 requested 16, returned 1 ... systemd-1 [002] 122.392653: cq_schedule: cq.id=4 kworker/2:1H-482 [002] 122.392688: cq_process: cq.id=4 wake-up took 35 [us] from interrupt kworker/2:1H-482 [002] 122.392693: cq_poll: cq.id=4 requested 16, returned 16 kworker/2:1H-482 [002] 122.392836: cq_poll: cq.id=4 requested 16, returned 16 kworker/2:1H-482 [002] 122.392970: cq_poll: cq.id=4 requested 16, returned 16 kworker/2:1H-482 [002] 122.393083: cq_poll: cq.id=4 requested 16, returned 16 kworker/2:1H-482 [002] 122.393195: cq_poll: cq.id=4 requested 16, returned 3 Several features to note in this output: - The WCE count and context type are reported at allocation time - The CPU and kworker for each CQ is evident - The CQ's restracker ID is tagged on each trace event - CQ poll scheduling latency is measured - Details about how often single completions occur versus multiple completions are evident - The cost of the ULP's completion handler is recorded Link: https://lore.kernel.org/r/20191218201815.30584.3481.stgit@manet.1015granger.net Signed-off-by: Chuck Lever Reviewed-by: Parav Pandit Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/Makefile | 3 +- drivers/infiniband/core/cq.c | 27 +++- drivers/infiniband/core/trace.c | 14 ++ drivers/infiniband/core/verbs.c | 4 + include/rdma/ib_verbs.h | 5 + include/trace/events/rdma_core.h | 271 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 320 insertions(+), 4 deletions(-) create mode 100644 drivers/infiniband/core/trace.c create mode 100644 include/trace/events/rdma_core.h (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile index f22555f982e2..2b86a514c9bf 100644 --- a/drivers/infiniband/core/Makefile +++ b/drivers/infiniband/core/Makefile @@ -11,7 +11,8 @@ ib_core-y := packer.o ud_header.o verbs.o cq.o rw.o sysfs.o \ device.o fmr_pool.o cache.o netlink.o \ roce_gid_mgmt.o mr_pool.o addr.o sa_query.o \ multicast.o mad.o smi.o agent.o mad_rmpp.o \ - nldev.o restrack.o counters.o ib_core_uverbs.o + nldev.o restrack.o counters.o ib_core_uverbs.o \ + trace.o ib_core-$(CONFIG_SECURITY_INFINIBAND) += security.o ib_core-$(CONFIG_CGROUP_RDMA) += cgroup.o diff --git a/drivers/infiniband/core/cq.c b/drivers/infiniband/core/cq.c index bbfded6d5d3d..4f25b2400694 100644 --- a/drivers/infiniband/core/cq.c +++ b/drivers/infiniband/core/cq.c @@ -7,6 +7,8 @@ #include #include +#include + /* # of WCs to poll for with a single call to ib_poll_cq */ #define IB_POLL_BATCH 16 #define IB_POLL_BATCH_DIRECT 8 @@ -41,6 +43,7 @@ static void ib_cq_rdma_dim_work(struct work_struct *w) dim->state = DIM_START_MEASURE; + trace_cq_modify(cq, comps, usec); cq->device->ops.modify_cq(cq, comps, usec); } @@ -65,18 +68,29 @@ static void rdma_dim_init(struct ib_cq *cq) INIT_WORK(&dim->work, ib_cq_rdma_dim_work); } +static int __poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc) +{ + int rc; + + rc = ib_poll_cq(cq, num_entries, wc); + trace_cq_poll(cq, num_entries, rc); + return rc; +} + static int __ib_process_cq(struct ib_cq *cq, int budget, struct ib_wc *wcs, int batch) { int i, n, completed = 0; + trace_cq_process(cq); + /* * budget might be (-1) if the caller does not * want to bound this call, thus we need unsigned * minimum here. */ - while ((n = ib_poll_cq(cq, min_t(u32, batch, - budget - completed), wcs)) > 0) { + while ((n = __poll_cq(cq, min_t(u32, batch, + budget - completed), wcs)) > 0) { for (i = 0; i < n; i++) { struct ib_wc *wc = &wcs[i]; @@ -131,8 +145,10 @@ static int ib_poll_handler(struct irq_poll *iop, int budget) completed = __ib_process_cq(cq, budget, cq->wc, IB_POLL_BATCH); if (completed < budget) { irq_poll_complete(&cq->iop); - if (ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0) + if (ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0) { + trace_cq_reschedule(cq); irq_poll_sched(&cq->iop); + } } if (dim) @@ -143,6 +159,7 @@ static int ib_poll_handler(struct irq_poll *iop, int budget) static void ib_cq_completion_softirq(struct ib_cq *cq, void *private) { + trace_cq_schedule(cq); irq_poll_sched(&cq->iop); } @@ -162,6 +179,7 @@ static void ib_cq_poll_work(struct work_struct *work) static void ib_cq_completion_workqueue(struct ib_cq *cq, void *private) { + trace_cq_schedule(cq); queue_work(cq->comp_wq, &cq->work); } @@ -239,6 +257,7 @@ struct ib_cq *__ib_alloc_cq_user(struct ib_device *dev, void *private, goto out_destroy_cq; } + trace_cq_alloc(cq, nr_cqe, comp_vector, poll_ctx); return cq; out_destroy_cq: @@ -248,6 +267,7 @@ out_free_wc: kfree(cq->wc); out_free_cq: kfree(cq); + trace_cq_alloc_error(nr_cqe, comp_vector, poll_ctx, ret); return ERR_PTR(ret); } EXPORT_SYMBOL(__ib_alloc_cq_user); @@ -304,6 +324,7 @@ void ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata) WARN_ON_ONCE(1); } + trace_cq_free(cq); rdma_restrack_del(&cq->res); cq->device->ops.destroy_cq(cq, udata); if (cq->dim) diff --git a/drivers/infiniband/core/trace.c b/drivers/infiniband/core/trace.c new file mode 100644 index 000000000000..6c3514beac4d --- /dev/null +++ b/drivers/infiniband/core/trace.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Trace points for core RDMA functions. + * + * Author: Chuck Lever + * + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + */ + +#define CREATE_TRACE_POINTS + +#include + +#include diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index dd765e176cdd..289b2f7a9d5e 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -53,6 +53,8 @@ #include "core_priv.h" +#include + static int ib_resolve_eth_dmac(struct ib_device *device, struct rdma_ah_attr *ah_attr); @@ -2744,6 +2746,7 @@ void ib_drain_sq(struct ib_qp *qp) qp->device->ops.drain_sq(qp); else __ib_drain_sq(qp); + trace_cq_drain_complete(qp->send_cq); } EXPORT_SYMBOL(ib_drain_sq); @@ -2772,6 +2775,7 @@ void ib_drain_rq(struct ib_qp *qp) qp->device->ops.drain_rq(qp); else __ib_drain_rq(qp); + trace_cq_drain_complete(qp->recv_cq); } EXPORT_SYMBOL(ib_drain_rq); diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 5608e14e3aad..42f28d39f28c 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -1558,6 +1558,11 @@ struct ib_cq { }; struct workqueue_struct *comp_wq; struct dim *dim; + + /* updated only by trace points */ + ktime_t timestamp; + bool interrupt; + /* * Implementation details of the RDMA core, don't use in drivers: */ diff --git a/include/trace/events/rdma_core.h b/include/trace/events/rdma_core.h new file mode 100644 index 000000000000..08f481554e7f --- /dev/null +++ b/include/trace/events/rdma_core.h @@ -0,0 +1,271 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Trace point definitions for core RDMA functions. + * + * Author: Chuck Lever + * + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM rdma_core + +#if !defined(_TRACE_RDMA_CORE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_RDMA_CORE_H + +#include +#include + +/* + * enum ib_poll_context, from include/rdma/ib_verbs.h + */ +#define IB_POLL_CTX_LIST \ + ib_poll_ctx(DIRECT) \ + ib_poll_ctx(SOFTIRQ) \ + ib_poll_ctx(WORKQUEUE) \ + ib_poll_ctx_end(UNBOUND_WORKQUEUE) + +#undef ib_poll_ctx +#undef ib_poll_ctx_end + +#define ib_poll_ctx(x) TRACE_DEFINE_ENUM(IB_POLL_##x); +#define ib_poll_ctx_end(x) TRACE_DEFINE_ENUM(IB_POLL_##x); + +IB_POLL_CTX_LIST + +#undef ib_poll_ctx +#undef ib_poll_ctx_end + +#define ib_poll_ctx(x) { IB_POLL_##x, #x }, +#define ib_poll_ctx_end(x) { IB_POLL_##x, #x } + +#define rdma_show_ib_poll_ctx(x) \ + __print_symbolic(x, IB_POLL_CTX_LIST) + +/** + ** Completion Queue events + **/ + +TRACE_EVENT(cq_schedule, + TP_PROTO( + struct ib_cq *cq + ), + + TP_ARGS(cq), + + TP_STRUCT__entry( + __field(u32, cq_id) + ), + + TP_fast_assign( + cq->timestamp = ktime_get(); + cq->interrupt = true; + + __entry->cq_id = cq->res.id; + ), + + TP_printk("cq.id=%u", __entry->cq_id) +); + +TRACE_EVENT(cq_reschedule, + TP_PROTO( + struct ib_cq *cq + ), + + TP_ARGS(cq), + + TP_STRUCT__entry( + __field(u32, cq_id) + ), + + TP_fast_assign( + cq->timestamp = ktime_get(); + cq->interrupt = false; + + __entry->cq_id = cq->res.id; + ), + + TP_printk("cq.id=%u", __entry->cq_id) +); + +TRACE_EVENT(cq_process, + TP_PROTO( + const struct ib_cq *cq + ), + + TP_ARGS(cq), + + TP_STRUCT__entry( + __field(u32, cq_id) + __field(bool, interrupt) + __field(s64, latency) + ), + + TP_fast_assign( + ktime_t latency = ktime_sub(ktime_get(), cq->timestamp); + + __entry->cq_id = cq->res.id; + __entry->latency = ktime_to_us(latency); + __entry->interrupt = cq->interrupt; + ), + + TP_printk("cq.id=%u wake-up took %lld [us] from %s", + __entry->cq_id, __entry->latency, + __entry->interrupt ? "interrupt" : "reschedule" + ) +); + +TRACE_EVENT(cq_poll, + TP_PROTO( + const struct ib_cq *cq, + int requested, + int rc + ), + + TP_ARGS(cq, requested, rc), + + TP_STRUCT__entry( + __field(u32, cq_id) + __field(int, requested) + __field(int, rc) + ), + + TP_fast_assign( + __entry->cq_id = cq->res.id; + __entry->requested = requested; + __entry->rc = rc; + ), + + TP_printk("cq.id=%u requested %d, returned %d", + __entry->cq_id, __entry->requested, __entry->rc + ) +); + +TRACE_EVENT(cq_drain_complete, + TP_PROTO( + const struct ib_cq *cq + ), + + TP_ARGS(cq), + + TP_STRUCT__entry( + __field(u32, cq_id) + ), + + TP_fast_assign( + __entry->cq_id = cq->res.id; + ), + + TP_printk("cq.id=%u", + __entry->cq_id + ) +); + + +TRACE_EVENT(cq_modify, + TP_PROTO( + const struct ib_cq *cq, + u16 comps, + u16 usec + ), + + TP_ARGS(cq, comps, usec), + + TP_STRUCT__entry( + __field(u32, cq_id) + __field(unsigned int, comps) + __field(unsigned int, usec) + ), + + TP_fast_assign( + __entry->cq_id = cq->res.id; + __entry->comps = comps; + __entry->usec = usec; + ), + + TP_printk("cq.id=%u comps=%u usec=%u", + __entry->cq_id, __entry->comps, __entry->usec + ) +); + +TRACE_EVENT(cq_alloc, + TP_PROTO( + const struct ib_cq *cq, + int nr_cqe, + int comp_vector, + enum ib_poll_context poll_ctx + ), + + TP_ARGS(cq, nr_cqe, comp_vector, poll_ctx), + + TP_STRUCT__entry( + __field(u32, cq_id) + __field(int, nr_cqe) + __field(int, comp_vector) + __field(unsigned long, poll_ctx) + ), + + TP_fast_assign( + __entry->cq_id = cq->res.id; + __entry->nr_cqe = nr_cqe; + __entry->comp_vector = comp_vector; + __entry->poll_ctx = poll_ctx; + ), + + TP_printk("cq.id=%u nr_cqe=%d comp_vector=%d poll_ctx=%s", + __entry->cq_id, __entry->nr_cqe, __entry->comp_vector, + rdma_show_ib_poll_ctx(__entry->poll_ctx) + ) +); + +TRACE_EVENT(cq_alloc_error, + TP_PROTO( + int nr_cqe, + int comp_vector, + enum ib_poll_context poll_ctx, + int rc + ), + + TP_ARGS(nr_cqe, comp_vector, poll_ctx, rc), + + TP_STRUCT__entry( + __field(int, rc) + __field(int, nr_cqe) + __field(int, comp_vector) + __field(unsigned long, poll_ctx) + ), + + TP_fast_assign( + __entry->rc = rc; + __entry->nr_cqe = nr_cqe; + __entry->comp_vector = comp_vector; + __entry->poll_ctx = poll_ctx; + ), + + TP_printk("nr_cqe=%d comp_vector=%d poll_ctx=%s rc=%d", + __entry->nr_cqe, __entry->comp_vector, + rdma_show_ib_poll_ctx(__entry->poll_ctx), __entry->rc + ) +); + +TRACE_EVENT(cq_free, + TP_PROTO( + const struct ib_cq *cq + ), + + TP_ARGS(cq), + + TP_STRUCT__entry( + __field(u32, cq_id) + ), + + TP_fast_assign( + __entry->cq_id = cq->res.id; + ), + + TP_printk("cq.id=%u", __entry->cq_id) +); + +#endif /* _TRACE_RDMA_CORE_H */ + +#include -- cgit v1.2.3-59-g8ed1b From 622db5b6439ab4d28a44d2d49d58050e59cac781 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Wed, 18 Dec 2019 15:18:20 -0500 Subject: RDMA/core: Add trace points to follow MR allocation Track the lifetime of ib_mr objects. Here's sample output from a test run with NFS/RDMA: <...>-361 [009] 79238.772782: mr_alloc: pd.id=3 mr.id=11 type=MEM_REG max_num_sg=30 rc=0 <...>-361 [009] 79238.772812: mr_alloc: pd.id=3 mr.id=12 type=MEM_REG max_num_sg=30 rc=0 <...>-361 [009] 79238.772839: mr_alloc: pd.id=3 mr.id=13 type=MEM_REG max_num_sg=30 rc=0 <...>-361 [009] 79238.772866: mr_alloc: pd.id=3 mr.id=14 type=MEM_REG max_num_sg=30 rc=0 <...>-361 [009] 79238.772893: mr_alloc: pd.id=3 mr.id=15 type=MEM_REG max_num_sg=30 rc=0 <...>-361 [009] 79238.772921: mr_alloc: pd.id=3 mr.id=16 type=MEM_REG max_num_sg=30 rc=0 <...>-361 [009] 79238.772947: mr_alloc: pd.id=3 mr.id=17 type=MEM_REG max_num_sg=30 rc=0 <...>-361 [009] 79238.772974: mr_alloc: pd.id=3 mr.id=18 type=MEM_REG max_num_sg=30 rc=0 <...>-361 [009] 79238.773001: mr_alloc: pd.id=3 mr.id=19 type=MEM_REG max_num_sg=30 rc=0 <...>-361 [009] 79238.773028: mr_alloc: pd.id=3 mr.id=20 type=MEM_REG max_num_sg=30 rc=0 <...>-361 [009] 79238.773055: mr_alloc: pd.id=3 mr.id=21 type=MEM_REG max_num_sg=30 rc=0 <...>-361 [009] 79240.270942: mr_alloc: pd.id=3 mr.id=22 type=MEM_REG max_num_sg=30 rc=0 <...>-361 [009] 79240.270975: mr_alloc: pd.id=3 mr.id=23 type=MEM_REG max_num_sg=30 rc=0 <...>-361 [009] 79240.271007: mr_alloc: pd.id=3 mr.id=24 type=MEM_REG max_num_sg=30 rc=0 <...>-361 [009] 79240.271036: mr_alloc: pd.id=3 mr.id=25 type=MEM_REG max_num_sg=30 rc=0 <...>-361 [009] 79240.271067: mr_alloc: pd.id=3 mr.id=26 type=MEM_REG max_num_sg=30 rc=0 <...>-361 [009] 79240.271095: mr_alloc: pd.id=3 mr.id=27 type=MEM_REG max_num_sg=30 rc=0 <...>-361 [009] 79240.271121: mr_alloc: pd.id=3 mr.id=28 type=MEM_REG max_num_sg=30 rc=0 <...>-361 [009] 79240.271153: mr_alloc: pd.id=3 mr.id=29 type=MEM_REG max_num_sg=30 rc=0 <...>-361 [009] 79240.271181: mr_alloc: pd.id=3 mr.id=30 type=MEM_REG max_num_sg=30 rc=0 <...>-361 [009] 79240.271208: mr_alloc: pd.id=3 mr.id=31 type=MEM_REG max_num_sg=30 rc=0 <...>-361 [009] 79240.271236: mr_alloc: pd.id=3 mr.id=32 type=MEM_REG max_num_sg=30 rc=0 <...>-4351 [001] 79242.299400: mr_dereg: mr.id=32 <...>-4351 [001] 79242.299467: mr_dereg: mr.id=31 <...>-4351 [001] 79242.299554: mr_dereg: mr.id=30 <...>-4351 [001] 79242.299615: mr_dereg: mr.id=29 <...>-4351 [001] 79242.299684: mr_dereg: mr.id=28 <...>-4351 [001] 79242.299748: mr_dereg: mr.id=27 <...>-4351 [001] 79242.299812: mr_dereg: mr.id=26 <...>-4351 [001] 79242.299874: mr_dereg: mr.id=25 <...>-4351 [001] 79242.299944: mr_dereg: mr.id=24 <...>-4351 [001] 79242.300009: mr_dereg: mr.id=23 <...>-4351 [001] 79242.300190: mr_dereg: mr.id=22 <...>-4351 [001] 79242.300263: mr_dereg: mr.id=21 <...>-4351 [001] 79242.300326: mr_dereg: mr.id=20 <...>-4351 [001] 79242.300388: mr_dereg: mr.id=19 <...>-4351 [001] 79242.300450: mr_dereg: mr.id=18 <...>-4351 [001] 79242.300516: mr_dereg: mr.id=17 <...>-4351 [001] 79242.300629: mr_dereg: mr.id=16 <...>-4351 [001] 79242.300718: mr_dereg: mr.id=15 <...>-4351 [001] 79242.300784: mr_dereg: mr.id=14 <...>-4351 [001] 79242.300879: mr_dereg: mr.id=13 <...>-4351 [001] 79242.300945: mr_dereg: mr.id=12 <...>-4351 [001] 79242.301012: mr_dereg: mr.id=11 Some features of the output: - The lifetime and owner PD of each MR is clearly visible. - The type of MR is captured, as is the SGE array size. - Failing MR allocation can be recorded. Link: https://lore.kernel.org/r/20191218201820.30584.34636.stgit@manet.1015granger.net Signed-off-by: Chuck Lever Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/verbs.c | 39 +++++++++---- include/trace/events/rdma_core.h | 123 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 11 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index 289b2f7a9d5e..47d54c31eb2a 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -52,6 +52,7 @@ #include #include "core_priv.h" +#include #include @@ -1999,6 +2000,7 @@ int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata) struct ib_sig_attrs *sig_attrs = mr->sig_attrs; int ret; + trace_mr_dereg(mr); rdma_restrack_del(&mr->res); ret = mr->device->ops.dereg_mr(mr, udata); if (!ret) { @@ -2030,11 +2032,16 @@ struct ib_mr *ib_alloc_mr_user(struct ib_pd *pd, enum ib_mr_type mr_type, { struct ib_mr *mr; - if (!pd->device->ops.alloc_mr) - return ERR_PTR(-EOPNOTSUPP); + if (!pd->device->ops.alloc_mr) { + mr = ERR_PTR(-EOPNOTSUPP); + goto out; + } - if (WARN_ON_ONCE(mr_type == IB_MR_TYPE_INTEGRITY)) - return ERR_PTR(-EINVAL); + if (mr_type == IB_MR_TYPE_INTEGRITY) { + WARN_ON_ONCE(1); + mr = ERR_PTR(-EINVAL); + goto out; + } mr = pd->device->ops.alloc_mr(pd, mr_type, max_num_sg, udata); if (!IS_ERR(mr)) { @@ -2050,6 +2057,8 @@ struct ib_mr *ib_alloc_mr_user(struct ib_pd *pd, enum ib_mr_type mr_type, mr->sig_attrs = NULL; } +out: + trace_mr_alloc(pd, mr_type, max_num_sg, mr); return mr; } EXPORT_SYMBOL(ib_alloc_mr_user); @@ -2074,21 +2083,27 @@ struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd, struct ib_sig_attrs *sig_attrs; if (!pd->device->ops.alloc_mr_integrity || - !pd->device->ops.map_mr_sg_pi) - return ERR_PTR(-EOPNOTSUPP); + !pd->device->ops.map_mr_sg_pi) { + mr = ERR_PTR(-EOPNOTSUPP); + goto out; + } - if (!max_num_meta_sg) - return ERR_PTR(-EINVAL); + if (!max_num_meta_sg) { + mr = ERR_PTR(-EINVAL); + goto out; + } sig_attrs = kzalloc(sizeof(struct ib_sig_attrs), GFP_KERNEL); - if (!sig_attrs) - return ERR_PTR(-ENOMEM); + if (!sig_attrs) { + mr = ERR_PTR(-ENOMEM); + goto out; + } mr = pd->device->ops.alloc_mr_integrity(pd, max_num_data_sg, max_num_meta_sg); if (IS_ERR(mr)) { kfree(sig_attrs); - return mr; + goto out; } mr->device = pd->device; @@ -2102,6 +2117,8 @@ struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd, mr->type = IB_MR_TYPE_INTEGRITY; mr->sig_attrs = sig_attrs; +out: + trace_mr_integ_alloc(pd, max_num_data_sg, max_num_meta_sg, mr); return mr; } EXPORT_SYMBOL(ib_alloc_mr_integrity); diff --git a/include/trace/events/rdma_core.h b/include/trace/events/rdma_core.h index 08f481554e7f..17642aa54437 100644 --- a/include/trace/events/rdma_core.h +++ b/include/trace/events/rdma_core.h @@ -266,6 +266,129 @@ TRACE_EVENT(cq_free, TP_printk("cq.id=%u", __entry->cq_id) ); +/** + ** Memory Region events + **/ + +/* + * enum ib_mr_type, from include/rdma/ib_verbs.h + */ +#define IB_MR_TYPE_LIST \ + ib_mr_type_item(MEM_REG) \ + ib_mr_type_item(SG_GAPS) \ + ib_mr_type_item(DM) \ + ib_mr_type_item(USER) \ + ib_mr_type_item(DMA) \ + ib_mr_type_end(INTEGRITY) + +#undef ib_mr_type_item +#undef ib_mr_type_end + +#define ib_mr_type_item(x) TRACE_DEFINE_ENUM(IB_MR_TYPE_##x); +#define ib_mr_type_end(x) TRACE_DEFINE_ENUM(IB_MR_TYPE_##x); + +IB_MR_TYPE_LIST + +#undef ib_mr_type_item +#undef ib_mr_type_end + +#define ib_mr_type_item(x) { IB_MR_TYPE_##x, #x }, +#define ib_mr_type_end(x) { IB_MR_TYPE_##x, #x } + +#define rdma_show_ib_mr_type(x) \ + __print_symbolic(x, IB_MR_TYPE_LIST) + +TRACE_EVENT(mr_alloc, + TP_PROTO( + const struct ib_pd *pd, + enum ib_mr_type mr_type, + u32 max_num_sg, + const struct ib_mr *mr + ), + + TP_ARGS(pd, mr_type, max_num_sg, mr), + + TP_STRUCT__entry( + __field(u32, pd_id) + __field(u32, mr_id) + __field(u32, max_num_sg) + __field(int, rc) + __field(unsigned long, mr_type) + ), + + TP_fast_assign( + __entry->pd_id = pd->res.id; + if (IS_ERR(mr)) { + __entry->mr_id = 0; + __entry->rc = PTR_ERR(mr); + } else { + __entry->mr_id = mr->res.id; + __entry->rc = 0; + } + __entry->max_num_sg = max_num_sg; + __entry->mr_type = mr_type; + ), + + TP_printk("pd.id=%u mr.id=%u type=%s max_num_sg=%u rc=%d", + __entry->pd_id, __entry->mr_id, + rdma_show_ib_mr_type(__entry->mr_type), + __entry->max_num_sg, __entry->rc) +); + +TRACE_EVENT(mr_integ_alloc, + TP_PROTO( + const struct ib_pd *pd, + u32 max_num_data_sg, + u32 max_num_meta_sg, + const struct ib_mr *mr + ), + + TP_ARGS(pd, max_num_data_sg, max_num_meta_sg, mr), + + TP_STRUCT__entry( + __field(u32, pd_id) + __field(u32, mr_id) + __field(u32, max_num_data_sg) + __field(u32, max_num_meta_sg) + __field(int, rc) + ), + + TP_fast_assign( + __entry->pd_id = pd->res.id; + if (IS_ERR(mr)) { + __entry->mr_id = 0; + __entry->rc = PTR_ERR(mr); + } else { + __entry->mr_id = mr->res.id; + __entry->rc = 0; + } + __entry->max_num_data_sg = max_num_data_sg; + __entry->max_num_meta_sg = max_num_meta_sg; + ), + + TP_printk("pd.id=%u mr.id=%u max_num_data_sg=%u max_num_meta_sg=%u rc=%d", + __entry->pd_id, __entry->mr_id, __entry->max_num_data_sg, + __entry->max_num_meta_sg, __entry->rc) +); + +TRACE_EVENT(mr_dereg, + TP_PROTO( + const struct ib_mr *mr + ), + + TP_ARGS(mr), + + TP_STRUCT__entry( + __field(u32, id) + ), + + TP_fast_assign( + __entry->id = mr->res.id; + ), + + TP_printk("mr.id=%u", __entry->id) +); + #endif /* _TRACE_RDMA_CORE_H */ #include -- cgit v1.2.3-59-g8ed1b From 6b57cea9221b0247ad5111b348522625e489a8e4 Mon Sep 17 00:00:00 2001 From: Parav Pandit Date: Thu, 12 Dec 2019 13:30:22 +0200 Subject: IB/core: Let IB core distribute cache update events Currently when the low level driver notifies Pkey, GID, and port change events they are notified to the registered handlers in the order they are registered. IB core and other ULPs such as IPoIB are interested in GID, LID, Pkey change events. Since all GID queries done by ULPs are serviced by IB core, and the IB core deferes cache updates to a work queue, it is possible for other clients to see stale cache data when they handle their own events. For example, the below call tree shows how ipoib will call rdma_query_gid() concurrently with the update to the cache sitting in the WQ. mlx5_ib_handle_event() ib_dispatch_event() ib_cache_event() queue_work() -> slow cache update [..] ipoib_event() queue_work() [..] work handler ipoib_ib_dev_flush_light() __ipoib_ib_dev_flush() ipoib_dev_addr_changed_valid() rdma_query_gid() <- Returns old GID, cache not updated. Move all the event dispatch to a work queue so that the cache update is always done before any clients are notified. Fixes: f35faa4ba956 ("IB/core: Simplify ib_query_gid to always refer to cache") Link: https://lore.kernel.org/r/20191212113024.336702-3-leon@kernel.org Signed-off-by: Parav Pandit Signed-off-by: Leon Romanovsky Reviewed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/cache.c | 121 ++++++++++++++++++++++-------------- drivers/infiniband/core/core_priv.h | 1 + drivers/infiniband/core/device.c | 33 +++------- include/rdma/ib_verbs.h | 9 ++- 4 files changed, 92 insertions(+), 72 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c index d535995711c3..e55f345799e4 100644 --- a/drivers/infiniband/core/cache.c +++ b/drivers/infiniband/core/cache.c @@ -51,9 +51,8 @@ struct ib_pkey_cache { struct ib_update_work { struct work_struct work; - struct ib_device *device; - u8 port_num; - bool enforce_security; + struct ib_event event; + bool enforce_security; }; union ib_gid zgid; @@ -130,7 +129,7 @@ static void dispatch_gid_change_event(struct ib_device *ib_dev, u8 port) event.element.port_num = port; event.event = IB_EVENT_GID_CHANGE; - ib_dispatch_event(&event); + ib_dispatch_event_clients(&event); } static const char * const gid_type_str[] = { @@ -1381,9 +1380,8 @@ err: return ret; } -static void ib_cache_update(struct ib_device *device, - u8 port, - bool enforce_security) +static int +ib_cache_update(struct ib_device *device, u8 port, bool enforce_security) { struct ib_port_attr *tprops = NULL; struct ib_pkey_cache *pkey_cache = NULL, *old_pkey_cache; @@ -1391,11 +1389,11 @@ static void ib_cache_update(struct ib_device *device, int ret; if (!rdma_is_port_valid(device, port)) - return; + return -EINVAL; tprops = kmalloc(sizeof *tprops, GFP_KERNEL); if (!tprops) - return; + return -ENOMEM; ret = ib_query_port(device, port, tprops); if (ret) { @@ -1413,8 +1411,10 @@ static void ib_cache_update(struct ib_device *device, pkey_cache = kmalloc(struct_size(pkey_cache, table, tprops->pkey_tbl_len), GFP_KERNEL); - if (!pkey_cache) + if (!pkey_cache) { + ret = -ENOMEM; goto err; + } pkey_cache->table_len = tprops->pkey_tbl_len; @@ -1446,50 +1446,84 @@ static void ib_cache_update(struct ib_device *device, kfree(old_pkey_cache); kfree(tprops); - return; + return 0; err: kfree(pkey_cache); kfree(tprops); + return ret; +} + +static void ib_cache_event_task(struct work_struct *_work) +{ + struct ib_update_work *work = + container_of(_work, struct ib_update_work, work); + int ret; + + /* Before distributing the cache update event, first sync + * the cache. + */ + ret = ib_cache_update(work->event.device, work->event.element.port_num, + work->enforce_security); + + /* GID event is notified already for individual GID entries by + * dispatch_gid_change_event(). Hence, notifiy for rest of the + * events. + */ + if (!ret && work->event.event != IB_EVENT_GID_CHANGE) + ib_dispatch_event_clients(&work->event); + + kfree(work); } -static void ib_cache_task(struct work_struct *_work) +static void ib_generic_event_task(struct work_struct *_work) { struct ib_update_work *work = container_of(_work, struct ib_update_work, work); - ib_cache_update(work->device, - work->port_num, - work->enforce_security); + ib_dispatch_event_clients(&work->event); kfree(work); } -static void ib_cache_event(struct ib_event_handler *handler, - struct ib_event *event) +static bool is_cache_update_event(const struct ib_event *event) +{ + return (event->event == IB_EVENT_PORT_ERR || + event->event == IB_EVENT_PORT_ACTIVE || + event->event == IB_EVENT_LID_CHANGE || + event->event == IB_EVENT_PKEY_CHANGE || + event->event == IB_EVENT_CLIENT_REREGISTER || + event->event == IB_EVENT_GID_CHANGE); +} + +/** + * ib_dispatch_event - Dispatch an asynchronous event + * @event:Event to dispatch + * + * Low-level drivers must call ib_dispatch_event() to dispatch the + * event to all registered event handlers when an asynchronous event + * occurs. + */ +void ib_dispatch_event(const struct ib_event *event) { struct ib_update_work *work; - if (event->event == IB_EVENT_PORT_ERR || - event->event == IB_EVENT_PORT_ACTIVE || - event->event == IB_EVENT_LID_CHANGE || - event->event == IB_EVENT_PKEY_CHANGE || - event->event == IB_EVENT_CLIENT_REREGISTER || - event->event == IB_EVENT_GID_CHANGE) { - work = kmalloc(sizeof *work, GFP_ATOMIC); - if (work) { - INIT_WORK(&work->work, ib_cache_task); - work->device = event->device; - work->port_num = event->element.port_num; - if (event->event == IB_EVENT_PKEY_CHANGE || - event->event == IB_EVENT_GID_CHANGE) - work->enforce_security = true; - else - work->enforce_security = false; - - queue_work(ib_wq, &work->work); - } - } + work = kzalloc(sizeof(*work), GFP_ATOMIC); + if (!work) + return; + + if (is_cache_update_event(event)) + INIT_WORK(&work->work, ib_cache_event_task); + else + INIT_WORK(&work->work, ib_generic_event_task); + + work->event = *event; + if (event->event == IB_EVENT_PKEY_CHANGE || + event->event == IB_EVENT_GID_CHANGE) + work->enforce_security = true; + + queue_work(ib_wq, &work->work); } +EXPORT_SYMBOL(ib_dispatch_event); int ib_cache_setup_one(struct ib_device *device) { @@ -1505,9 +1539,6 @@ int ib_cache_setup_one(struct ib_device *device) rdma_for_each_port (device, p) ib_cache_update(device, p, true); - INIT_IB_EVENT_HANDLER(&device->cache.event_handler, - device, ib_cache_event); - ib_register_event_handler(&device->cache.event_handler); return 0; } @@ -1529,14 +1560,12 @@ void ib_cache_release_one(struct ib_device *device) void ib_cache_cleanup_one(struct ib_device *device) { - /* The cleanup function unregisters the event handler, - * waits for all in-progress workqueue elements and cleans - * up the GID cache. This function should be called after - * the device was removed from the devices list and all - * clients were removed, so the cache exists but is + /* The cleanup function waits for all in-progress workqueue + * elements and cleans up the GID cache. This function should be + * called after the device was removed from the devices list and + * all clients were removed, so the cache exists but is * non-functional and shouldn't be updated anymore. */ - ib_unregister_event_handler(&device->cache.event_handler); flush_workqueue(ib_wq); gid_table_cleanup_one(device); diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h index 3645e092e1c7..d657d90e618b 100644 --- a/drivers/infiniband/core/core_priv.h +++ b/drivers/infiniband/core/core_priv.h @@ -149,6 +149,7 @@ unsigned long roce_gid_type_mask_support(struct ib_device *ib_dev, u8 port); int ib_cache_setup_one(struct ib_device *device); void ib_cache_cleanup_one(struct ib_device *device); void ib_cache_release_one(struct ib_device *device); +void ib_dispatch_event_clients(struct ib_event *event); #ifdef CONFIG_CGROUP_RDMA void ib_device_register_rdmacg(struct ib_device *device); diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index 84dd74fe13b8..c38b2b0b078a 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -588,6 +588,7 @@ struct ib_device *_ib_alloc_device(size_t size) INIT_LIST_HEAD(&device->event_handler_list); spin_lock_init(&device->event_handler_lock); + init_rwsem(&device->event_handler_rwsem); mutex_init(&device->unregistration_lock); /* * client_data needs to be alloc because we don't want our mark to be @@ -1931,17 +1932,15 @@ EXPORT_SYMBOL(ib_set_client_data); * * ib_register_event_handler() registers an event handler that will be * called back when asynchronous IB events occur (as defined in - * chapter 11 of the InfiniBand Architecture Specification). This - * callback may occur in interrupt context. + * chapter 11 of the InfiniBand Architecture Specification). This + * callback occurs in workqueue context. */ void ib_register_event_handler(struct ib_event_handler *event_handler) { - unsigned long flags; - - spin_lock_irqsave(&event_handler->device->event_handler_lock, flags); + down_write(&event_handler->device->event_handler_rwsem); list_add_tail(&event_handler->list, &event_handler->device->event_handler_list); - spin_unlock_irqrestore(&event_handler->device->event_handler_lock, flags); + up_write(&event_handler->device->event_handler_rwsem); } EXPORT_SYMBOL(ib_register_event_handler); @@ -1954,35 +1953,23 @@ EXPORT_SYMBOL(ib_register_event_handler); */ void ib_unregister_event_handler(struct ib_event_handler *event_handler) { - unsigned long flags; - - spin_lock_irqsave(&event_handler->device->event_handler_lock, flags); + down_write(&event_handler->device->event_handler_rwsem); list_del(&event_handler->list); - spin_unlock_irqrestore(&event_handler->device->event_handler_lock, flags); + up_write(&event_handler->device->event_handler_rwsem); } EXPORT_SYMBOL(ib_unregister_event_handler); -/** - * ib_dispatch_event - Dispatch an asynchronous event - * @event:Event to dispatch - * - * Low-level drivers must call ib_dispatch_event() to dispatch the - * event to all registered event handlers when an asynchronous event - * occurs. - */ -void ib_dispatch_event(struct ib_event *event) +void ib_dispatch_event_clients(struct ib_event *event) { - unsigned long flags; struct ib_event_handler *handler; - spin_lock_irqsave(&event->device->event_handler_lock, flags); + down_read(&event->device->event_handler_rwsem); list_for_each_entry(handler, &event->device->event_handler_list, list) handler->handler(handler, event); - spin_unlock_irqrestore(&event->device->event_handler_lock, flags); + up_read(&event->device->event_handler_rwsem); } -EXPORT_SYMBOL(ib_dispatch_event); static int iw_query_port(struct ib_device *device, u8 port_num, diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 42f28d39f28c..f36fb657518f 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -2154,7 +2154,6 @@ struct ib_port_cache { struct ib_cache { rwlock_t lock; - struct ib_event_handler event_handler; }; struct ib_port_immutable { @@ -2632,7 +2631,11 @@ struct ib_device { struct rcu_head rcu_head; struct list_head event_handler_list; - spinlock_t event_handler_lock; + /* Protects event_handler_list */ + struct rw_semaphore event_handler_rwsem; + + /* Protects QP's event_handler calls and open_qp list */ + spinlock_t event_handler_lock; struct rw_semaphore client_data_rwsem; struct xarray client_data; @@ -2947,7 +2950,7 @@ bool ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state, void ib_register_event_handler(struct ib_event_handler *event_handler); void ib_unregister_event_handler(struct ib_event_handler *event_handler); -void ib_dispatch_event(struct ib_event *event); +void ib_dispatch_event(const struct ib_event *event); int ib_query_port(struct ib_device *device, u8 port_num, struct ib_port_attr *port_attr); -- cgit v1.2.3-59-g8ed1b From 17e1064632512db419cb9bb4555aec1763969b7d Mon Sep 17 00:00:00 2001 From: Parav Pandit Date: Thu, 12 Dec 2019 13:30:23 +0200 Subject: IB/core: Cut down single member ib_cache structure Given that ib_cache structure has only single member now, merge the cache lock directly in the ib_device. Link: https://lore.kernel.org/r/20191212113024.336702-4-leon@kernel.org Signed-off-by: Parav Pandit Signed-off-by: Leon Romanovsky Reviewed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/cache.c | 30 +++++++++++++++--------------- include/rdma/ib_verbs.h | 7 ++----- 2 files changed, 17 insertions(+), 20 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c index e55f345799e4..17bfedd24cc3 100644 --- a/drivers/infiniband/core/cache.c +++ b/drivers/infiniband/core/cache.c @@ -1033,7 +1033,7 @@ int ib_get_cached_pkey(struct ib_device *device, if (!rdma_is_port_valid(device, port_num)) return -EINVAL; - read_lock_irqsave(&device->cache.lock, flags); + read_lock_irqsave(&device->cache_lock, flags); cache = device->port_data[port_num].cache.pkey; @@ -1042,7 +1042,7 @@ int ib_get_cached_pkey(struct ib_device *device, else *pkey = cache->table[index]; - read_unlock_irqrestore(&device->cache.lock, flags); + read_unlock_irqrestore(&device->cache_lock, flags); return ret; } @@ -1057,9 +1057,9 @@ int ib_get_cached_subnet_prefix(struct ib_device *device, if (!rdma_is_port_valid(device, port_num)) return -EINVAL; - read_lock_irqsave(&device->cache.lock, flags); + read_lock_irqsave(&device->cache_lock, flags); *sn_pfx = device->port_data[port_num].cache.subnet_prefix; - read_unlock_irqrestore(&device->cache.lock, flags); + read_unlock_irqrestore(&device->cache_lock, flags); return 0; } @@ -1079,7 +1079,7 @@ int ib_find_cached_pkey(struct ib_device *device, if (!rdma_is_port_valid(device, port_num)) return -EINVAL; - read_lock_irqsave(&device->cache.lock, flags); + read_lock_irqsave(&device->cache_lock, flags); cache = device->port_data[port_num].cache.pkey; @@ -1100,7 +1100,7 @@ int ib_find_cached_pkey(struct ib_device *device, ret = 0; } - read_unlock_irqrestore(&device->cache.lock, flags); + read_unlock_irqrestore(&device->cache_lock, flags); return ret; } @@ -1119,7 +1119,7 @@ int ib_find_exact_cached_pkey(struct ib_device *device, if (!rdma_is_port_valid(device, port_num)) return -EINVAL; - read_lock_irqsave(&device->cache.lock, flags); + read_lock_irqsave(&device->cache_lock, flags); cache = device->port_data[port_num].cache.pkey; @@ -1132,7 +1132,7 @@ int ib_find_exact_cached_pkey(struct ib_device *device, break; } - read_unlock_irqrestore(&device->cache.lock, flags); + read_unlock_irqrestore(&device->cache_lock, flags); return ret; } @@ -1148,9 +1148,9 @@ int ib_get_cached_lmc(struct ib_device *device, if (!rdma_is_port_valid(device, port_num)) return -EINVAL; - read_lock_irqsave(&device->cache.lock, flags); + read_lock_irqsave(&device->cache_lock, flags); *lmc = device->port_data[port_num].cache.lmc; - read_unlock_irqrestore(&device->cache.lock, flags); + read_unlock_irqrestore(&device->cache_lock, flags); return ret; } @@ -1166,9 +1166,9 @@ int ib_get_cached_port_state(struct ib_device *device, if (!rdma_is_port_valid(device, port_num)) return -EINVAL; - read_lock_irqsave(&device->cache.lock, flags); + read_lock_irqsave(&device->cache_lock, flags); *port_state = device->port_data[port_num].cache.port_state; - read_unlock_irqrestore(&device->cache.lock, flags); + read_unlock_irqrestore(&device->cache_lock, flags); return ret; } @@ -1428,7 +1428,7 @@ ib_cache_update(struct ib_device *device, u8 port, bool enforce_security) } } - write_lock_irq(&device->cache.lock); + write_lock_irq(&device->cache_lock); old_pkey_cache = device->port_data[port].cache.pkey; @@ -1437,7 +1437,7 @@ ib_cache_update(struct ib_device *device, u8 port, bool enforce_security) device->port_data[port].cache.port_state = tprops->state; device->port_data[port].cache.subnet_prefix = tprops->subnet_prefix; - write_unlock_irq(&device->cache.lock); + write_unlock_irq(&device->cache_lock); if (enforce_security) ib_security_cache_change(device, @@ -1530,7 +1530,7 @@ int ib_cache_setup_one(struct ib_device *device) unsigned int p; int err; - rwlock_init(&device->cache.lock); + rwlock_init(&device->cache_lock); err = gid_table_setup_one(device); if (err) diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index f36fb657518f..37dac147a946 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -2152,10 +2152,6 @@ struct ib_port_cache { enum ib_port_state port_state; }; -struct ib_cache { - rwlock_t lock; -}; - struct ib_port_immutable { int pkey_tbl_len; int gid_tbl_len; @@ -2641,7 +2637,8 @@ struct ib_device { struct xarray client_data; struct mutex unregistration_lock; - struct ib_cache cache; + /* Synchronize GID, Pkey cache entries, subnet prefix, LMC */ + rwlock_t cache_lock; /** * port_data is indexed by port number */ -- cgit v1.2.3-59-g8ed1b From 40adf686128856c4add948fb002d43e2c507d1aa Mon Sep 17 00:00:00 2001 From: Parav Pandit Date: Thu, 12 Dec 2019 13:30:24 +0200 Subject: IB/core: Rename event_handler_lock to qp_open_list_lock This lock is used to protect the qp->open_list linked list. As a side effect it seems to also globally serialize the qp event_handler, but it isn't clear if that is a deliberate design. Link: https://lore.kernel.org/r/20191212113024.336702-5-leon@kernel.org Signed-off-by: Parav Pandit Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/device.c | 2 +- drivers/infiniband/core/verbs.c | 12 ++++++------ include/rdma/ib_verbs.h | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index c38b2b0b078a..e4c3ea727eac 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -587,7 +587,7 @@ struct ib_device *_ib_alloc_device(size_t size) rdma_init_coredev(&device->coredev, device, &init_net); INIT_LIST_HEAD(&device->event_handler_list); - spin_lock_init(&device->event_handler_lock); + spin_lock_init(&device->qp_open_list_lock); init_rwsem(&device->event_handler_rwsem); mutex_init(&device->unregistration_lock); /* diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index 47d54c31eb2a..78b27aff2846 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -1056,11 +1056,11 @@ static void __ib_shared_qp_event_handler(struct ib_event *event, void *context) struct ib_qp *qp = context; unsigned long flags; - spin_lock_irqsave(&qp->device->event_handler_lock, flags); + spin_lock_irqsave(&qp->device->qp_open_list_lock, flags); list_for_each_entry(event->element.qp, &qp->open_list, open_list) if (event->element.qp->event_handler) event->element.qp->event_handler(event, event->element.qp->qp_context); - spin_unlock_irqrestore(&qp->device->event_handler_lock, flags); + spin_unlock_irqrestore(&qp->device->qp_open_list_lock, flags); } static void __ib_insert_xrcd_qp(struct ib_xrcd *xrcd, struct ib_qp *qp) @@ -1097,9 +1097,9 @@ static struct ib_qp *__ib_open_qp(struct ib_qp *real_qp, qp->qp_num = real_qp->qp_num; qp->qp_type = real_qp->qp_type; - spin_lock_irqsave(&real_qp->device->event_handler_lock, flags); + spin_lock_irqsave(&real_qp->device->qp_open_list_lock, flags); list_add(&qp->open_list, &real_qp->open_list); - spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags); + spin_unlock_irqrestore(&real_qp->device->qp_open_list_lock, flags); return qp; } @@ -1827,9 +1827,9 @@ int ib_close_qp(struct ib_qp *qp) if (real_qp == qp) return -EINVAL; - spin_lock_irqsave(&real_qp->device->event_handler_lock, flags); + spin_lock_irqsave(&real_qp->device->qp_open_list_lock, flags); list_del(&qp->open_list); - spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags); + spin_unlock_irqrestore(&real_qp->device->qp_open_list_lock, flags); atomic_dec(&real_qp->usecnt); if (qp->qp_sec) diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 37dac147a946..cea4e198701e 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -2631,7 +2631,7 @@ struct ib_device { struct rw_semaphore event_handler_rwsem; /* Protects QP's event_handler calls and open_qp list */ - spinlock_t event_handler_lock; + spinlock_t qp_open_list_lock; struct rw_semaphore client_data_rwsem; struct xarray client_data; -- cgit v1.2.3-59-g8ed1b From 74f75cda754eb69a77f910ceb5bc85f8e9ba56a5 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Thu, 9 Jan 2020 09:47:30 +0100 Subject: IB/core: Fix build failure without hugepages HPAGE_SHIFT is only defined on architectures that support hugepages: drivers/infiniband/core/umem_odp.c: In function 'ib_umem_odp_get': drivers/infiniband/core/umem_odp.c:245:26: error: 'HPAGE_SHIFT' undeclared (first use in this function); did you mean 'PAGE_SHIFT'? Enclose this in an #ifdef. Fixes: 9ff1b6466a29 ("IB/core: Fix ODP with IB_ACCESS_HUGETLB handling") Link: https://lore.kernel.org/r/20200109084740.2872079-1-arnd@arndb.de Signed-off-by: Arnd Bergmann Reviewed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/umem_odp.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c index f42fa31c24a2..b9baf7d0a5cb 100644 --- a/drivers/infiniband/core/umem_odp.c +++ b/drivers/infiniband/core/umem_odp.c @@ -241,10 +241,11 @@ struct ib_umem_odp *ib_umem_odp_get(struct ib_udata *udata, unsigned long addr, umem_odp->umem.owning_mm = mm = current->mm; umem_odp->notifier.ops = ops; + umem_odp->page_shift = PAGE_SHIFT; +#ifdef CONFIG_HUGETLB_PAGE if (access & IB_ACCESS_HUGETLB) umem_odp->page_shift = HPAGE_SHIFT; - else - umem_odp->page_shift = PAGE_SHIFT; +#endif umem_odp->tgid = get_task_pid(current->group_leader, PIDTYPE_PID); ret = ib_init_umem_odp(umem_odp, ops); -- cgit v1.2.3-59-g8ed1b From 14e23bd6d22123f6f3b2747701fa6cd4c6d05873 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 8 Jan 2020 19:22:03 +0200 Subject: RDMA/core: Fix locking in ib_uverbs_event_read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This should not be using ib_dev to test for disassociation, during disassociation is_closed is set under lock and the waitq is triggered. Instead check is_closed and be sure to re-obtain the lock to test the value after the wait_event returns. Fixes: 036b10635739 ("IB/uverbs: Enable device removal when there are active user space applications") Link: https://lore.kernel.org/r/1578504126-9400-12-git-send-email-yishaih@mellanox.com Signed-off-by: Yishai Hadas Reviewed-by: Håkon Bugge Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/uverbs_main.c | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index 970d8e31dd65..8f5de4dcad97 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -220,7 +220,6 @@ void ib_uverbs_release_file(struct kref *ref) } static ssize_t ib_uverbs_event_read(struct ib_uverbs_event_queue *ev_queue, - struct ib_uverbs_file *uverbs_file, struct file *filp, char __user *buf, size_t count, loff_t *pos, size_t eventsz) @@ -238,19 +237,16 @@ static ssize_t ib_uverbs_event_read(struct ib_uverbs_event_queue *ev_queue, if (wait_event_interruptible(ev_queue->poll_wait, (!list_empty(&ev_queue->event_list) || - /* The barriers built into wait_event_interruptible() - * and wake_up() guarentee this will see the null set - * without using RCU - */ - !uverbs_file->device->ib_dev))) + ev_queue->is_closed))) return -ERESTARTSYS; + spin_lock_irq(&ev_queue->lock); + /* If device was disassociated and no event exists set an error */ - if (list_empty(&ev_queue->event_list) && - !uverbs_file->device->ib_dev) + if (list_empty(&ev_queue->event_list) && ev_queue->is_closed) { + spin_unlock_irq(&ev_queue->lock); return -EIO; - - spin_lock_irq(&ev_queue->lock); + } } event = list_entry(ev_queue->event_list.next, struct ib_uverbs_event, list); @@ -285,8 +281,7 @@ static ssize_t ib_uverbs_async_event_read(struct file *filp, char __user *buf, { struct ib_uverbs_async_event_file *file = filp->private_data; - return ib_uverbs_event_read(&file->ev_queue, file->uverbs_file, filp, - buf, count, pos, + return ib_uverbs_event_read(&file->ev_queue, filp, buf, count, pos, sizeof(struct ib_uverbs_async_event_desc)); } @@ -296,9 +291,8 @@ static ssize_t ib_uverbs_comp_event_read(struct file *filp, char __user *buf, struct ib_uverbs_completion_event_file *comp_ev_file = filp->private_data; - return ib_uverbs_event_read(&comp_ev_file->ev_queue, - comp_ev_file->uobj.ufile, filp, - buf, count, pos, + return ib_uverbs_event_read(&comp_ev_file->ev_queue, filp, buf, count, + pos, sizeof(struct ib_uverbs_comp_event_desc)); } @@ -321,7 +315,9 @@ static __poll_t ib_uverbs_event_poll(struct ib_uverbs_event_queue *ev_queue, static __poll_t ib_uverbs_async_event_poll(struct file *filp, struct poll_table_struct *wait) { - return ib_uverbs_event_poll(filp->private_data, filp, wait); + struct ib_uverbs_async_event_file *file = filp->private_data; + + return ib_uverbs_event_poll(&file->ev_queue, filp, wait); } static __poll_t ib_uverbs_comp_event_poll(struct file *filp, @@ -335,9 +331,9 @@ static __poll_t ib_uverbs_comp_event_poll(struct file *filp, static int ib_uverbs_async_event_fasync(int fd, struct file *filp, int on) { - struct ib_uverbs_event_queue *ev_queue = filp->private_data; + struct ib_uverbs_async_event_file *file = filp->private_data; - return fasync_helper(fd, filp, on, &ev_queue->async_queue); + return fasync_helper(fd, filp, on, &file->ev_queue.async_queue); } static int ib_uverbs_comp_event_fasync(int fd, struct file *filp, int on) -- cgit v1.2.3-59-g8ed1b From 1e123d96b85fd41278efc994f8096e1df86fd427 Mon Sep 17 00:00:00 2001 From: Guoqing Jiang Date: Thu, 9 Jan 2020 14:40:43 +0100 Subject: RDMA/core: Remove err in iw_query_port Since we can return device->ops.query_port directly, so no need to keep those lines. Link: https://lore.kernel.org/r/20200109134043.15568-1-guoqing.jiang@cloud.ionos.com Signed-off-by: Guoqing Jiang Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/device.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index e4c3ea727eac..f6c255202d7f 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -1977,7 +1977,6 @@ static int iw_query_port(struct ib_device *device, { struct in_device *inetdev; struct net_device *netdev; - int err; memset(port_attr, 0, sizeof(*port_attr)); @@ -2008,11 +2007,7 @@ static int iw_query_port(struct ib_device *device, } dev_put(netdev); - err = device->ops.query_port(device, port_num, port_attr); - if (err) - return err; - - return 0; + return device->ops.query_port(device, port_num, port_attr); } static int __ib_query_port(struct ib_device *device, -- cgit v1.2.3-59-g8ed1b From 8bdf9dd984c18375d1090ddeb1792511f619c5c1 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 13 Jan 2020 14:33:10 +0000 Subject: RDMA/uverbs: Remove needs_kfree_rcu from uverbs_obj_type_class After device disassociation the uapi_objects are destroyed and freed, however it is still possible that core code can be holding a kref on the uobject. When it finally goes to uverbs_uobject_free() via the kref_put() it can trigger a use-after-free on the uapi_object. Since needs_kfree_rcu is a micro optimization that only benefits file uobjects, just get rid of it. There is no harm in using kfree_rcu even if it isn't required, and the number of involved objects is small. Link: https://lore.kernel.org/r/20200113143306.GA28717@ziepe.ca Signed-off-by: Michael Guralnik Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/rdma_core.c | 23 +---------------------- include/rdma/uverbs_types.h | 1 - 2 files changed, 1 insertion(+), 23 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/rdma_core.c b/drivers/infiniband/core/rdma_core.c index 6c72773faf29..17bdbe38fdfa 100644 --- a/drivers/infiniband/core/rdma_core.c +++ b/drivers/infiniband/core/rdma_core.c @@ -49,13 +49,7 @@ void uverbs_uobject_get(struct ib_uobject *uobject) static void uverbs_uobject_free(struct kref *ref) { - struct ib_uobject *uobj = - container_of(ref, struct ib_uobject, ref); - - if (uobj->uapi_object->type_class->needs_kfree_rcu) - kfree_rcu(uobj, rcu); - else - kfree(uobj); + kfree_rcu(container_of(ref, struct ib_uobject, ref), rcu); } void uverbs_uobject_put(struct ib_uobject *uobject) @@ -744,20 +738,6 @@ const struct uverbs_obj_type_class uverbs_idr_class = { .lookup_put = lookup_put_idr_uobject, .destroy_hw = destroy_hw_idr_uobject, .remove_handle = remove_handle_idr_uobject, - /* - * When we destroy an object, we first just lock it for WRITE and - * actually DESTROY it in the finalize stage. So, the problematic - * scenario is when we just started the finalize stage of the - * destruction (nothing was executed yet). Now, the other thread - * fetched the object for READ access, but it didn't lock it yet. - * The DESTROY thread continues and starts destroying the object. - * When the other thread continue - without the RCU, it would - * access freed memory. However, the rcu_read_lock delays the free - * until the rcu_read_lock of the READ operation quits. Since the - * exclusive lock of the object is still taken by the DESTROY flow, the - * READ operation will get -EBUSY and it'll just bail out. - */ - .needs_kfree_rcu = true, }; EXPORT_SYMBOL(uverbs_idr_class); @@ -920,7 +900,6 @@ const struct uverbs_obj_type_class uverbs_fd_class = { .lookup_put = lookup_put_fd_uobject, .destroy_hw = destroy_hw_fd_uobject, .remove_handle = remove_handle_fd_uobject, - .needs_kfree_rcu = false, }; EXPORT_SYMBOL(uverbs_fd_class); diff --git a/include/rdma/uverbs_types.h b/include/rdma/uverbs_types.h index d57a5ba00c74..0b0f5a5f392d 100644 --- a/include/rdma/uverbs_types.h +++ b/include/rdma/uverbs_types.h @@ -98,7 +98,6 @@ struct uverbs_obj_type_class { enum rdma_remove_reason why, struct uverbs_attr_bundle *attrs); void (*remove_handle)(struct ib_uobject *uobj); - u8 needs_kfree_rcu; }; struct uverbs_obj_type { -- cgit v1.2.3-59-g8ed1b From 6898d1c661d79f4707d8ba82991b2195822780ca Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 8 Jan 2020 19:21:53 +0200 Subject: RDMA/mlx5: Use RCU and direct refcounts to keep memory alive dispatch_event_fd() runs from a notifier with minimal locking, and relies on RCU and a file refcount to keep the uobject and eventfd alive. As the next patch wants to remove the file_operations release function from the drivers, re-organize things so that the devx_event_notifier() path uses the existing RCU to manage the lifetime of the uobject and eventfd. Move the refcount puts to a call_rcu so that the objects are guaranteed to exist and remove the indirect file refcount. Link: https://lore.kernel.org/r/1578504126-9400-2-git-send-email-yishaih@mellanox.com Signed-off-by: Yishai Hadas Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/rdma_core.c | 11 ++++++----- drivers/infiniband/core/rdma_core.h | 15 --------------- drivers/infiniband/hw/mlx5/devx.c | 34 +++++++++++++++++----------------- include/rdma/uverbs_types.h | 12 ++++++++++++ 4 files changed, 35 insertions(+), 37 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/rdma_core.c b/drivers/infiniband/core/rdma_core.c index 17bdbe38fdfa..aef6fb823206 100644 --- a/drivers/infiniband/core/rdma_core.c +++ b/drivers/infiniband/core/rdma_core.c @@ -42,20 +42,21 @@ #include "core_priv.h" #include "rdma_core.h" -void uverbs_uobject_get(struct ib_uobject *uobject) -{ - kref_get(&uobject->ref); -} - static void uverbs_uobject_free(struct kref *ref) { kfree_rcu(container_of(ref, struct ib_uobject, ref), rcu); } +/* + * In order to indicate we no longer needs this uobject, uverbs_uobject_put + * is called. When the reference count is decreased, the uobject is freed. + * For example, this is used when attaching a completion channel to a CQ. + */ void uverbs_uobject_put(struct ib_uobject *uobject) { kref_put(&uobject->ref, uverbs_uobject_free); } +EXPORT_SYMBOL(uverbs_uobject_put); static int uverbs_try_lock_object(struct ib_uobject *uobj, enum rdma_lookup_mode mode) diff --git a/drivers/infiniband/core/rdma_core.h b/drivers/infiniband/core/rdma_core.h index e63fbda25e1d..d5d58a10bb28 100644 --- a/drivers/infiniband/core/rdma_core.h +++ b/drivers/infiniband/core/rdma_core.h @@ -50,21 +50,6 @@ void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile, int uobj_destroy(struct ib_uobject *uobj, struct uverbs_attr_bundle *attrs); -/* - * uverbs_uobject_get is called in order to increase the reference count on - * an uobject. This is useful when a handler wants to keep the uobject's memory - * alive, regardless if this uobject is still alive in the context's objects - * repository. Objects are put via uverbs_uobject_put. - */ -void uverbs_uobject_get(struct ib_uobject *uobject); - -/* - * In order to indicate we no longer needs this uobject, uverbs_uobject_put - * is called. When the reference count is decreased, the uobject is freed. - * For example, this is used when attaching a completion channel to a CQ. - */ -void uverbs_uobject_put(struct ib_uobject *uobject); - /* Indicate this fd is no longer used by this consumer, but its memory isn't * necessarily released yet. When the last reference is put, we release the * memory. After this call is executed, calling uverbs_uobject_get isn't diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c index 9d0a18cf9e5e..968fff018fc0 100644 --- a/drivers/infiniband/hw/mlx5/devx.c +++ b/drivers/infiniband/hw/mlx5/devx.c @@ -72,7 +72,6 @@ struct devx_event_subscription { struct rcu_head rcu; u64 cookie; struct devx_async_event_file *ev_file; - struct file *filp; /* Upon hot unplug we need a direct access to */ struct eventfd_ctx *eventfd; }; @@ -2032,6 +2031,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)( goto err; list_add_tail(&event_sub->event_list, &sub_list); + uverbs_uobject_get(&ev_file->uobj); if (use_eventfd) { event_sub->eventfd = eventfd_ctx_fdget(redirect_fd); @@ -2045,7 +2045,6 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)( event_sub->cookie = cookie; event_sub->ev_file = ev_file; - event_sub->filp = fd_uobj->object; /* May be needed upon cleanup the devx object/subscription */ event_sub->xa_key_level1 = key_level1; event_sub->xa_key_level2 = obj_id; @@ -2099,7 +2098,7 @@ err: if (event_sub->eventfd) eventfd_ctx_put(event_sub->eventfd); - + uverbs_uobject_put(&event_sub->ev_file->uobj); kfree(event_sub); } @@ -2361,17 +2360,10 @@ static void dispatch_event_fd(struct list_head *fd_list, struct devx_event_subscription *item; list_for_each_entry_rcu(item, fd_list, xa_list) { - if (!get_file_rcu(item->filp)) - continue; - - if (item->eventfd) { + if (item->eventfd) eventfd_signal(item->eventfd, 1); - fput(item->filp); - continue; - } - - deliver_event(item, data); - fput(item->filp); + else + deliver_event(item, data); } } @@ -2653,6 +2645,17 @@ static __poll_t devx_async_event_poll(struct file *filp, return pollflags; } +static void devx_free_subscription(struct rcu_head *rcu) +{ + struct devx_event_subscription *event_sub = + container_of(rcu, struct devx_event_subscription, rcu); + + if (event_sub->eventfd) + eventfd_ctx_put(event_sub->eventfd); + uverbs_uobject_put(&event_sub->ev_file->uobj); + kfree(event_sub); +} + static int devx_async_event_close(struct inode *inode, struct file *filp) { struct devx_async_event_file *ev_file = filp->private_data; @@ -2665,12 +2668,9 @@ static int devx_async_event_close(struct inode *inode, struct file *filp) list_for_each_entry_safe(event_sub, event_sub_tmp, &ev_file->subscribed_events_list, file_list) { devx_cleanup_subscription(dev, event_sub); - if (event_sub->eventfd) - eventfd_ctx_put(event_sub->eventfd); - list_del_rcu(&event_sub->file_list); /* subscription may not be used by the read API any more */ - kfree_rcu(event_sub, rcu); + call_rcu(&event_sub->rcu, devx_free_subscription); } mutex_unlock(&dev->devx_event_table.event_xa_lock); diff --git a/include/rdma/uverbs_types.h b/include/rdma/uverbs_types.h index 0b0f5a5f392d..ca65d4704b4b 100644 --- a/include/rdma/uverbs_types.h +++ b/include/rdma/uverbs_types.h @@ -144,6 +144,18 @@ void rdma_alloc_abort_uobject(struct ib_uobject *uobj, int __must_check rdma_alloc_commit_uobject(struct ib_uobject *uobj, struct uverbs_attr_bundle *attrs); +/* + * uverbs_uobject_get is called in order to increase the reference count on + * an uobject. This is useful when a handler wants to keep the uobject's memory + * alive, regardless if this uobject is still alive in the context's objects + * repository. Objects are put via uverbs_uobject_put. + */ +static inline void uverbs_uobject_get(struct ib_uobject *uobject) +{ + kref_get(&uobject->ref); +} +void uverbs_uobject_put(struct ib_uobject *uobject); + struct uverbs_obj_fd_type { /* * In fd based objects, uverbs_obj_type_ops points to generic -- cgit v1.2.3-59-g8ed1b From f7c8416ccea52b41e29227b3a5066540f51ee471 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 8 Jan 2020 19:21:54 +0200 Subject: RDMA/core: Simplify destruction of FD uobjects FD uobjects have a weird split between the struct file and uobject world. Simplify this to make them pure uobjects and use a generic release method for all struct file operations. This fixes the control flow so that mlx5_cmd_cleanup_async_ctx() is always called before erasing the linked list contents to make the concurrancy simpler to understand. For this to work the uobject destruction must fence anything that it is cleaning up - the design must not rely on struct file lifetime. Only deliver_event() relies on the struct file to when adding new events to the queue, add a is_destroyed check under lock to block it. Link: https://lore.kernel.org/r/1578504126-9400-3-git-send-email-yishaih@mellanox.com Signed-off-by: Yishai Hadas Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/rdma_core.c | 34 +++++---- drivers/infiniband/core/rdma_core.h | 8 --- drivers/infiniband/core/uverbs_main.c | 23 +----- drivers/infiniband/core/uverbs_std_types.c | 23 +++--- drivers/infiniband/core/uverbs_uapi.c | 6 +- drivers/infiniband/hw/mlx5/devx.c | 111 +++++++++++++---------------- include/rdma/uverbs_types.h | 12 ++-- 7 files changed, 94 insertions(+), 123 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/rdma_core.c b/drivers/infiniband/core/rdma_core.c index aef6fb823206..0ed0341b8e30 100644 --- a/drivers/infiniband/core/rdma_core.c +++ b/drivers/infiniband/core/rdma_core.c @@ -353,9 +353,9 @@ lookup_get_fd_uobject(const struct uverbs_api_object *obj, uobject = f->private_data; /* - * fget(id) ensures we are not currently running uverbs_close_fd, - * and the caller is expected to ensure that uverbs_close_fd is never - * done while a call top lookup is possible. + * fget(id) ensures we are not currently running + * uverbs_uobject_fd_release(), and the caller is expected to ensure + * that release is never done while a call to lookup is possible. */ if (f->f_op != fd_type->fops) { fput(f); @@ -548,7 +548,7 @@ static int __must_check destroy_hw_fd_uobject(struct ib_uobject *uobj, { const struct uverbs_obj_fd_type *fd_type = container_of( uobj->uapi_object->type_attrs, struct uverbs_obj_fd_type, type); - int ret = fd_type->context_closed(uobj, why); + int ret = fd_type->destroy_object(uobj, why); if (ib_is_destroy_retryable(ret, why, uobj)) return ret; @@ -587,9 +587,9 @@ static int alloc_commit_fd_uobject(struct ib_uobject *uobj) /* * The kref for uobj is moved into filp->private data and put in - * uverbs_close_fd(). Once alloc_commit() succeeds uverbs_close_fd() - * must be guaranteed to be called from the provided fops release - * callback. + * uverbs_close_fd(). Once alloc_commit() succeeds + * uverbs_uobject_fd_release() must be guaranteed to be called from + * the provided fops release callback. */ filp = anon_inode_getfile(fd_type->name, fd_type->fops, @@ -600,7 +600,7 @@ static int alloc_commit_fd_uobject(struct ib_uobject *uobj) uobj->object = filp; - /* Matching put will be done in uverbs_close_fd() */ + /* Matching put will be done in uverbs_uobject_fd_release() */ kref_get(&uobj->ufile->ref); /* This shouldn't be used anymore. Use the file object instead */ @@ -608,7 +608,7 @@ static int alloc_commit_fd_uobject(struct ib_uobject *uobj) /* * NOTE: Once we install the file we loose ownership of our kref on - * uobj. It will be put by uverbs_close_fd() + * uobj. It will be put by uverbs_uobject_fd_release() */ fd_install(fd, filp); @@ -676,7 +676,10 @@ static void lookup_put_fd_uobject(struct ib_uobject *uobj, struct file *filp = uobj->object; WARN_ON(mode != UVERBS_LOOKUP_READ); - /* This indirectly calls uverbs_close_fd and free the object */ + /* + * This indirectly calls uverbs_uobject_fd_release() and free the + * object + */ fput(filp); } @@ -742,9 +745,13 @@ const struct uverbs_obj_type_class uverbs_idr_class = { }; EXPORT_SYMBOL(uverbs_idr_class); -void uverbs_close_fd(struct file *f) +/* + * Users of UVERBS_TYPE_ALLOC_FD should set this function as the struct + * file_operations release method. + */ +int uverbs_uobject_fd_release(struct inode *inode, struct file *filp) { - struct ib_uobject *uobj = f->private_data; + struct ib_uobject *uobj = filp->private_data; struct ib_uverbs_file *ufile = uobj->ufile; struct uverbs_attr_bundle attrs = { .context = uobj->context, @@ -768,8 +775,9 @@ void uverbs_close_fd(struct file *f) /* Pairs with filp->private_data in alloc_begin_fd_uobject */ uverbs_uobject_put(uobj); + return 0; } -EXPORT_SYMBOL(uverbs_close_fd); +EXPORT_SYMBOL(uverbs_uobject_fd_release); /* * Drop the ucontext off the ufile and completely disconnect it from the diff --git a/drivers/infiniband/core/rdma_core.h b/drivers/infiniband/core/rdma_core.h index d5d58a10bb28..92694253e776 100644 --- a/drivers/infiniband/core/rdma_core.h +++ b/drivers/infiniband/core/rdma_core.h @@ -50,14 +50,6 @@ void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile, int uobj_destroy(struct ib_uobject *uobj, struct uverbs_attr_bundle *attrs); -/* Indicate this fd is no longer used by this consumer, but its memory isn't - * necessarily released yet. When the last reference is put, we release the - * memory. After this call is executed, calling uverbs_uobject_get isn't - * allowed. - * This must be called from the release file_operations of the file! - */ -void uverbs_close_fd(struct file *f); - /* * Get an ib_uobject that corresponds to the given id from ufile, assuming * the object is from the given type. Lock it to the required access when diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index 8f5de4dcad97..da56fda259fd 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -373,32 +373,11 @@ static int ib_uverbs_async_event_close(struct inode *inode, struct file *filp) return 0; } -static int ib_uverbs_comp_event_close(struct inode *inode, struct file *filp) -{ - struct ib_uobject *uobj = filp->private_data; - struct ib_uverbs_completion_event_file *file = container_of( - uobj, struct ib_uverbs_completion_event_file, uobj); - struct ib_uverbs_event *entry, *tmp; - - spin_lock_irq(&file->ev_queue.lock); - list_for_each_entry_safe(entry, tmp, &file->ev_queue.event_list, list) { - if (entry->counter) - list_del(&entry->obj_list); - kfree(entry); - } - file->ev_queue.is_closed = 1; - spin_unlock_irq(&file->ev_queue.lock); - - uverbs_close_fd(filp); - - return 0; -} - const struct file_operations uverbs_event_fops = { .owner = THIS_MODULE, .read = ib_uverbs_comp_event_read, .poll = ib_uverbs_comp_event_poll, - .release = ib_uverbs_comp_event_close, + .release = uverbs_uobject_fd_release, .fasync = ib_uverbs_comp_event_fasync, .llseek = no_llseek, }; diff --git a/drivers/infiniband/core/uverbs_std_types.c b/drivers/infiniband/core/uverbs_std_types.c index 35b2e2c640cc..def038a0fe77 100644 --- a/drivers/infiniband/core/uverbs_std_types.c +++ b/drivers/infiniband/core/uverbs_std_types.c @@ -202,22 +202,29 @@ static int uverbs_free_pd(struct ib_uobject *uobject, return 0; } -static int uverbs_hot_unplug_completion_event_file(struct ib_uobject *uobj, - enum rdma_remove_reason why) +static int +uverbs_completion_event_file_destroy_uobj(struct ib_uobject *uobj, + enum rdma_remove_reason why) { - struct ib_uverbs_completion_event_file *comp_event_file = + struct ib_uverbs_completion_event_file *file = container_of(uobj, struct ib_uverbs_completion_event_file, uobj); - struct ib_uverbs_event_queue *event_queue = &comp_event_file->ev_queue; + struct ib_uverbs_event_queue *event_queue = &file->ev_queue; + struct ib_uverbs_event *entry, *tmp; spin_lock_irq(&event_queue->lock); event_queue->is_closed = 1; spin_unlock_irq(&event_queue->lock); + wake_up_interruptible(&event_queue->poll_wait); + kill_fasync(&event_queue->async_queue, SIGIO, POLL_IN); - if (why == RDMA_REMOVE_DRIVER_REMOVE) { - wake_up_interruptible(&event_queue->poll_wait); - kill_fasync(&event_queue->async_queue, SIGIO, POLL_IN); + spin_lock_irq(&event_queue->lock); + list_for_each_entry_safe(entry, tmp, &event_queue->event_list, list) { + if (entry->counter) + list_del(&entry->obj_list); + kfree(entry); } + spin_unlock_irq(&event_queue->lock); return 0; }; @@ -230,7 +237,7 @@ EXPORT_SYMBOL(uverbs_destroy_def_handler); DECLARE_UVERBS_NAMED_OBJECT( UVERBS_OBJECT_COMP_CHANNEL, UVERBS_TYPE_ALLOC_FD(sizeof(struct ib_uverbs_completion_event_file), - uverbs_hot_unplug_completion_event_file, + uverbs_completion_event_file_destroy_uobj, &uverbs_event_fops, "[infinibandevent]", O_RDONLY)); diff --git a/drivers/infiniband/core/uverbs_uapi.c b/drivers/infiniband/core/uverbs_uapi.c index 00c547887132..9b84a126187a 100644 --- a/drivers/infiniband/core/uverbs_uapi.c +++ b/drivers/infiniband/core/uverbs_uapi.c @@ -195,9 +195,9 @@ static int uapi_merge_obj_tree(struct uverbs_api *uapi, * disassociation, and the FD types require the driver to use * struct file_operations.owner to prevent the driver module * code from unloading while the file is open. This provides - * enough safety that uverbs_close_fd() will continue to work. - * Drivers using FD are responsible to handle disassociation of - * the device on their own. + * enough safety that uverbs_uobject_fd_release() will + * continue to work. Drivers using FD are responsible to + * handle disassociation of the device on their own. */ if (WARN_ON(is_driver && obj->type_attrs->type_class != &uverbs_idr_class && diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c index 968fff018fc0..02125d82b96d 100644 --- a/drivers/infiniband/hw/mlx5/devx.c +++ b/drivers/infiniband/hw/mlx5/devx.c @@ -2328,6 +2328,9 @@ static int deliver_event(struct devx_event_subscription *event_sub, return 0; } + /* is_destroyed is ignored here because we don't have any memory + * allocation to clean up for the omit_data case + */ list_add_tail(&event_sub->event_list, &ev_file->event_list); spin_unlock_irqrestore(&ev_file->lock, flags); wake_up_interruptible(&ev_file->poll_wait); @@ -2347,7 +2350,10 @@ static int deliver_event(struct devx_event_subscription *event_sub, memcpy(event_data->hdr.out_data, data, sizeof(struct mlx5_eqe)); spin_lock_irqsave(&ev_file->lock, flags); - list_add_tail(&event_data->list, &ev_file->event_list); + if (!ev_file->is_destroyed) + list_add_tail(&event_data->list, &ev_file->event_list); + else + kfree(event_data); spin_unlock_irqrestore(&ev_file->lock, flags); wake_up_interruptible(&ev_file->poll_wait); @@ -2501,23 +2507,6 @@ static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf, return ret; } -static int devx_async_cmd_event_close(struct inode *inode, struct file *filp) -{ - struct ib_uobject *uobj = filp->private_data; - struct devx_async_cmd_event_file *comp_ev_file = container_of( - uobj, struct devx_async_cmd_event_file, uobj); - struct devx_async_data *entry, *tmp; - - spin_lock_irq(&comp_ev_file->ev_queue.lock); - list_for_each_entry_safe(entry, tmp, - &comp_ev_file->ev_queue.event_list, list) - kvfree(entry); - spin_unlock_irq(&comp_ev_file->ev_queue.lock); - - uverbs_close_fd(filp); - return 0; -} - static __poll_t devx_async_cmd_event_poll(struct file *filp, struct poll_table_struct *wait) { @@ -2541,7 +2530,7 @@ static const struct file_operations devx_async_cmd_event_fops = { .owner = THIS_MODULE, .read = devx_async_cmd_event_read, .poll = devx_async_cmd_event_poll, - .release = devx_async_cmd_event_close, + .release = uverbs_uobject_fd_release, .llseek = no_llseek, }; @@ -2656,78 +2645,74 @@ static void devx_free_subscription(struct rcu_head *rcu) kfree(event_sub); } -static int devx_async_event_close(struct inode *inode, struct file *filp) -{ - struct devx_async_event_file *ev_file = filp->private_data; - struct devx_event_subscription *event_sub, *event_sub_tmp; - struct devx_async_event_data *entry, *tmp; - struct mlx5_ib_dev *dev = ev_file->dev; - - mutex_lock(&dev->devx_event_table.event_xa_lock); - /* delete the subscriptions which are related to this FD */ - list_for_each_entry_safe(event_sub, event_sub_tmp, - &ev_file->subscribed_events_list, file_list) { - devx_cleanup_subscription(dev, event_sub); - list_del_rcu(&event_sub->file_list); - /* subscription may not be used by the read API any more */ - call_rcu(&event_sub->rcu, devx_free_subscription); - } - - mutex_unlock(&dev->devx_event_table.event_xa_lock); - - /* free the pending events allocation */ - if (!ev_file->omit_data) { - spin_lock_irq(&ev_file->lock); - list_for_each_entry_safe(entry, tmp, - &ev_file->event_list, list) - kfree(entry); /* read can't come any more */ - spin_unlock_irq(&ev_file->lock); - } - - uverbs_close_fd(filp); - put_device(&dev->ib_dev.dev); - return 0; -} - static const struct file_operations devx_async_event_fops = { .owner = THIS_MODULE, .read = devx_async_event_read, .poll = devx_async_event_poll, - .release = devx_async_event_close, + .release = uverbs_uobject_fd_release, .llseek = no_llseek, }; -static int devx_hot_unplug_async_cmd_event_file(struct ib_uobject *uobj, - enum rdma_remove_reason why) +static int devx_async_cmd_event_destroy_uobj(struct ib_uobject *uobj, + enum rdma_remove_reason why) { struct devx_async_cmd_event_file *comp_ev_file = container_of(uobj, struct devx_async_cmd_event_file, uobj); struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue; + struct devx_async_data *entry, *tmp; spin_lock_irq(&ev_queue->lock); ev_queue->is_destroyed = 1; spin_unlock_irq(&ev_queue->lock); - - if (why == RDMA_REMOVE_DRIVER_REMOVE) - wake_up_interruptible(&ev_queue->poll_wait); + wake_up_interruptible(&ev_queue->poll_wait); mlx5_cmd_cleanup_async_ctx(&comp_ev_file->async_ctx); + + spin_lock_irq(&comp_ev_file->ev_queue.lock); + list_for_each_entry_safe(entry, tmp, + &comp_ev_file->ev_queue.event_list, list) + kvfree(entry); + spin_unlock_irq(&comp_ev_file->ev_queue.lock); return 0; }; -static int devx_hot_unplug_async_event_file(struct ib_uobject *uobj, - enum rdma_remove_reason why) +static int devx_async_event_destroy_uobj(struct ib_uobject *uobj, + enum rdma_remove_reason why) { struct devx_async_event_file *ev_file = container_of(uobj, struct devx_async_event_file, uobj); + struct devx_event_subscription *event_sub, *event_sub_tmp; + struct devx_async_event_data *entry, *tmp; + struct mlx5_ib_dev *dev = ev_file->dev; spin_lock_irq(&ev_file->lock); ev_file->is_destroyed = 1; spin_unlock_irq(&ev_file->lock); - wake_up_interruptible(&ev_file->poll_wait); + + mutex_lock(&dev->devx_event_table.event_xa_lock); + /* delete the subscriptions which are related to this FD */ + list_for_each_entry_safe(event_sub, event_sub_tmp, + &ev_file->subscribed_events_list, file_list) { + devx_cleanup_subscription(dev, event_sub); + list_del_rcu(&event_sub->file_list); + /* subscription may not be used by the read API any more */ + call_rcu(&event_sub->rcu, devx_free_subscription); + } + mutex_unlock(&dev->devx_event_table.event_xa_lock); + + /* free the pending events allocation */ + if (!ev_file->omit_data) { + spin_lock_irq(&ev_file->lock); + list_for_each_entry_safe(entry, tmp, + &ev_file->event_list, list) + kfree(entry); /* read can't come any more */ + spin_unlock_irq(&ev_file->lock); + } + + put_device(&dev->ib_dev.dev); return 0; }; @@ -2913,7 +2898,7 @@ DECLARE_UVERBS_NAMED_METHOD( DECLARE_UVERBS_NAMED_OBJECT( MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD, UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_cmd_event_file), - devx_hot_unplug_async_cmd_event_file, + devx_async_cmd_event_destroy_uobj, &devx_async_cmd_event_fops, "[devx_async_cmd]", O_RDONLY), &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)); @@ -2931,7 +2916,7 @@ DECLARE_UVERBS_NAMED_METHOD( DECLARE_UVERBS_NAMED_OBJECT( MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD, UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_event_file), - devx_hot_unplug_async_event_file, + devx_async_event_destroy_uobj, &devx_async_event_fops, "[devx_async_event]", O_RDONLY), &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC)); diff --git a/include/rdma/uverbs_types.h b/include/rdma/uverbs_types.h index ca65d4704b4b..b633fa2543e3 100644 --- a/include/rdma/uverbs_types.h +++ b/include/rdma/uverbs_types.h @@ -161,11 +161,11 @@ struct uverbs_obj_fd_type { * In fd based objects, uverbs_obj_type_ops points to generic * fd operations. In order to specialize the underlying types (e.g. * completion_channel), we use fops, name and flags for fd creation. - * context_closed is called when the context is closed either when - * the driver is removed or the process terminated. + * destroy_object is called when the uobject is to be destroyed, + * because the driver is removed or the FD is closed. */ struct uverbs_obj_type type; - int (*context_closed)(struct ib_uobject *uobj, + int (*destroy_object)(struct ib_uobject *uobj, enum rdma_remove_reason why); const struct file_operations *fops; const char *name; @@ -174,11 +174,11 @@ struct uverbs_obj_fd_type { extern const struct uverbs_obj_type_class uverbs_idr_class; extern const struct uverbs_obj_type_class uverbs_fd_class; -void uverbs_close_fd(struct file *f); +int uverbs_uobject_fd_release(struct inode *inode, struct file *filp); #define UVERBS_BUILD_BUG_ON(cond) (sizeof(char[1 - 2 * !!(cond)]) - \ sizeof(char)) -#define UVERBS_TYPE_ALLOC_FD(_obj_size, _context_closed, _fops, _name, _flags)\ +#define UVERBS_TYPE_ALLOC_FD(_obj_size, _destroy_object, _fops, _name, _flags) \ ((&((const struct uverbs_obj_fd_type) \ {.type = { \ .type_class = &uverbs_fd_class, \ @@ -186,7 +186,7 @@ void uverbs_close_fd(struct file *f); UVERBS_BUILD_BUG_ON((_obj_size) < \ sizeof(struct ib_uobject)), \ }, \ - .context_closed = _context_closed, \ + .destroy_object = _destroy_object, \ .fops = _fops, \ .name = _name, \ .flags = _flags}))->type) -- cgit v1.2.3-59-g8ed1b From 849e149063bd10eb6211c14617491a0bc9516c2f Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 8 Jan 2020 19:21:56 +0200 Subject: RDMA/core: Do not allow alloc_commit to fail This is a left over from an earlier version that creates a lot of complexity for error unwind, particularly for FD uobjects. The only reason this was done is so that anon_inode_get_file() could be called with the final fops and a fully setup uobject. Both need to be setup since unwinding anon_inode_get_file() via fput will call the driver's release(). Now that the driver does not provide release, we no longer need to worry about this complicated sequence, simply create the struct file at the start and allow the core code's release function to deal with the abort case. This allows all the confusing error paths around commit to be removed. Link: https://lore.kernel.org/r/1578504126-9400-5-git-send-email-yishaih@mellanox.com Signed-off-by: Yishai Hadas Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/rdma_core.c | 118 ++++++++++++++++----------------- drivers/infiniband/core/rdma_core.h | 21 +----- drivers/infiniband/core/uverbs_cmd.c | 40 ++++++----- drivers/infiniband/core/uverbs_ioctl.c | 45 ++++--------- include/rdma/uverbs_std_types.h | 10 --- include/rdma/uverbs_types.h | 6 +- 6 files changed, 99 insertions(+), 141 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/rdma_core.c b/drivers/infiniband/core/rdma_core.c index 0ed0341b8e30..a9f5263c9559 100644 --- a/drivers/infiniband/core/rdma_core.c +++ b/drivers/infiniband/core/rdma_core.c @@ -130,7 +130,11 @@ static int uverbs_destroy_uobject(struct ib_uobject *uobj, lockdep_assert_held(&ufile->hw_destroy_rwsem); assert_uverbs_usecnt(uobj, UVERBS_LOOKUP_WRITE); - if (uobj->object) { + if (reason == RDMA_REMOVE_ABORT) { + WARN_ON(!list_empty(&uobj->list)); + WARN_ON(!uobj->context); + uobj->uapi_object->type_class->alloc_abort(uobj); + } else if (uobj->object) { ret = uobj->uapi_object->type_class->destroy_hw(uobj, reason, attrs); if (ret) { @@ -146,12 +150,6 @@ static int uverbs_destroy_uobject(struct ib_uobject *uobj, uobj->object = NULL; } - if (reason == RDMA_REMOVE_ABORT) { - WARN_ON(!list_empty(&uobj->list)); - WARN_ON(!uobj->context); - uobj->uapi_object->type_class->alloc_abort(uobj); - } - uobj->context = NULL; /* @@ -450,22 +448,40 @@ static struct ib_uobject * alloc_begin_fd_uobject(const struct uverbs_api_object *obj, struct ib_uverbs_file *ufile) { + const struct uverbs_obj_fd_type *fd_type = + container_of(obj->type_attrs, struct uverbs_obj_fd_type, type); int new_fd; struct ib_uobject *uobj; + struct file *filp; + + if (WARN_ON(fd_type->fops->release != &uverbs_uobject_fd_release)) + return ERR_PTR(-EINVAL); new_fd = get_unused_fd_flags(O_CLOEXEC); if (new_fd < 0) return ERR_PTR(new_fd); uobj = alloc_uobj(ufile, obj); - if (IS_ERR(uobj)) { - put_unused_fd(new_fd); - return uobj; + if (IS_ERR(uobj)) + goto err_fd; + + /* Note that uverbs_uobject_fd_release() is called during abort */ + filp = anon_inode_getfile(fd_type->name, fd_type->fops, NULL, + fd_type->flags); + if (IS_ERR(filp)) { + uobj = ERR_CAST(filp); + goto err_uobj; } + uobj->object = filp; uobj->id = new_fd; uobj->ufile = ufile; + return uobj; +err_uobj: + uverbs_uobject_put(uobj); +err_fd: + put_unused_fd(new_fd); return uobj; } @@ -539,6 +555,9 @@ static void remove_handle_idr_uobject(struct ib_uobject *uobj) static void alloc_abort_fd_uobject(struct ib_uobject *uobj) { + struct file *filp = uobj->object; + + fput(filp); put_unused_fd(uobj->id); } @@ -560,7 +579,7 @@ static void remove_handle_fd_uobject(struct ib_uobject *uobj) { } -static int alloc_commit_idr_uobject(struct ib_uobject *uobj) +static void alloc_commit_idr_uobject(struct ib_uobject *uobj) { struct ib_uverbs_file *ufile = uobj->ufile; void *old; @@ -574,31 +593,12 @@ static int alloc_commit_idr_uobject(struct ib_uobject *uobj) */ old = xa_store(&ufile->idr, uobj->id, uobj, GFP_KERNEL); WARN_ON(old != NULL); - - return 0; } -static int alloc_commit_fd_uobject(struct ib_uobject *uobj) +static void alloc_commit_fd_uobject(struct ib_uobject *uobj) { - const struct uverbs_obj_fd_type *fd_type = container_of( - uobj->uapi_object->type_attrs, struct uverbs_obj_fd_type, type); int fd = uobj->id; - struct file *filp; - - /* - * The kref for uobj is moved into filp->private data and put in - * uverbs_close_fd(). Once alloc_commit() succeeds - * uverbs_uobject_fd_release() must be guaranteed to be called from - * the provided fops release callback. - */ - filp = anon_inode_getfile(fd_type->name, - fd_type->fops, - uobj, - fd_type->flags); - if (IS_ERR(filp)) - return PTR_ERR(filp); - - uobj->object = filp; + struct file *filp = uobj->object; /* Matching put will be done in uverbs_uobject_fd_release() */ kref_get(&uobj->ufile->ref); @@ -610,9 +610,8 @@ static int alloc_commit_fd_uobject(struct ib_uobject *uobj) * NOTE: Once we install the file we loose ownership of our kref on * uobj. It will be put by uverbs_uobject_fd_release() */ + filp->private_data = uobj; fd_install(fd, filp); - - return 0; } /* @@ -620,19 +619,13 @@ static int alloc_commit_fd_uobject(struct ib_uobject *uobj) * caller can no longer assume uobj is valid. If this function fails it * destroys the uboject, including the attached HW object. */ -int __must_check rdma_alloc_commit_uobject(struct ib_uobject *uobj, - struct uverbs_attr_bundle *attrs) +void rdma_alloc_commit_uobject(struct ib_uobject *uobj, + struct uverbs_attr_bundle *attrs) { struct ib_uverbs_file *ufile = attrs->ufile; - int ret; /* alloc_commit consumes the uobj kref */ - ret = uobj->uapi_object->type_class->alloc_commit(uobj); - if (ret) { - uverbs_destroy_uobject(uobj, RDMA_REMOVE_ABORT, attrs); - up_read(&ufile->hw_destroy_rwsem); - return ret; - } + uobj->uapi_object->type_class->alloc_commit(uobj); /* kref is held so long as the uobj is on the uobj list. */ uverbs_uobject_get(uobj); @@ -645,8 +638,6 @@ int __must_check rdma_alloc_commit_uobject(struct ib_uobject *uobj, /* Matches the down_read in rdma_alloc_begin_uobject */ up_read(&ufile->hw_destroy_rwsem); - - return 0; } /* @@ -658,7 +649,6 @@ void rdma_alloc_abort_uobject(struct ib_uobject *uobj, { struct ib_uverbs_file *ufile = uobj->ufile; - uobj->object = NULL; uverbs_destroy_uobject(uobj, RDMA_REMOVE_ABORT, attrs); /* Matches the down_read in rdma_alloc_begin_uobject */ @@ -751,14 +741,23 @@ EXPORT_SYMBOL(uverbs_idr_class); */ int uverbs_uobject_fd_release(struct inode *inode, struct file *filp) { - struct ib_uobject *uobj = filp->private_data; - struct ib_uverbs_file *ufile = uobj->ufile; - struct uverbs_attr_bundle attrs = { - .context = uobj->context, - .ufile = ufile, - }; + struct ib_uverbs_file *ufile; + struct ib_uobject *uobj; + + /* + * This can only happen if the fput came from alloc_abort_fd_uobject() + */ + if (!filp->private_data) + return 0; + uobj = filp->private_data; + ufile = uobj->ufile; if (down_read_trylock(&ufile->hw_destroy_rwsem)) { + struct uverbs_attr_bundle attrs = { + .context = uobj->context, + .ufile = ufile, + }; + /* * lookup_get_fd_uobject holds the kref on the struct file any * time a FD uobj is locked, which prevents this release @@ -770,7 +769,7 @@ int uverbs_uobject_fd_release(struct inode *inode, struct file *filp) up_read(&ufile->hw_destroy_rwsem); } - /* Matches the get in alloc_begin_fd_uobject */ + /* Matches the get in alloc_commit_fd_uobject() */ kref_put(&ufile->ref, ib_uverbs_release_file); /* Pairs with filp->private_data in alloc_begin_fd_uobject */ @@ -938,12 +937,10 @@ uverbs_get_uobject_from_file(u16 object_id, enum uverbs_obj_access access, } } -int uverbs_finalize_object(struct ib_uobject *uobj, - enum uverbs_obj_access access, bool commit, - struct uverbs_attr_bundle *attrs) +void uverbs_finalize_object(struct ib_uobject *uobj, + enum uverbs_obj_access access, bool commit, + struct uverbs_attr_bundle *attrs) { - int ret = 0; - /* * refcounts should be handled at the object level and not at the * uobject level. Refcounts of the objects themselves are done in @@ -963,14 +960,11 @@ int uverbs_finalize_object(struct ib_uobject *uobj, break; case UVERBS_ACCESS_NEW: if (commit) - ret = rdma_alloc_commit_uobject(uobj, attrs); + rdma_alloc_commit_uobject(uobj, attrs); else rdma_alloc_abort_uobject(uobj, attrs); break; default: WARN_ON(true); - ret = -EOPNOTSUPP; } - - return ret; } diff --git a/drivers/infiniband/core/rdma_core.h b/drivers/infiniband/core/rdma_core.h index 92694253e776..29f905e8c2a8 100644 --- a/drivers/infiniband/core/rdma_core.h +++ b/drivers/infiniband/core/rdma_core.h @@ -63,24 +63,9 @@ struct ib_uobject * uverbs_get_uobject_from_file(u16 object_id, enum uverbs_obj_access access, s64 id, struct uverbs_attr_bundle *attrs); -/* - * Note that certain finalize stages could return a status: - * (a) alloc_commit could return a failure if the object is committed at the - * same time when the context is destroyed. - * (b) remove_commit could fail if the object wasn't destroyed successfully. - * Since multiple objects could be finalized in one transaction, it is very NOT - * recommended to have several finalize actions which have side effects. - * For example, it's NOT recommended to have a certain action which has both - * a commit action and a destroy action or two destroy objects in the same - * action. The rule of thumb is to have one destroy or commit action with - * multiple lookups. - * The first non zero return value of finalize_object is returned from this - * function. For example, this could happen when we couldn't destroy an - * object. - */ -int uverbs_finalize_object(struct ib_uobject *uobj, - enum uverbs_obj_access access, bool commit, - struct uverbs_attr_bundle *attrs); +void uverbs_finalize_object(struct ib_uobject *uobj, + enum uverbs_obj_access access, bool commit, + struct uverbs_attr_bundle *attrs); int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx); diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index 06ed32c8662f..74f6ae475ca0 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -446,7 +446,8 @@ static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs) if (ret) goto err_copy; - return uobj_alloc_commit(uobj, attrs); + rdma_alloc_commit_uobject(uobj, attrs); + return 0; err_copy: ib_dealloc_pd_user(pd, uverbs_get_cleared_udata(attrs)); @@ -642,7 +643,8 @@ static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs) mutex_unlock(&ibudev->xrcd_tree_mutex); - return uobj_alloc_commit(&obj->uobject, attrs); + rdma_alloc_commit_uobject(&obj->uobject, attrs); + return 0; err_copy: if (inode) { @@ -774,7 +776,8 @@ static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs) uobj_put_obj_read(pd); - return uobj_alloc_commit(uobj, attrs); + rdma_alloc_commit_uobject(uobj, attrs); + return 0; err_copy: ib_dereg_mr_user(mr, uverbs_get_cleared_udata(attrs)); @@ -928,7 +931,8 @@ static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs) goto err_copy; uobj_put_obj_read(pd); - return uobj_alloc_commit(uobj, attrs); + rdma_alloc_commit_uobject(uobj, attrs); + return 0; err_copy: uverbs_dealloc_mw(mw); @@ -980,7 +984,8 @@ static int ib_uverbs_create_comp_channel(struct uverbs_attr_bundle *attrs) return ret; } - return uobj_alloc_commit(uobj, attrs); + rdma_alloc_commit_uobject(uobj, attrs); + return 0; } static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs, @@ -1049,9 +1054,7 @@ static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs, if (ret) goto err_cb; - ret = uobj_alloc_commit(&obj->uobject, attrs); - if (ret) - return ERR_PTR(ret); + rdma_alloc_commit_uobject(&obj->uobject, attrs); return obj; err_cb: @@ -1491,7 +1494,8 @@ static int create_qp(struct uverbs_attr_bundle *attrs, if (ind_tbl) uobj_put_obj_read(ind_tbl); - return uobj_alloc_commit(&obj->uevent.uobject, attrs); + rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs); + return 0; err_cb: ib_destroy_qp_user(qp, uverbs_get_cleared_udata(attrs)); @@ -1623,7 +1627,8 @@ static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs) qp->uobject = &obj->uevent.uobject; uobj_put_read(xrcd_uobj); - return uobj_alloc_commit(&obj->uevent.uobject, attrs); + rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs); + return 0; err_destroy: ib_destroy_qp_user(qp, uverbs_get_cleared_udata(attrs)); @@ -2465,7 +2470,8 @@ static int ib_uverbs_create_ah(struct uverbs_attr_bundle *attrs) goto err_copy; uobj_put_obj_read(pd); - return uobj_alloc_commit(uobj, attrs); + rdma_alloc_commit_uobject(uobj, attrs); + return 0; err_copy: rdma_destroy_ah_user(ah, RDMA_DESTROY_AH_SLEEPABLE, @@ -2977,7 +2983,8 @@ static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs) uobj_put_obj_read(pd); uobj_put_obj_read(cq); - return uobj_alloc_commit(&obj->uevent.uobject, attrs); + rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs); + return 0; err_copy: ib_destroy_wq(wq, uverbs_get_cleared_udata(attrs)); @@ -3151,7 +3158,8 @@ static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs) for (j = 0; j < num_read_wqs; j++) uobj_put_obj_read(wqs[j]); - return uobj_alloc_commit(uobj, attrs); + rdma_alloc_commit_uobject(uobj, attrs); + return 0; err_copy: ib_destroy_rwq_ind_table(rwq_ind_tbl); @@ -3329,7 +3337,8 @@ static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs) kfree(flow_attr); if (cmd.flow_attr.num_of_specs) kfree(kern_flow_attr); - return uobj_alloc_commit(uobj, attrs); + rdma_alloc_commit_uobject(uobj, attrs); + return 0; err_copy: if (!qp->device->ops.destroy_flow(flow_id)) atomic_dec(&qp->usecnt); @@ -3477,7 +3486,8 @@ static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs, uobj_put_obj_read(attr.ext.cq); uobj_put_obj_read(pd); - return uobj_alloc_commit(&obj->uevent.uobject, attrs); + rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs); + return 0; err_copy: ib_destroy_srq_user(srq, uverbs_get_cleared_udata(attrs)); diff --git a/drivers/infiniband/core/uverbs_ioctl.c b/drivers/infiniband/core/uverbs_ioctl.c index 269938f59d3f..538affbc517e 100644 --- a/drivers/infiniband/core/uverbs_ioctl.c +++ b/drivers/infiniband/core/uverbs_ioctl.c @@ -220,24 +220,17 @@ static int uverbs_process_idrs_array(struct bundle_priv *pbundle, return ret; } -static int uverbs_free_idrs_array(const struct uverbs_api_attr *attr_uapi, - struct uverbs_objs_arr_attr *attr, - bool commit, struct uverbs_attr_bundle *attrs) +static void uverbs_free_idrs_array(const struct uverbs_api_attr *attr_uapi, + struct uverbs_objs_arr_attr *attr, + bool commit, + struct uverbs_attr_bundle *attrs) { const struct uverbs_attr_spec *spec = &attr_uapi->spec; - int current_ret; - int ret = 0; size_t i; - for (i = 0; i != attr->len; i++) { - current_ret = uverbs_finalize_object(attr->uobjects[i], - spec->u2.objs_arr.access, - commit, attrs); - if (!ret) - ret = current_ret; - } - - return ret; + for (i = 0; i != attr->len; i++) + uverbs_finalize_object(attr->uobjects[i], + spec->u2.objs_arr.access, commit, attrs); } static int uverbs_process_attr(struct bundle_priv *pbundle, @@ -495,26 +488,22 @@ static int ib_uverbs_run_method(struct bundle_priv *pbundle, return ret; } -static int bundle_destroy(struct bundle_priv *pbundle, bool commit) +static void bundle_destroy(struct bundle_priv *pbundle, bool commit) { unsigned int key_bitmap_len = pbundle->method_elm->key_bitmap_len; struct bundle_alloc_head *memblock; unsigned int i; - int ret = 0; /* fast path for simple uobjects */ i = -1; while ((i = find_next_bit(pbundle->uobj_finalize, key_bitmap_len, i + 1)) < key_bitmap_len) { struct uverbs_attr *attr = &pbundle->bundle.attrs[i]; - int current_ret; - current_ret = uverbs_finalize_object( + uverbs_finalize_object( attr->obj_attr.uobject, attr->obj_attr.attr_elm->spec.u.obj.access, commit, &pbundle->bundle); - if (!ret) - ret = current_ret; } i = -1; @@ -523,7 +512,6 @@ static int bundle_destroy(struct bundle_priv *pbundle, bool commit) struct uverbs_attr *attr = &pbundle->bundle.attrs[i]; const struct uverbs_api_attr *attr_uapi; void __rcu **slot; - int current_ret; slot = uapi_get_attr_for_method( pbundle, @@ -534,11 +522,8 @@ static int bundle_destroy(struct bundle_priv *pbundle, bool commit) attr_uapi = rcu_dereference_protected(*slot, true); if (attr_uapi->spec.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) { - current_ret = uverbs_free_idrs_array( - attr_uapi, &attr->objs_arr_attr, commit, - &pbundle->bundle); - if (!ret) - ret = current_ret; + uverbs_free_idrs_array(attr_uapi, &attr->objs_arr_attr, + commit, &pbundle->bundle); } } @@ -548,8 +533,6 @@ static int bundle_destroy(struct bundle_priv *pbundle, bool commit) memblock = memblock->next; kvfree(tmp); } - - return ret; } static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile, @@ -562,7 +545,6 @@ static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile, struct bundle_priv *pbundle; struct bundle_priv onstack; void __rcu **slot; - int destroy_ret; int ret; if (unlikely(hdr->driver_id != uapi->driver_id)) @@ -610,10 +592,7 @@ static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile, memset(pbundle->spec_finalize, 0, sizeof(pbundle->spec_finalize)); ret = ib_uverbs_run_method(pbundle, hdr->num_attrs); - destroy_ret = bundle_destroy(pbundle, ret == 0); - if (unlikely(destroy_ret && !ret)) - return destroy_ret; - + bundle_destroy(pbundle, ret == 0); return ret; } diff --git a/include/rdma/uverbs_std_types.h b/include/rdma/uverbs_std_types.h index 05eabfd5d0d3..c6bcaad4a8e0 100644 --- a/include/rdma/uverbs_std_types.h +++ b/include/rdma/uverbs_std_types.h @@ -104,16 +104,6 @@ static inline void uobj_put_write(struct ib_uobject *uobj) rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_WRITE); } -static inline int __must_check -uobj_alloc_commit(struct ib_uobject *uobj, struct uverbs_attr_bundle *attrs) -{ - int ret = rdma_alloc_commit_uobject(uobj, attrs); - - if (ret) - return ret; - return 0; -} - static inline void uobj_alloc_abort(struct ib_uobject *uobj, struct uverbs_attr_bundle *attrs) { diff --git a/include/rdma/uverbs_types.h b/include/rdma/uverbs_types.h index b633fa2543e3..01d59df4e9e7 100644 --- a/include/rdma/uverbs_types.h +++ b/include/rdma/uverbs_types.h @@ -85,7 +85,7 @@ struct uverbs_obj_type_class { struct ib_uobject *(*alloc_begin)(const struct uverbs_api_object *obj, struct ib_uverbs_file *ufile); /* This consumes the kref on uobj */ - int (*alloc_commit)(struct ib_uobject *uobj); + void (*alloc_commit)(struct ib_uobject *uobj); /* This does not consume the kref on uobj */ void (*alloc_abort)(struct ib_uobject *uobj); @@ -141,8 +141,8 @@ struct ib_uobject *rdma_alloc_begin_uobject(const struct uverbs_api_object *obj, struct uverbs_attr_bundle *attrs); void rdma_alloc_abort_uobject(struct ib_uobject *uobj, struct uverbs_attr_bundle *attrs); -int __must_check rdma_alloc_commit_uobject(struct ib_uobject *uobj, - struct uverbs_attr_bundle *attrs); +void rdma_alloc_commit_uobject(struct ib_uobject *uobj, + struct uverbs_attr_bundle *attrs); /* * uverbs_uobject_get is called in order to increase the reference count on -- cgit v1.2.3-59-g8ed1b From 4ec1dcfcdf3756555adb5122c2443ea27741296a Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 8 Jan 2020 19:21:57 +0200 Subject: RDMA/core: Make ib_ucq_object use ib_uevent_object Any uobject that sends events into the async_event_file should be using ib_uevent_object so it can use the standard uevent based helper functions. CQ pushes events into both the async_event and the comp_channel in an open coded way. Move the async events related stuff to ib_uevent_object. Link: https://lore.kernel.org/r/1578504126-9400-6-git-send-email-yishaih@mellanox.com Signed-off-by: Yishai Hadas Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/uverbs.h | 5 ++--- drivers/infiniband/core/uverbs_cmd.c | 20 +++++++++----------- drivers/infiniband/core/uverbs_main.c | 17 ++++++----------- drivers/infiniband/core/uverbs_std_types_cq.c | 18 ++++++++---------- 4 files changed, 25 insertions(+), 35 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h index 63f7f7db5902..9fa044699cf0 100644 --- a/drivers/infiniband/core/uverbs.h +++ b/drivers/infiniband/core/uverbs.h @@ -183,6 +183,7 @@ struct ib_uverbs_mcast_entry { struct ib_uevent_object { struct ib_uobject uobject; + /* List member for ib_uverbs_async_event_file list */ struct list_head event_list; u32 events_reported; }; @@ -210,11 +211,9 @@ struct ib_uwq_object { }; struct ib_ucq_object { - struct ib_uobject uobject; + struct ib_uevent_object uevent; struct list_head comp_list; - struct list_head async_list; u32 comp_events_reported; - u32 async_events_reported; }; extern const struct file_operations uverbs_event_fops; diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index 74f6ae475ca0..3a2a27830ae2 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -1015,11 +1015,9 @@ static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs, } } - obj->uobject.user_handle = cmd->user_handle; - obj->comp_events_reported = 0; - obj->async_events_reported = 0; + obj->uevent.uobject.user_handle = cmd->user_handle; INIT_LIST_HEAD(&obj->comp_list); - INIT_LIST_HEAD(&obj->async_list); + INIT_LIST_HEAD(&obj->uevent.event_list); attr.cqe = cmd->cqe; attr.comp_vector = cmd->comp_vector; @@ -1031,7 +1029,7 @@ static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs, goto err_file; } cq->device = ib_dev; - cq->uobject = &obj->uobject; + cq->uobject = &obj->uevent.uobject; cq->comp_handler = ib_uverbs_comp_handler; cq->event_handler = ib_uverbs_cq_event_handler; cq->cq_context = ev_file ? &ev_file->ev_queue : NULL; @@ -1041,9 +1039,9 @@ static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs, if (ret) goto err_free; - obj->uobject.object = cq; + obj->uevent.uobject.object = cq; memset(&resp, 0, sizeof resp); - resp.base.cq_handle = obj->uobject.id; + resp.base.cq_handle = obj->uevent.uobject.id; resp.base.cqe = cq->cqe; resp.response_length = uverbs_response_length(attrs, sizeof(resp)); @@ -1054,7 +1052,7 @@ static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs, if (ret) goto err_cb; - rdma_alloc_commit_uobject(&obj->uobject, attrs); + rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs); return obj; err_cb: @@ -1067,7 +1065,7 @@ err_file: ib_uverbs_release_ucq(attrs->ufile, ev_file, obj); err: - uobj_alloc_abort(&obj->uobject, attrs); + uobj_alloc_abort(&obj->uevent.uobject, attrs); return ERR_PTR(ret); } @@ -1261,10 +1259,10 @@ static int ib_uverbs_destroy_cq(struct uverbs_attr_bundle *attrs) if (IS_ERR(uobj)) return PTR_ERR(uobj); - obj = container_of(uobj, struct ib_ucq_object, uobject); + obj = container_of(uobj, struct ib_ucq_object, uevent.uobject); memset(&resp, 0, sizeof(resp)); resp.comp_events_reported = obj->comp_events_reported; - resp.async_events_reported = obj->async_events_reported; + resp.async_events_reported = obj->uevent.events_reported; uobj_put_destroy(uobj); diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index da56fda259fd..940bb291a57d 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -150,12 +150,7 @@ void ib_uverbs_release_ucq(struct ib_uverbs_file *file, uverbs_uobject_put(&ev_file->uobj); } - spin_lock_irq(&file->async_file->ev_queue.lock); - list_for_each_entry_safe(evt, tmp, &uobj->async_list, obj_list) { - list_del(&evt->list); - kfree(evt); - } - spin_unlock_irq(&file->async_file->ev_queue.lock); + ib_uverbs_release_uevent(file, &uobj->uevent); } void ib_uverbs_release_uevent(struct ib_uverbs_file *file, @@ -413,7 +408,7 @@ void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context) return; } - uobj = container_of(cq->uobject, struct ib_ucq_object, uobject); + uobj = container_of(cq->uobject, struct ib_ucq_object, uevent.uobject); entry->desc.comp.cq_handle = cq->uobject->user_handle; entry->counter = &uobj->comp_events_reported; @@ -462,12 +457,12 @@ static void ib_uverbs_async_handler(struct ib_uverbs_file *file, void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr) { - struct ib_ucq_object *uobj = container_of(event->element.cq->uobject, - struct ib_ucq_object, uobject); + struct ib_uevent_object *uobj = container_of( + event->element.cq->uobject, struct ib_uevent_object, uobject); ib_uverbs_async_handler(uobj->uobject.ufile, uobj->uobject.user_handle, - event->event, &uobj->async_list, - &uobj->async_events_reported); + event->event, &uobj->event_list, + &uobj->events_reported); } void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr) diff --git a/drivers/infiniband/core/uverbs_std_types_cq.c b/drivers/infiniband/core/uverbs_std_types_cq.c index e39fe6a8aac4..fbc605a95e51 100644 --- a/drivers/infiniband/core/uverbs_std_types_cq.c +++ b/drivers/infiniband/core/uverbs_std_types_cq.c @@ -41,7 +41,7 @@ static int uverbs_free_cq(struct ib_uobject *uobject, struct ib_cq *cq = uobject->object; struct ib_uverbs_event_queue *ev_queue = cq->cq_context; struct ib_ucq_object *ucq = - container_of(uobject, struct ib_ucq_object, uobject); + container_of(uobject, struct ib_ucq_object, uevent.uobject); int ret; ret = ib_destroy_cq_user(cq, &attrs->driver_udata); @@ -63,7 +63,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( { struct ib_ucq_object *obj = container_of( uverbs_attr_get_uobject(attrs, UVERBS_ATTR_CREATE_CQ_HANDLE), - typeof(*obj), uobject); + typeof(*obj), uevent.uobject); struct ib_device *ib_dev = attrs->context->device; int ret; u64 user_handle; @@ -106,10 +106,8 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( goto err_event_file; } - obj->comp_events_reported = 0; - obj->async_events_reported = 0; INIT_LIST_HEAD(&obj->comp_list); - INIT_LIST_HEAD(&obj->async_list); + INIT_LIST_HEAD(&obj->uevent.event_list); cq = rdma_zalloc_drv_obj(ib_dev, ib_cq); if (!cq) { @@ -118,7 +116,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( } cq->device = ib_dev; - cq->uobject = &obj->uobject; + cq->uobject = &obj->uevent.uobject; cq->comp_handler = ib_uverbs_comp_handler; cq->event_handler = ib_uverbs_cq_event_handler; cq->cq_context = ev_file ? &ev_file->ev_queue : NULL; @@ -129,8 +127,8 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( if (ret) goto err_free; - obj->uobject.object = cq; - obj->uobject.user_handle = user_handle; + obj->uevent.uobject.object = cq; + obj->uevent.uobject.user_handle = user_handle; rdma_restrack_uadd(&cq->res); ret = uverbs_copy_to(attrs, UVERBS_ATTR_CREATE_CQ_RESP_CQE, &cq->cqe, @@ -182,10 +180,10 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_DESTROY)( struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, UVERBS_ATTR_DESTROY_CQ_HANDLE); struct ib_ucq_object *obj = - container_of(uobj, struct ib_ucq_object, uobject); + container_of(uobj, struct ib_ucq_object, uevent.uobject); struct ib_uverbs_destroy_cq_resp resp = { .comp_events_reported = obj->comp_events_reported, - .async_events_reported = obj->async_events_reported + .async_events_reported = obj->uevent.events_reported }; return uverbs_copy_to(attrs, UVERBS_ATTR_DESTROY_CQ_RESP, &resp, -- cgit v1.2.3-59-g8ed1b From 5bd48c18c8cea0154800b40ec75201fa71684312 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 8 Jan 2020 19:21:58 +0200 Subject: RDMA/core: Do not erase the type of ib_cq.uobject This is a struct ib_ucq_object pointer, instead of using container_of() all over the place just store it with its actual type. Link: https://lore.kernel.org/r/1578504126-9400-7-git-send-email-yishaih@mellanox.com Signed-off-by: Yishai Hadas Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/nldev.c | 3 +- drivers/infiniband/core/uverbs_cmd.c | 40 +++++++++++++++++---------- drivers/infiniband/core/uverbs_main.c | 7 ++--- drivers/infiniband/core/uverbs_std_types_cq.c | 2 +- include/rdma/ib_verbs.h | 4 ++- 5 files changed, 34 insertions(+), 22 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c index cbf6041a5d4a..37b433aa7306 100644 --- a/drivers/infiniband/core/nldev.c +++ b/drivers/infiniband/core/nldev.c @@ -41,6 +41,7 @@ #include "core_priv.h" #include "cma_priv.h" #include "restrack.h" +#include "uverbs.h" typedef int (*res_fill_func_t)(struct sk_buff*, bool, struct rdma_restrack_entry*, uint32_t); @@ -599,7 +600,7 @@ static int fill_res_cq_entry(struct sk_buff *msg, bool has_cap_net_admin, goto err; if (!rdma_is_kernel_res(res) && nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN, - cq->uobject->context->res.id)) + cq->uobject->uevent.uobject.context->res.id)) goto err; if (fill_res_name_pid(msg, res)) diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index 3a2a27830ae2..b08679a5da25 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -1029,7 +1029,7 @@ static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs, goto err_file; } cq->device = ib_dev; - cq->uobject = &obj->uevent.uobject; + cq->uobject = obj; cq->comp_handler = ib_uverbs_comp_handler; cq->event_handler = ib_uverbs_cq_event_handler; cq->cq_context = ev_file ? &ev_file->ev_queue : NULL; @@ -1134,7 +1134,8 @@ static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs) ret = uverbs_response(attrs, &resp, sizeof(resp)); out: - uobj_put_obj_read(cq); + rdma_lookup_put_uobject(&cq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); return ret; } @@ -1217,7 +1218,8 @@ static int ib_uverbs_poll_cq(struct uverbs_attr_bundle *attrs) ret = uverbs_output_written(attrs, UVERBS_ATTR_CORE_OUT); out_put: - uobj_put_obj_read(cq); + rdma_lookup_put_uobject(&cq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); return ret; } @@ -1238,8 +1240,8 @@ static int ib_uverbs_req_notify_cq(struct uverbs_attr_bundle *attrs) ib_req_notify_cq(cq, cmd.solicited_only ? IB_CQ_SOLICITED : IB_CQ_NEXT_COMP); - uobj_put_obj_read(cq); - + rdma_lookup_put_uobject(&cq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); return 0; } @@ -1484,9 +1486,11 @@ static int create_qp(struct uverbs_attr_bundle *attrs, if (pd) uobj_put_obj_read(pd); if (scq) - uobj_put_obj_read(scq); + rdma_lookup_put_uobject(&scq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); if (rcq && rcq != scq) - uobj_put_obj_read(rcq); + rdma_lookup_put_uobject(&rcq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); if (srq) uobj_put_obj_read(srq); if (ind_tbl) @@ -1503,9 +1507,11 @@ err_put: if (pd) uobj_put_obj_read(pd); if (scq) - uobj_put_obj_read(scq); + rdma_lookup_put_uobject(&scq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); if (rcq && rcq != scq) - uobj_put_obj_read(rcq); + rdma_lookup_put_uobject(&rcq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); if (srq) uobj_put_obj_read(srq); if (ind_tbl) @@ -2980,14 +2986,16 @@ static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs) goto err_copy; uobj_put_obj_read(pd); - uobj_put_obj_read(cq); + rdma_lookup_put_uobject(&cq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs); return 0; err_copy: ib_destroy_wq(wq, uverbs_get_cleared_udata(attrs)); err_put_cq: - uobj_put_obj_read(cq); + rdma_lookup_put_uobject(&cq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); err_put_pd: uobj_put_obj_read(pd); err_uobj: @@ -3481,7 +3489,8 @@ static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs, uobj_put_read(xrcd_uobj); if (ib_srq_has_cq(cmd->srq_type)) - uobj_put_obj_read(attr.ext.cq); + rdma_lookup_put_uobject(&attr.ext.cq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); uobj_put_obj_read(pd); rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs); @@ -3498,7 +3507,8 @@ err_put: err_put_cq: if (ib_srq_has_cq(cmd->srq_type)) - uobj_put_obj_read(attr.ext.cq); + rdma_lookup_put_uobject(&attr.ext.cq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); err_put_xrcd: if (cmd->srq_type == IB_SRQT_XRC) { @@ -3714,8 +3724,8 @@ static int ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle *attrs) ret = rdma_set_cq_moderation(cq, cmd.attr.cq_count, cmd.attr.cq_period); - uobj_put_obj_read(cq); - + rdma_lookup_put_uobject(&cq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); return ret; } diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index 940bb291a57d..2849bf40bc97 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -408,9 +408,9 @@ void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context) return; } - uobj = container_of(cq->uobject, struct ib_ucq_object, uevent.uobject); + uobj = cq->uobject; - entry->desc.comp.cq_handle = cq->uobject->user_handle; + entry->desc.comp.cq_handle = cq->uobject->uevent.uobject.user_handle; entry->counter = &uobj->comp_events_reported; list_add_tail(&entry->list, &ev_queue->event_list); @@ -457,8 +457,7 @@ static void ib_uverbs_async_handler(struct ib_uverbs_file *file, void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr) { - struct ib_uevent_object *uobj = container_of( - event->element.cq->uobject, struct ib_uevent_object, uobject); + struct ib_uevent_object *uobj = &event->element.cq->uobject->uevent; ib_uverbs_async_handler(uobj->uobject.ufile, uobj->uobject.user_handle, event->event, &uobj->event_list, diff --git a/drivers/infiniband/core/uverbs_std_types_cq.c b/drivers/infiniband/core/uverbs_std_types_cq.c index fbc605a95e51..a41c758042cc 100644 --- a/drivers/infiniband/core/uverbs_std_types_cq.c +++ b/drivers/infiniband/core/uverbs_std_types_cq.c @@ -116,7 +116,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)( } cq->device = ib_dev; - cq->uobject = &obj->uevent.uobject; + cq->uobject = obj; cq->comp_handler = ib_uverbs_comp_handler; cq->event_handler = ib_uverbs_cq_event_handler; cq->cq_context = ev_file ? &ev_file->ev_queue : NULL; diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index cea4e198701e..e9ab986ab323 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -77,6 +77,8 @@ extern struct workqueue_struct *ib_wq; extern struct workqueue_struct *ib_comp_wq; extern struct workqueue_struct *ib_comp_unbound_wq; +struct ib_ucq_object; + __printf(3, 4) __cold void ibdev_printk(const char *level, const struct ib_device *ibdev, const char *format, ...); @@ -1544,7 +1546,7 @@ enum ib_poll_context { struct ib_cq { struct ib_device *device; - struct ib_uobject *uobject; + struct ib_ucq_object *uobject; ib_comp_handler comp_handler; void (*event_handler)(struct ib_event *, void *); void *cq_context; -- cgit v1.2.3-59-g8ed1b From 620d3f8176cbb3a9c0c7962a05fb15310a9998d4 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 8 Jan 2020 19:21:59 +0200 Subject: RDMA/core: Do not erase the type of ib_qp.uobject This is a struct ib_uqp_object pointer, instead of using container_of() all over the place just store it with its actual type. Link: https://lore.kernel.org/r/1578504126-9400-8-git-send-email-yishaih@mellanox.com Signed-off-by: Yishai Hadas Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/core_priv.h | 2 +- drivers/infiniband/core/uverbs_cmd.c | 34 +++++++++++++++++++++------------- drivers/infiniband/core/uverbs_main.c | 3 +-- include/rdma/ib_verbs.h | 3 ++- 4 files changed, 25 insertions(+), 17 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h index d657d90e618b..b1457b3464d3 100644 --- a/drivers/infiniband/core/core_priv.h +++ b/drivers/infiniband/core/core_priv.h @@ -321,7 +321,7 @@ static inline struct ib_qp *_ib_create_qp(struct ib_device *dev, struct ib_pd *pd, struct ib_qp_init_attr *attr, struct ib_udata *udata, - struct ib_uobject *uobj) + struct ib_uqp_object *uobj) { enum ib_qp_type qp_type = attr->qp_type; struct ib_qp *qp; diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index b08679a5da25..4d84d083847c 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -1424,7 +1424,7 @@ static int create_qp(struct uverbs_attr_bundle *attrs, qp = ib_create_qp(pd, &attr); else qp = _ib_create_qp(device, pd, &attr, &attrs->driver_udata, - &obj->uevent.uobject); + obj); if (IS_ERR(qp)) { ret = PTR_ERR(qp); @@ -1457,7 +1457,7 @@ static int create_qp(struct uverbs_attr_bundle *attrs, atomic_inc(&ind_tbl->usecnt); } else { /* It is done in _ib_create_qp for other QP types */ - qp->uobject = &obj->uevent.uobject; + qp->uobject = obj; } obj->uevent.uobject.object = qp; @@ -1628,7 +1628,7 @@ static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs) obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject); atomic_inc(&obj->uxrcd->refcnt); - qp->uobject = &obj->uevent.uobject; + qp->uobject = obj; uobj_put_read(xrcd_uobj); rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs); @@ -1693,7 +1693,8 @@ static int ib_uverbs_query_qp(struct uverbs_attr_bundle *attrs) ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr); - uobj_put_obj_read(qp); + rdma_lookup_put_uobject(&qp->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); if (ret) goto out; @@ -1930,7 +1931,8 @@ static int modify_qp(struct uverbs_attr_bundle *attrs, &attrs->driver_udata); release_qp: - uobj_put_obj_read(qp); + rdma_lookup_put_uobject(&qp->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); out: kfree(attr); @@ -2194,7 +2196,8 @@ static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs) ret = ret2; out_put: - uobj_put_obj_read(qp); + rdma_lookup_put_uobject(&qp->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); while (wr) { if (is_ud && ud_wr(wr)->ah) @@ -2336,7 +2339,8 @@ static int ib_uverbs_post_recv(struct uverbs_attr_bundle *attrs) resp.bad_wr = 0; ret = qp->device->ops.post_recv(qp->real_qp, wr, &bad_wr); - uobj_put_obj_read(qp); + rdma_lookup_put_uobject(&qp->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); if (ret) { for (next = wr; next; next = next->next) { ++resp.bad_wr; @@ -2517,7 +2521,7 @@ static int ib_uverbs_attach_mcast(struct uverbs_attr_bundle *attrs) if (!qp) return -EINVAL; - obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); + obj = qp->uobject; mutex_lock(&obj->mcast_lock); list_for_each_entry(mcast, &obj->mcast_list, list) @@ -2544,7 +2548,8 @@ static int ib_uverbs_attach_mcast(struct uverbs_attr_bundle *attrs) out_put: mutex_unlock(&obj->mcast_lock); - uobj_put_obj_read(qp); + rdma_lookup_put_uobject(&qp->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); return ret; } @@ -2566,7 +2571,7 @@ static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs) if (!qp) return -EINVAL; - obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); + obj = qp->uobject; mutex_lock(&obj->mcast_lock); list_for_each_entry(mcast, &obj->mcast_list, list) @@ -2587,7 +2592,8 @@ static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs) out_put: mutex_unlock(&obj->mcast_lock); - uobj_put_obj_read(qp); + rdma_lookup_put_uobject(&qp->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); return ret; } @@ -3339,7 +3345,8 @@ static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs) if (err) goto err_copy; - uobj_put_obj_read(qp); + rdma_lookup_put_uobject(&qp->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); kfree(flow_attr); if (cmd.flow_attr.num_of_specs) kfree(kern_flow_attr); @@ -3353,7 +3360,8 @@ err_free: err_free_flow_attr: kfree(flow_attr); err_put: - uobj_put_obj_read(qp); + rdma_lookup_put_uobject(&qp->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); err_uobj: uobj_alloc_abort(uobj, attrs); err_free_attr: diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index 2849bf40bc97..12da68fca1c1 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -472,8 +472,7 @@ void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr) if (!event->element.qp->uobject) return; - uobj = container_of(event->element.qp->uobject, - struct ib_uevent_object, uobject); + uobj = &event->element.qp->uobject->uevent; ib_uverbs_async_handler(context_ptr, uobj->uobject.user_handle, event->event, &uobj->event_list, diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index e9ab986ab323..b4dbc5f9636a 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -72,6 +72,7 @@ #define IB_FW_VERSION_NAME_MAX ETHTOOL_FWVERS_LEN struct ib_umem_odp; +struct ib_uqp_object; extern struct workqueue_struct *ib_wq; extern struct workqueue_struct *ib_comp_wq; @@ -1735,7 +1736,7 @@ struct ib_qp { atomic_t usecnt; struct list_head open_list; struct ib_qp *real_qp; - struct ib_uobject *uobject; + struct ib_uqp_object *uobject; void (*event_handler)(struct ib_event *, void *); void *qp_context; /* sgid_attrs associated with the AV's */ -- cgit v1.2.3-59-g8ed1b From 9fbe334c6a67c3c09f187e4b9b0e6eaf0ad31429 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 8 Jan 2020 19:22:00 +0200 Subject: RDMA/core: Do not erase the type of ib_srq.uobject This is a struct ib_usrq_object pointer, instead of using container_of() all over the place just store it with its actual type. Link: https://lore.kernel.org/r/1578504126-9400-9-git-send-email-yishaih@mellanox.com Signed-off-by: Yishai Hadas Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/uverbs_cmd.c | 17 +++++++++++------ drivers/infiniband/core/uverbs_main.c | 3 +-- include/rdma/ib_verbs.h | 3 ++- 3 files changed, 14 insertions(+), 9 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index 4d84d083847c..8350e023d3b5 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -1492,7 +1492,8 @@ static int create_qp(struct uverbs_attr_bundle *attrs, rdma_lookup_put_uobject(&rcq->uobject->uevent.uobject, UVERBS_LOOKUP_READ); if (srq) - uobj_put_obj_read(srq); + rdma_lookup_put_uobject(&srq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); if (ind_tbl) uobj_put_obj_read(ind_tbl); @@ -1513,7 +1514,8 @@ err_put: rdma_lookup_put_uobject(&rcq->uobject->uevent.uobject, UVERBS_LOOKUP_READ); if (srq) - uobj_put_obj_read(srq); + rdma_lookup_put_uobject(&srq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); if (ind_tbl) uobj_put_obj_read(ind_tbl); @@ -2390,7 +2392,8 @@ static int ib_uverbs_post_srq_recv(struct uverbs_attr_bundle *attrs) resp.bad_wr = 0; ret = srq->device->ops.post_srq_recv(srq, wr, &bad_wr); - uobj_put_obj_read(srq); + rdma_lookup_put_uobject(&srq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); if (ret) for (next = wr; next; next = next->next) { @@ -3458,7 +3461,7 @@ static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs, srq->device = pd->device; srq->pd = pd; srq->srq_type = cmd->srq_type; - srq->uobject = &obj->uevent.uobject; + srq->uobject = obj; srq->event_handler = attr.event_handler; srq->srq_context = attr.srq_context; @@ -3584,7 +3587,8 @@ static int ib_uverbs_modify_srq(struct uverbs_attr_bundle *attrs) ret = srq->device->ops.modify_srq(srq, &attr, cmd.attr_mask, &attrs->driver_udata); - uobj_put_obj_read(srq); + rdma_lookup_put_uobject(&srq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); return ret; } @@ -3607,7 +3611,8 @@ static int ib_uverbs_query_srq(struct uverbs_attr_bundle *attrs) ret = ib_query_srq(srq, &attr); - uobj_put_obj_read(srq); + rdma_lookup_put_uobject(&srq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); if (ret) return ret; diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index 12da68fca1c1..a501f76bc2f9 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -493,8 +493,7 @@ void ib_uverbs_srq_event_handler(struct ib_event *event, void *context_ptr) { struct ib_uevent_object *uobj; - uobj = container_of(event->element.srq->uobject, - struct ib_uevent_object, uobject); + uobj = &event->element.srq->uobject->uevent; ib_uverbs_async_handler(context_ptr, uobj->uobject.user_handle, event->event, &uobj->event_list, diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index b4dbc5f9636a..7990b55b1b40 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -73,6 +73,7 @@ struct ib_umem_odp; struct ib_uqp_object; +struct ib_usrq_object; extern struct workqueue_struct *ib_wq; extern struct workqueue_struct *ib_comp_wq; @@ -1575,7 +1576,7 @@ struct ib_cq { struct ib_srq { struct ib_device *device; struct ib_pd *pd; - struct ib_uobject *uobject; + struct ib_usrq_object *uobject; void (*event_handler)(struct ib_event *, void *); void *srq_context; enum ib_srq_type srq_type; -- cgit v1.2.3-59-g8ed1b From e04dd13159b0ddc0ff7f5e110bf99af3c65fabd3 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 8 Jan 2020 19:22:01 +0200 Subject: RDMA/core: Do not erase the type of ib_wq.uobject This is a struct ib_uwq_object pointer, instead of using container_of() all over the place just store it with its actual type. Link: https://lore.kernel.org/r/1578504126-9400-10-git-send-email-yishaih@mellanox.com Signed-off-by: Yishai Hadas Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/uverbs_cmd.c | 13 ++++++++----- drivers/infiniband/core/uverbs_main.c | 3 +-- include/rdma/ib_verbs.h | 3 ++- 3 files changed, 11 insertions(+), 8 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index 8350e023d3b5..66f86b4e5e81 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -2971,7 +2971,7 @@ static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs) goto err_put_cq; } - wq->uobject = &obj->uevent.uobject; + wq->uobject = obj; obj->uevent.uobject.object = wq; wq->wq_type = wq_init_attr.wq_type; wq->cq = cq; @@ -2981,7 +2981,7 @@ static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs) atomic_set(&wq->usecnt, 0); atomic_inc(&pd->usecnt); atomic_inc(&cq->usecnt); - wq->uobject = &obj->uevent.uobject; + wq->uobject = obj; obj->uevent.uobject.object = wq; memset(&resp, 0, sizeof(resp)); @@ -3070,7 +3070,8 @@ static int ib_uverbs_ex_modify_wq(struct uverbs_attr_bundle *attrs) } ret = wq->device->ops.modify_wq(wq, &wq_attr, cmd.attr_mask, &attrs->driver_udata); - uobj_put_obj_read(wq); + rdma_lookup_put_uobject(&wq->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); return ret; } @@ -3171,7 +3172,8 @@ static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs) kfree(wqs_handles); for (j = 0; j < num_read_wqs; j++) - uobj_put_obj_read(wqs[j]); + rdma_lookup_put_uobject(&wqs[j]->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); rdma_alloc_commit_uobject(uobj, attrs); return 0; @@ -3182,7 +3184,8 @@ err_uobj: uobj_alloc_abort(uobj, attrs); put_wqs: for (j = 0; j < num_read_wqs; j++) - uobj_put_obj_read(wqs[j]); + rdma_lookup_put_uobject(&wqs[j]->uobject->uevent.uobject, + UVERBS_LOOKUP_READ); err_free: kfree(wqs_handles); kfree(wqs); diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index a501f76bc2f9..87f7a94adf57 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -481,8 +481,7 @@ void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr) void ib_uverbs_wq_event_handler(struct ib_event *event, void *context_ptr) { - struct ib_uevent_object *uobj = container_of(event->element.wq->uobject, - struct ib_uevent_object, uobject); + struct ib_uevent_object *uobj = &event->element.wq->uobject->uevent; ib_uverbs_async_handler(context_ptr, uobj->uobject.user_handle, event->event, &uobj->event_list, diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 7990b55b1b40..d8031f6f327e 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -74,6 +74,7 @@ struct ib_umem_odp; struct ib_uqp_object; struct ib_usrq_object; +struct ib_uwq_object; extern struct workqueue_struct *ib_wq; extern struct workqueue_struct *ib_comp_wq; @@ -1621,7 +1622,7 @@ enum ib_wq_state { struct ib_wq { struct ib_device *device; - struct ib_uobject *uobject; + struct ib_uwq_object *uobject; void *wq_context; void (*event_handler)(struct ib_event *, void *); struct ib_pd *pd; -- cgit v1.2.3-59-g8ed1b From 817d65765069d1e7dee97473a6465947db1bea80 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 8 Jan 2020 19:22:02 +0200 Subject: RDMA/core: Simplify type usage for ib_uverbs_async_handler() This function works on an ib_uverbs_async_file. Accept that as a parameter instead of the struct ib_uverbs_file. Consoldiate all the callers working from an ib_uevent_object to a single function and locate the async_file directly from the struct ib_uobject instead of using context_ptr. Link: https://lore.kernel.org/r/1578504126-9400-11-git-send-email-yishaih@mellanox.com Signed-off-by: Yishai Hadas Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/uverbs.h | 2 - drivers/infiniband/core/uverbs_cmd.c | 9 +--- drivers/infiniband/core/uverbs_main.c | 79 +++++++++++++++-------------------- 3 files changed, 34 insertions(+), 56 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h index 9fa044699cf0..8384b66b661c 100644 --- a/drivers/infiniband/core/uverbs.h +++ b/drivers/infiniband/core/uverbs.h @@ -235,8 +235,6 @@ void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr); void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr); void ib_uverbs_wq_event_handler(struct ib_event *event, void *context_ptr); void ib_uverbs_srq_event_handler(struct ib_event *event, void *context_ptr); -void ib_uverbs_event_handler(struct ib_event_handler *handler, - struct ib_event *event); int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject, struct ib_xrcd *xrcd, enum rdma_remove_reason why, struct uverbs_attr_bundle *attrs); diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index 66f86b4e5e81..51117e784d40 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -1378,7 +1378,6 @@ static int create_qp(struct uverbs_attr_bundle *attrs, } attr.event_handler = ib_uverbs_qp_event_handler; - attr.qp_context = attrs->ufile; attr.send_cq = scq; attr.recv_cq = rcq; attr.srq = srq; @@ -1394,7 +1393,6 @@ static int create_qp(struct uverbs_attr_bundle *attrs, attr.cap.max_recv_sge = cmd->max_recv_sge; attr.cap.max_inline_data = cmd->max_inline_data; - obj->uevent.events_reported = 0; INIT_LIST_HEAD(&obj->uevent.event_list); INIT_LIST_HEAD(&obj->mcast_list); @@ -1442,7 +1440,6 @@ static int create_qp(struct uverbs_attr_bundle *attrs, qp->srq = attr.srq; qp->rwq_ind_tbl = ind_tbl; qp->event_handler = attr.event_handler; - qp->qp_context = attr.qp_context; qp->qp_type = attr.qp_type; atomic_set(&qp->usecnt, 0); atomic_inc(&pd->usecnt); @@ -1577,7 +1574,7 @@ static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs) struct ib_xrcd *xrcd; struct ib_uobject *uninitialized_var(xrcd_uobj); struct ib_qp *qp; - struct ib_qp_open_attr attr; + struct ib_qp_open_attr attr = {}; int ret; struct ib_device *ib_dev; @@ -1603,11 +1600,9 @@ static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs) } attr.event_handler = ib_uverbs_qp_event_handler; - attr.qp_context = attrs->ufile; attr.qp_num = cmd.qpn; attr.qp_type = cmd.qp_type; - obj->uevent.events_reported = 0; INIT_LIST_HEAD(&obj->uevent.event_list); INIT_LIST_HEAD(&obj->mcast_list); @@ -2962,7 +2957,6 @@ static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs) wq_init_attr.wq_type = cmd.wq_type; wq_init_attr.event_handler = ib_uverbs_wq_event_handler; wq_init_attr.create_flags = cmd.create_flags; - obj->uevent.events_reported = 0; INIT_LIST_HEAD(&obj->uevent.event_list); wq = pd->device->ops.create_wq(pd, &wq_init_attr, &attrs->driver_udata); @@ -3452,7 +3446,6 @@ static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs, attr.attr.max_sge = cmd->max_sge; attr.attr.srq_limit = cmd->srq_limit; - obj->uevent.events_reported = 0; INIT_LIST_HEAD(&obj->uevent.event_list); srq = rdma_zalloc_drv_obj(ib_dev, ib_srq); diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index 87f7a94adf57..97770e72392f 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -421,92 +421,79 @@ void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context) kill_fasync(&ev_queue->async_queue, SIGIO, POLL_IN); } -static void ib_uverbs_async_handler(struct ib_uverbs_file *file, - __u64 element, __u64 event, - struct list_head *obj_list, - u32 *counter) +static void +ib_uverbs_async_handler(struct ib_uverbs_async_event_file *async_file, + __u64 element, __u64 event, struct list_head *obj_list, + u32 *counter) { struct ib_uverbs_event *entry; unsigned long flags; - spin_lock_irqsave(&file->async_file->ev_queue.lock, flags); - if (file->async_file->ev_queue.is_closed) { - spin_unlock_irqrestore(&file->async_file->ev_queue.lock, flags); + spin_lock_irqsave(&async_file->ev_queue.lock, flags); + if (async_file->ev_queue.is_closed) { + spin_unlock_irqrestore(&async_file->ev_queue.lock, flags); return; } entry = kmalloc(sizeof(*entry), GFP_ATOMIC); if (!entry) { - spin_unlock_irqrestore(&file->async_file->ev_queue.lock, flags); + spin_unlock_irqrestore(&async_file->ev_queue.lock, flags); return; } - entry->desc.async.element = element; + entry->desc.async.element = element; entry->desc.async.event_type = event; - entry->desc.async.reserved = 0; - entry->counter = counter; + entry->desc.async.reserved = 0; + entry->counter = counter; - list_add_tail(&entry->list, &file->async_file->ev_queue.event_list); + list_add_tail(&entry->list, &async_file->ev_queue.event_list); if (obj_list) list_add_tail(&entry->obj_list, obj_list); - spin_unlock_irqrestore(&file->async_file->ev_queue.lock, flags); + spin_unlock_irqrestore(&async_file->ev_queue.lock, flags); - wake_up_interruptible(&file->async_file->ev_queue.poll_wait); - kill_fasync(&file->async_file->ev_queue.async_queue, SIGIO, POLL_IN); + wake_up_interruptible(&async_file->ev_queue.poll_wait); + kill_fasync(&async_file->ev_queue.async_queue, SIGIO, POLL_IN); } -void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr) +static void uverbs_uobj_event(struct ib_uevent_object *eobj, + struct ib_event *event) { - struct ib_uevent_object *uobj = &event->element.cq->uobject->uevent; + ib_uverbs_async_handler(eobj->uobject.ufile->async_file, + eobj->uobject.user_handle, event->event, + &eobj->event_list, &eobj->events_reported); +} - ib_uverbs_async_handler(uobj->uobject.ufile, uobj->uobject.user_handle, - event->event, &uobj->event_list, - &uobj->events_reported); +void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr) +{ + uverbs_uobj_event(&event->element.cq->uobject->uevent, event); } void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr) { - struct ib_uevent_object *uobj; - /* for XRC target qp's, check that qp is live */ if (!event->element.qp->uobject) return; - uobj = &event->element.qp->uobject->uevent; - - ib_uverbs_async_handler(context_ptr, uobj->uobject.user_handle, - event->event, &uobj->event_list, - &uobj->events_reported); + uverbs_uobj_event(&event->element.qp->uobject->uevent, event); } void ib_uverbs_wq_event_handler(struct ib_event *event, void *context_ptr) { - struct ib_uevent_object *uobj = &event->element.wq->uobject->uevent; - - ib_uverbs_async_handler(context_ptr, uobj->uobject.user_handle, - event->event, &uobj->event_list, - &uobj->events_reported); + uverbs_uobj_event(&event->element.wq->uobject->uevent, event); } void ib_uverbs_srq_event_handler(struct ib_event *event, void *context_ptr) { - struct ib_uevent_object *uobj; - - uobj = &event->element.srq->uobject->uevent; - - ib_uverbs_async_handler(context_ptr, uobj->uobject.user_handle, - event->event, &uobj->event_list, - &uobj->events_reported); + uverbs_uobj_event(&event->element.srq->uobject->uevent, event); } -void ib_uverbs_event_handler(struct ib_event_handler *handler, - struct ib_event *event) +static void ib_uverbs_event_handler(struct ib_event_handler *handler, + struct ib_event *event) { - struct ib_uverbs_file *file = - container_of(handler, struct ib_uverbs_file, event_handler); - - ib_uverbs_async_handler(file, event->element.port_num, event->event, - NULL, NULL); + ib_uverbs_async_handler( + container_of(handler, struct ib_uverbs_file, event_handler) + ->async_file, + event->element.port_num, event->event, NULL, NULL); } void ib_uverbs_free_async_event_file(struct ib_uverbs_file *file) -- cgit v1.2.3-59-g8ed1b From 39e83af817610e80a9582df94d44173bcb6f85e4 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 8 Jan 2020 19:22:04 +0200 Subject: RDMA/core: Remove the ufile arg from rdma_alloc_begin_uobject Now that all callers provide a non-NULL attrs the ufile is redundant. Adjust things so that the context handling is done inside alloc_uobj, and the ib_uverbs_get_ucontext_file() is avoided if we already have the context. Link: https://lore.kernel.org/r/1578504126-9400-13-git-send-email-yishaih@mellanox.com Signed-off-by: Yishai Hadas Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/rdma_core.c | 36 +++++++++++++++++++----------------- include/rdma/uverbs_std_types.h | 3 +-- include/rdma/uverbs_types.h | 3 +-- 3 files changed, 21 insertions(+), 21 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/rdma_core.c b/drivers/infiniband/core/rdma_core.c index a9f5263c9559..50251da123f7 100644 --- a/drivers/infiniband/core/rdma_core.c +++ b/drivers/infiniband/core/rdma_core.c @@ -256,15 +256,20 @@ int __uobj_perform_destroy(const struct uverbs_api_object *obj, u32 id, } /* alloc_uobj must be undone by uverbs_destroy_uobject() */ -static struct ib_uobject *alloc_uobj(struct ib_uverbs_file *ufile, +static struct ib_uobject *alloc_uobj(struct uverbs_attr_bundle *attrs, const struct uverbs_api_object *obj) { + struct ib_uverbs_file *ufile = attrs->ufile; struct ib_uobject *uobj; - struct ib_ucontext *ucontext; - ucontext = ib_uverbs_get_ucontext_file(ufile); - if (IS_ERR(ucontext)) - return ERR_CAST(ucontext); + if (!attrs->context) { + struct ib_ucontext *ucontext = + ib_uverbs_get_ucontext_file(ufile); + + if (IS_ERR(ucontext)) + return ERR_CAST(ucontext); + attrs->context = ucontext; + } uobj = kzalloc(obj->type_attrs->obj_size, GFP_KERNEL); if (!uobj) @@ -274,7 +279,7 @@ static struct ib_uobject *alloc_uobj(struct ib_uverbs_file *ufile, * The object is added to the list in the commit stage. */ uobj->ufile = ufile; - uobj->context = ucontext; + uobj->context = attrs->context; INIT_LIST_HEAD(&uobj->list); uobj->uapi_object = obj; /* @@ -417,12 +422,12 @@ free: static struct ib_uobject * alloc_begin_idr_uobject(const struct uverbs_api_object *obj, - struct ib_uverbs_file *ufile) + struct uverbs_attr_bundle *attrs) { int ret; struct ib_uobject *uobj; - uobj = alloc_uobj(ufile, obj); + uobj = alloc_uobj(attrs, obj); if (IS_ERR(uobj)) return uobj; @@ -438,7 +443,7 @@ alloc_begin_idr_uobject(const struct uverbs_api_object *obj, return uobj; remove: - xa_erase(&ufile->idr, uobj->id); + xa_erase(&attrs->ufile->idr, uobj->id); uobj_put: uverbs_uobject_put(uobj); return ERR_PTR(ret); @@ -446,7 +451,7 @@ uobj_put: static struct ib_uobject * alloc_begin_fd_uobject(const struct uverbs_api_object *obj, - struct ib_uverbs_file *ufile) + struct uverbs_attr_bundle *attrs) { const struct uverbs_obj_fd_type *fd_type = container_of(obj->type_attrs, struct uverbs_obj_fd_type, type); @@ -461,7 +466,7 @@ alloc_begin_fd_uobject(const struct uverbs_api_object *obj, if (new_fd < 0) return ERR_PTR(new_fd); - uobj = alloc_uobj(ufile, obj); + uobj = alloc_uobj(attrs, obj); if (IS_ERR(uobj)) goto err_fd; @@ -475,7 +480,6 @@ alloc_begin_fd_uobject(const struct uverbs_api_object *obj, uobj->object = filp; uobj->id = new_fd; - uobj->ufile = ufile; return uobj; err_uobj: @@ -486,9 +490,9 @@ err_fd: } struct ib_uobject *rdma_alloc_begin_uobject(const struct uverbs_api_object *obj, - struct ib_uverbs_file *ufile, struct uverbs_attr_bundle *attrs) { + struct ib_uverbs_file *ufile = attrs->ufile; struct ib_uobject *ret; if (IS_ERR(obj)) @@ -502,13 +506,11 @@ struct ib_uobject *rdma_alloc_begin_uobject(const struct uverbs_api_object *obj, if (!down_read_trylock(&ufile->hw_destroy_rwsem)) return ERR_PTR(-EIO); - ret = obj->type_class->alloc_begin(obj, ufile); + ret = obj->type_class->alloc_begin(obj, attrs); if (IS_ERR(ret)) { up_read(&ufile->hw_destroy_rwsem); return ret; } - if (attrs) - attrs->context = ret->context; return ret; } @@ -930,7 +932,7 @@ uverbs_get_uobject_from_file(u16 object_id, enum uverbs_obj_access access, return rdma_lookup_get_uobject(obj, attrs->ufile, id, UVERBS_LOOKUP_WRITE, attrs); case UVERBS_ACCESS_NEW: - return rdma_alloc_begin_uobject(obj, attrs->ufile, attrs); + return rdma_alloc_begin_uobject(obj, attrs); default: WARN_ON(true); return ERR_PTR(-EOPNOTSUPP); diff --git a/include/rdma/uverbs_std_types.h b/include/rdma/uverbs_std_types.h index c6bcaad4a8e0..1b28ce1aba07 100644 --- a/include/rdma/uverbs_std_types.h +++ b/include/rdma/uverbs_std_types.h @@ -114,8 +114,7 @@ static inline struct ib_uobject * __uobj_alloc(const struct uverbs_api_object *obj, struct uverbs_attr_bundle *attrs, struct ib_device **ib_dev) { - struct ib_uobject *uobj = - rdma_alloc_begin_uobject(obj, attrs->ufile, attrs); + struct ib_uobject *uobj = rdma_alloc_begin_uobject(obj, attrs); if (!IS_ERR(uobj)) *ib_dev = attrs->context->device; diff --git a/include/rdma/uverbs_types.h b/include/rdma/uverbs_types.h index 01d59df4e9e7..f1cbdae67250 100644 --- a/include/rdma/uverbs_types.h +++ b/include/rdma/uverbs_types.h @@ -83,7 +83,7 @@ enum rdma_lookup_mode { */ struct uverbs_obj_type_class { struct ib_uobject *(*alloc_begin)(const struct uverbs_api_object *obj, - struct ib_uverbs_file *ufile); + struct uverbs_attr_bundle *attrs); /* This consumes the kref on uobj */ void (*alloc_commit)(struct ib_uobject *uobj); /* This does not consume the kref on uobj */ @@ -137,7 +137,6 @@ struct ib_uobject *rdma_lookup_get_uobject(const struct uverbs_api_object *obj, void rdma_lookup_put_uobject(struct ib_uobject *uobj, enum rdma_lookup_mode mode); struct ib_uobject *rdma_alloc_begin_uobject(const struct uverbs_api_object *obj, - struct ib_uverbs_file *ufile, struct uverbs_attr_bundle *attrs); void rdma_alloc_abort_uobject(struct ib_uobject *uobj, struct uverbs_attr_bundle *attrs); -- cgit v1.2.3-59-g8ed1b From 3e032c0e92aa0c4c0b46c5e2d6d41706c8fce488 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 8 Jan 2020 19:22:05 +0200 Subject: RDMA/core: Make ib_uverbs_async_event_file into a uobject This makes async events aligned with completion events as both are full uobjects of FD type and use the same uobject lifecycle. A bunch of duplicate code is consolidated and the general flow between the two FDs is now very similar. Link: https://lore.kernel.org/r/1578504126-9400-14-git-send-email-yishaih@mellanox.com Signed-off-by: Yishai Hadas Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/Makefile | 3 +- drivers/infiniband/core/rdma_core.h | 1 + drivers/infiniband/core/uverbs.h | 15 +-- drivers/infiniband/core/uverbs_cmd.c | 30 ++--- drivers/infiniband/core/uverbs_main.c | 130 ++++----------------- drivers/infiniband/core/uverbs_std_types.c | 25 ++-- .../infiniband/core/uverbs_std_types_async_fd.c | 33 ++++++ drivers/infiniband/core/uverbs_uapi.c | 1 + include/uapi/rdma/ib_user_ioctl_cmds.h | 1 + 9 files changed, 96 insertions(+), 143 deletions(-) create mode 100644 drivers/infiniband/core/uverbs_std_types_async_fd.c (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile index 2b86a514c9bf..d1b14887960e 100644 --- a/drivers/infiniband/core/Makefile +++ b/drivers/infiniband/core/Makefile @@ -35,6 +35,7 @@ ib_uverbs-y := uverbs_main.o uverbs_cmd.o uverbs_marshall.o \ uverbs_std_types_cq.o \ uverbs_std_types_flow_action.o uverbs_std_types_dm.o \ uverbs_std_types_mr.o uverbs_std_types_counters.o \ - uverbs_uapi.o uverbs_std_types_device.o + uverbs_uapi.o uverbs_std_types_device.o \ + uverbs_std_types_async_fd.o ib_uverbs-$(CONFIG_INFINIBAND_USER_MEM) += umem.o ib_uverbs-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += umem_odp.o diff --git a/drivers/infiniband/core/rdma_core.h b/drivers/infiniband/core/rdma_core.h index 29f905e8c2a8..33978e0f1262 100644 --- a/drivers/infiniband/core/rdma_core.h +++ b/drivers/infiniband/core/rdma_core.h @@ -151,6 +151,7 @@ void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm, unsigned int num_attrs); void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile); +extern const struct uapi_definition uverbs_def_obj_async_fd[]; extern const struct uapi_definition uverbs_def_obj_counters[]; extern const struct uapi_definition uverbs_def_obj_cq[]; extern const struct uapi_definition uverbs_def_obj_device[]; diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h index 8384b66b661c..ccde5d20a6cf 100644 --- a/drivers/infiniband/core/uverbs.h +++ b/drivers/infiniband/core/uverbs.h @@ -111,7 +111,6 @@ struct ib_uverbs_device { struct srcu_struct disassociate_srcu; struct mutex lists_mutex; /* protect lists */ struct list_head uverbs_file_list; - struct list_head uverbs_events_file_list; struct uverbs_api *uapi; }; @@ -124,10 +123,9 @@ struct ib_uverbs_event_queue { }; struct ib_uverbs_async_event_file { + struct ib_uobject uobj; struct ib_uverbs_event_queue ev_queue; - struct ib_uverbs_file *uverbs_file; - struct kref ref; - struct list_head list; + struct ib_event_handler event_handler; }; struct ib_uverbs_completion_event_file { @@ -144,8 +142,7 @@ struct ib_uverbs_file { * ucontext_lock held */ struct ib_ucontext *ucontext; - struct ib_event_handler event_handler; - struct ib_uverbs_async_event_file *async_file; + struct ib_uverbs_async_event_file *async_file; struct list_head list; /* @@ -217,10 +214,10 @@ struct ib_ucq_object { }; extern const struct file_operations uverbs_event_fops; +extern const struct file_operations uverbs_async_event_fops; void ib_uverbs_init_event_queue(struct ib_uverbs_event_queue *ev_queue); -struct file *ib_uverbs_alloc_async_event_file(struct ib_uverbs_file *uverbs_file, - struct ib_device *ib_dev); -void ib_uverbs_free_async_event_file(struct ib_uverbs_file *uverbs_file); +void ib_uverbs_init_async_event_file(struct ib_uverbs_async_event_file *ev_file); +void ib_uverbs_free_event_queue(struct ib_uverbs_event_queue *event_queue); void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res); void ib_uverbs_release_ucq(struct ib_uverbs_file *file, diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index 51117e784d40..ced1384d316b 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -209,9 +209,9 @@ static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs) struct ib_uverbs_get_context cmd; struct ib_uverbs_get_context_resp resp; struct ib_ucontext *ucontext; - struct file *filp; struct ib_rdmacg_object cg_obj; struct ib_device *ib_dev; + struct ib_uobject *uobj; int ret; ret = uverbs_request(attrs, &cmd, sizeof(cmd)); @@ -254,30 +254,28 @@ static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs) xa_init_flags(&ucontext->mmap_xa, XA_FLAGS_ALLOC); - ret = get_unused_fd_flags(O_CLOEXEC); - if (ret < 0) + uobj = uobj_alloc(UVERBS_OBJECT_ASYNC_EVENT, attrs, &ib_dev); + if (IS_ERR(uobj)) { + ret = PTR_ERR(uobj); goto err_free; - resp.async_fd = ret; - - filp = ib_uverbs_alloc_async_event_file(file, ib_dev); - if (IS_ERR(filp)) { - ret = PTR_ERR(filp); - goto err_fd; } + resp.async_fd = uobj->id; resp.num_comp_vectors = file->device->num_comp_vectors; ret = uverbs_response(attrs, &resp, sizeof(resp)); if (ret) - goto err_file; + goto err_uobj; ret = ib_dev->ops.alloc_ucontext(ucontext, &attrs->driver_udata); if (ret) - goto err_file; + goto err_uobj; rdma_restrack_uadd(&ucontext->res); - fd_install(resp.async_fd, filp); + ib_uverbs_init_async_event_file( + container_of(uobj, struct ib_uverbs_async_event_file, uobj)); + rdma_alloc_commit_uobject(uobj, attrs); /* * Make sure that ib_uverbs_get_ucontext() sees the pointer update @@ -289,12 +287,8 @@ static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs) return 0; -err_file: - ib_uverbs_free_async_event_file(file); - fput(filp); - -err_fd: - put_unused_fd(resp.async_fd); +err_uobj: + rdma_alloc_abort_uobject(uobj, attrs); err_free: kfree(ucontext); diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index 97770e72392f..121e65f69c0b 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -125,14 +125,6 @@ static void ib_uverbs_release_dev(struct device *device) kfree(dev); } -static void ib_uverbs_release_async_event_file(struct kref *ref) -{ - struct ib_uverbs_async_event_file *file = - container_of(ref, struct ib_uverbs_async_event_file, ref); - - kfree(file); -} - void ib_uverbs_release_ucq(struct ib_uverbs_file *file, struct ib_uverbs_completion_event_file *ev_file, struct ib_ucq_object *uobj) @@ -203,8 +195,7 @@ void ib_uverbs_release_file(struct kref *ref) ib_uverbs_comp_dev(file->device); if (file->async_file) - kref_put(&file->async_file->ref, - ib_uverbs_release_async_event_file); + uverbs_uobject_put(&file->async_file->uobj); put_device(&file->device->dev); if (file->disassociate_page) @@ -339,35 +330,6 @@ static int ib_uverbs_comp_event_fasync(int fd, struct file *filp, int on) return fasync_helper(fd, filp, on, &comp_ev_file->ev_queue.async_queue); } -static int ib_uverbs_async_event_close(struct inode *inode, struct file *filp) -{ - struct ib_uverbs_async_event_file *file = filp->private_data; - struct ib_uverbs_file *uverbs_file = file->uverbs_file; - struct ib_uverbs_event *entry, *tmp; - int closed_already = 0; - - mutex_lock(&uverbs_file->device->lists_mutex); - spin_lock_irq(&file->ev_queue.lock); - closed_already = file->ev_queue.is_closed; - file->ev_queue.is_closed = 1; - list_for_each_entry_safe(entry, tmp, &file->ev_queue.event_list, list) { - if (entry->counter) - list_del(&entry->obj_list); - kfree(entry); - } - spin_unlock_irq(&file->ev_queue.lock); - if (!closed_already) { - list_del(&file->list); - ib_unregister_event_handler(&uverbs_file->event_handler); - } - mutex_unlock(&uverbs_file->device->lists_mutex); - - kref_put(&uverbs_file->ref, ib_uverbs_release_file); - kref_put(&file->ref, ib_uverbs_release_async_event_file); - - return 0; -} - const struct file_operations uverbs_event_fops = { .owner = THIS_MODULE, .read = ib_uverbs_comp_event_read, @@ -377,11 +339,11 @@ const struct file_operations uverbs_event_fops = { .llseek = no_llseek, }; -static const struct file_operations uverbs_async_event_fops = { +const struct file_operations uverbs_async_event_fops = { .owner = THIS_MODULE, .read = ib_uverbs_async_event_read, .poll = ib_uverbs_async_event_poll, - .release = ib_uverbs_async_event_close, + .release = uverbs_uobject_fd_release, .fasync = ib_uverbs_async_event_fasync, .llseek = no_llseek, }; @@ -491,17 +453,11 @@ static void ib_uverbs_event_handler(struct ib_event_handler *handler, struct ib_event *event) { ib_uverbs_async_handler( - container_of(handler, struct ib_uverbs_file, event_handler) - ->async_file, + container_of(handler, struct ib_uverbs_async_event_file, + event_handler), event->element.port_num, event->event, NULL, NULL); } -void ib_uverbs_free_async_event_file(struct ib_uverbs_file *file) -{ - kref_put(&file->async_file->ref, ib_uverbs_release_async_event_file); - file->async_file = NULL; -} - void ib_uverbs_init_event_queue(struct ib_uverbs_event_queue *ev_queue) { spin_lock_init(&ev_queue->lock); @@ -511,45 +467,23 @@ void ib_uverbs_init_event_queue(struct ib_uverbs_event_queue *ev_queue) ev_queue->async_queue = NULL; } -struct file *ib_uverbs_alloc_async_event_file(struct ib_uverbs_file *uverbs_file, - struct ib_device *ib_dev) +void ib_uverbs_init_async_event_file( + struct ib_uverbs_async_event_file *async_file) { - struct ib_uverbs_async_event_file *ev_file; - struct file *filp; - - ev_file = kzalloc(sizeof(*ev_file), GFP_KERNEL); - if (!ev_file) - return ERR_PTR(-ENOMEM); - - ib_uverbs_init_event_queue(&ev_file->ev_queue); - ev_file->uverbs_file = uverbs_file; - kref_get(&ev_file->uverbs_file->ref); - kref_init(&ev_file->ref); - filp = anon_inode_getfile("[infinibandevent]", &uverbs_async_event_fops, - ev_file, O_RDONLY); - if (IS_ERR(filp)) - goto err_put_refs; - - mutex_lock(&uverbs_file->device->lists_mutex); - list_add_tail(&ev_file->list, - &uverbs_file->device->uverbs_events_file_list); - mutex_unlock(&uverbs_file->device->lists_mutex); - - WARN_ON(uverbs_file->async_file); - uverbs_file->async_file = ev_file; - kref_get(&uverbs_file->async_file->ref); - INIT_IB_EVENT_HANDLER(&uverbs_file->event_handler, - ib_dev, - ib_uverbs_event_handler); - ib_register_event_handler(&uverbs_file->event_handler); - /* At that point async file stuff was fully set */ + struct ib_uverbs_file *uverbs_file = async_file->uobj.ufile; + struct ib_device *ib_dev = async_file->uobj.context->device; - return filp; + ib_uverbs_init_event_queue(&async_file->ev_queue); -err_put_refs: - kref_put(&ev_file->uverbs_file->ref, ib_uverbs_release_file); - kref_put(&ev_file->ref, ib_uverbs_release_async_event_file); - return filp; + if (!WARN_ON(uverbs_file->async_file)) { + uverbs_file->async_file = async_file; + /* Pairs with the put in ib_uverbs_release_file */ + uverbs_uobject_get(&async_file->uobj); + } + + INIT_IB_EVENT_HANDLER(&async_file->event_handler, ib_dev, + ib_uverbs_event_handler); + ib_register_event_handler(&async_file->event_handler); } static ssize_t verify_hdr(struct ib_uverbs_cmd_hdr *hdr, @@ -1178,7 +1112,6 @@ static void ib_uverbs_add_one(struct ib_device *device) mutex_init(&uverbs_dev->xrcd_tree_mutex); mutex_init(&uverbs_dev->lists_mutex); INIT_LIST_HEAD(&uverbs_dev->uverbs_file_list); - INIT_LIST_HEAD(&uverbs_dev->uverbs_events_file_list); rcu_assign_pointer(uverbs_dev->ib_dev, device); uverbs_dev->num_comp_vectors = device->num_comp_vectors; @@ -1223,7 +1156,6 @@ static void ib_uverbs_free_hw_resources(struct ib_uverbs_device *uverbs_dev, struct ib_device *ib_dev) { struct ib_uverbs_file *file; - struct ib_uverbs_async_event_file *event_file; struct ib_event event; /* Pending running commands to terminate */ @@ -1246,31 +1178,15 @@ static void ib_uverbs_free_hw_resources(struct ib_uverbs_device *uverbs_dev, */ mutex_unlock(&uverbs_dev->lists_mutex); - ib_uverbs_event_handler(&file->event_handler, &event); + if (file->async_file) + ib_uverbs_event_handler( + &file->async_file->event_handler, &event); + uverbs_destroy_ufile_hw(file, RDMA_REMOVE_DRIVER_REMOVE); kref_put(&file->ref, ib_uverbs_release_file); mutex_lock(&uverbs_dev->lists_mutex); } - - while (!list_empty(&uverbs_dev->uverbs_events_file_list)) { - event_file = list_first_entry(&uverbs_dev-> - uverbs_events_file_list, - struct ib_uverbs_async_event_file, - list); - spin_lock_irq(&event_file->ev_queue.lock); - event_file->ev_queue.is_closed = 1; - spin_unlock_irq(&event_file->ev_queue.lock); - - list_del(&event_file->list); - ib_unregister_event_handler( - &event_file->uverbs_file->event_handler); - event_file->uverbs_file->event_handler.device = - NULL; - - wake_up_interruptible(&event_file->ev_queue.poll_wait); - kill_fasync(&event_file->ev_queue.async_queue, SIGIO, POLL_IN); - } mutex_unlock(&uverbs_dev->lists_mutex); uverbs_disassociate_api(uverbs_dev->uapi); diff --git a/drivers/infiniband/core/uverbs_std_types.c b/drivers/infiniband/core/uverbs_std_types.c index def038a0fe77..efe70bcf79b1 100644 --- a/drivers/infiniband/core/uverbs_std_types.c +++ b/drivers/infiniband/core/uverbs_std_types.c @@ -202,17 +202,15 @@ static int uverbs_free_pd(struct ib_uobject *uobject, return 0; } -static int -uverbs_completion_event_file_destroy_uobj(struct ib_uobject *uobj, - enum rdma_remove_reason why) +void ib_uverbs_free_event_queue(struct ib_uverbs_event_queue *event_queue) { - struct ib_uverbs_completion_event_file *file = - container_of(uobj, struct ib_uverbs_completion_event_file, - uobj); - struct ib_uverbs_event_queue *event_queue = &file->ev_queue; struct ib_uverbs_event *entry, *tmp; spin_lock_irq(&event_queue->lock); + /* + * The user must ensure that no new items are added to the event_list + * once is_closed is set. + */ event_queue->is_closed = 1; spin_unlock_irq(&event_queue->lock); wake_up_interruptible(&event_queue->poll_wait); @@ -225,8 +223,19 @@ uverbs_completion_event_file_destroy_uobj(struct ib_uobject *uobj, kfree(entry); } spin_unlock_irq(&event_queue->lock); +} + +static int +uverbs_completion_event_file_destroy_uobj(struct ib_uobject *uobj, + enum rdma_remove_reason why) +{ + struct ib_uverbs_completion_event_file *file = + container_of(uobj, struct ib_uverbs_completion_event_file, + uobj); + + ib_uverbs_free_event_queue(&file->ev_queue); return 0; -}; +} int uverbs_destroy_def_handler(struct uverbs_attr_bundle *attrs) { diff --git a/drivers/infiniband/core/uverbs_std_types_async_fd.c b/drivers/infiniband/core/uverbs_std_types_async_fd.c new file mode 100644 index 000000000000..31ff96898b06 --- /dev/null +++ b/drivers/infiniband/core/uverbs_std_types_async_fd.c @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* + * Copyright (c) 2019, Mellanox Technologies inc. All rights reserved. + */ + +#include +#include +#include "rdma_core.h" +#include "uverbs.h" + +static int uverbs_async_event_destroy_uobj(struct ib_uobject *uobj, + enum rdma_remove_reason why) +{ + struct ib_uverbs_async_event_file *event_file = + container_of(uobj, struct ib_uverbs_async_event_file, uobj); + + ib_unregister_event_handler(&event_file->event_handler); + ib_uverbs_free_event_queue(&event_file->ev_queue); + return 0; +} + +DECLARE_UVERBS_NAMED_OBJECT( + UVERBS_OBJECT_ASYNC_EVENT, + UVERBS_TYPE_ALLOC_FD(sizeof(struct ib_uverbs_async_event_file), + uverbs_async_event_destroy_uobj, + &uverbs_async_event_fops, + "[infinibandevent]", + O_RDONLY)); + +const struct uapi_definition uverbs_def_obj_async_fd[] = { + UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_ASYNC_EVENT), + {} +}; diff --git a/drivers/infiniband/core/uverbs_uapi.c b/drivers/infiniband/core/uverbs_uapi.c index 9b84a126187a..3f121ac31e0a 100644 --- a/drivers/infiniband/core/uverbs_uapi.c +++ b/drivers/infiniband/core/uverbs_uapi.c @@ -626,6 +626,7 @@ void uverbs_destroy_api(struct uverbs_api *uapi) } static const struct uapi_definition uverbs_core_api[] = { + UAPI_DEF_CHAIN(uverbs_def_obj_async_fd), UAPI_DEF_CHAIN(uverbs_def_obj_counters), UAPI_DEF_CHAIN(uverbs_def_obj_cq), UAPI_DEF_CHAIN(uverbs_def_obj_device), diff --git a/include/uapi/rdma/ib_user_ioctl_cmds.h b/include/uapi/rdma/ib_user_ioctl_cmds.h index 64f0e3aacd3f..9cfadb5120d9 100644 --- a/include/uapi/rdma/ib_user_ioctl_cmds.h +++ b/include/uapi/rdma/ib_user_ioctl_cmds.h @@ -56,6 +56,7 @@ enum uverbs_default_objects { UVERBS_OBJECT_FLOW_ACTION, UVERBS_OBJECT_DM, UVERBS_OBJECT_COUNTERS, + UVERBS_OBJECT_ASYNC_EVENT, }; enum { -- cgit v1.2.3-59-g8ed1b From 5c55cfd6a553d008fcd54e4a4e3fed1340ee5090 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 8 Jan 2020 19:22:06 +0200 Subject: RDMA/core: Use READ_ONCE for ib_ufile.async_file The writer for async_file holds the ucontext_lock, while the readers are left unlocked. Most readers rely on an implicit locking, either by having a uobject (which cannot be created before a context) or by holding the ib_ufile kref. However ib_uverbs_free_hw_resources() has no implicit lock and has a possible race. Make this all clear and sane by using READ_ONCE consistently. Link: https://lore.kernel.org/r/1578504126-9400-15-git-send-email-yishaih@mellanox.com Signed-off-by: Yishai Hadas Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/uverbs.h | 6 ++---- drivers/infiniband/core/uverbs_cmd.c | 2 +- drivers/infiniband/core/uverbs_main.c | 29 +++++++++++---------------- drivers/infiniband/core/uverbs_std_types.c | 6 +++--- drivers/infiniband/core/uverbs_std_types_cq.c | 1 - 5 files changed, 18 insertions(+), 26 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h index ccde5d20a6cf..aaa5c7550913 100644 --- a/drivers/infiniband/core/uverbs.h +++ b/drivers/infiniband/core/uverbs.h @@ -220,11 +220,9 @@ void ib_uverbs_init_async_event_file(struct ib_uverbs_async_event_file *ev_file) void ib_uverbs_free_event_queue(struct ib_uverbs_event_queue *event_queue); void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res); -void ib_uverbs_release_ucq(struct ib_uverbs_file *file, - struct ib_uverbs_completion_event_file *ev_file, +void ib_uverbs_release_ucq(struct ib_uverbs_completion_event_file *ev_file, struct ib_ucq_object *uobj); -void ib_uverbs_release_uevent(struct ib_uverbs_file *file, - struct ib_uevent_object *uobj); +void ib_uverbs_release_uevent(struct ib_uevent_object *uobj); void ib_uverbs_release_file(struct kref *ref); void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context); diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index ced1384d316b..29b1b5ad8836 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -1056,7 +1056,7 @@ err_free: kfree(cq); err_file: if (ev_file) - ib_uverbs_release_ucq(attrs->ufile, ev_file, obj); + ib_uverbs_release_ucq(ev_file, obj); err: uobj_alloc_abort(&obj->uevent.uobject, attrs); diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index 121e65f69c0b..1f279b0a8e49 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -125,9 +125,8 @@ static void ib_uverbs_release_dev(struct device *device) kfree(dev); } -void ib_uverbs_release_ucq(struct ib_uverbs_file *file, - struct ib_uverbs_completion_event_file *ev_file, - struct ib_ucq_object *uobj) +void ib_uverbs_release_ucq(struct ib_uverbs_completion_event_file *ev_file, + struct ib_ucq_object *uobj) { struct ib_uverbs_event *evt, *tmp; @@ -142,20 +141,21 @@ void ib_uverbs_release_ucq(struct ib_uverbs_file *file, uverbs_uobject_put(&ev_file->uobj); } - ib_uverbs_release_uevent(file, &uobj->uevent); + ib_uverbs_release_uevent(&uobj->uevent); } -void ib_uverbs_release_uevent(struct ib_uverbs_file *file, - struct ib_uevent_object *uobj) +void ib_uverbs_release_uevent(struct ib_uevent_object *uobj) { + struct ib_uverbs_async_event_file *async_file = + READ_ONCE(uobj->uobject.ufile->async_file); struct ib_uverbs_event *evt, *tmp; - spin_lock_irq(&file->async_file->ev_queue.lock); + spin_lock_irq(&async_file->ev_queue.lock); list_for_each_entry_safe(evt, tmp, &uobj->event_list, obj_list) { list_del(&evt->list); kfree(evt); } - spin_unlock_irq(&file->async_file->ev_queue.lock); + spin_unlock_irq(&async_file->ev_queue.lock); } void ib_uverbs_detach_umcast(struct ib_qp *qp, @@ -420,7 +420,7 @@ ib_uverbs_async_handler(struct ib_uverbs_async_event_file *async_file, static void uverbs_uobj_event(struct ib_uevent_object *eobj, struct ib_event *event) { - ib_uverbs_async_handler(eobj->uobject.ufile->async_file, + ib_uverbs_async_handler(READ_ONCE(eobj->uobject.ufile->async_file), eobj->uobject.user_handle, event->event, &eobj->event_list, &eobj->events_reported); } @@ -476,9 +476,9 @@ void ib_uverbs_init_async_event_file( ib_uverbs_init_event_queue(&async_file->ev_queue); if (!WARN_ON(uverbs_file->async_file)) { - uverbs_file->async_file = async_file; /* Pairs with the put in ib_uverbs_release_file */ uverbs_uobject_get(&async_file->uobj); + smp_store_release(&uverbs_file->async_file, async_file); } INIT_IB_EVENT_HANDLER(&async_file->event_handler, ib_dev, @@ -1156,13 +1156,9 @@ static void ib_uverbs_free_hw_resources(struct ib_uverbs_device *uverbs_dev, struct ib_device *ib_dev) { struct ib_uverbs_file *file; - struct ib_event event; /* Pending running commands to terminate */ uverbs_disassociate_api_pre(uverbs_dev); - event.event = IB_EVENT_DEVICE_FATAL; - event.element.port_num = 0; - event.device = ib_dev; mutex_lock(&uverbs_dev->lists_mutex); while (!list_empty(&uverbs_dev->uverbs_file_list)) { @@ -1178,9 +1174,8 @@ static void ib_uverbs_free_hw_resources(struct ib_uverbs_device *uverbs_dev, */ mutex_unlock(&uverbs_dev->lists_mutex); - if (file->async_file) - ib_uverbs_event_handler( - &file->async_file->event_handler, &event); + ib_uverbs_async_handler(READ_ONCE(file->async_file), 0, + IB_EVENT_DEVICE_FATAL, NULL, NULL); uverbs_destroy_ufile_hw(file, RDMA_REMOVE_DRIVER_REMOVE); kref_put(&file->ref, ib_uverbs_release_file); diff --git a/drivers/infiniband/core/uverbs_std_types.c b/drivers/infiniband/core/uverbs_std_types.c index efe70bcf79b1..994d8744b246 100644 --- a/drivers/infiniband/core/uverbs_std_types.c +++ b/drivers/infiniband/core/uverbs_std_types.c @@ -105,7 +105,7 @@ static int uverbs_free_qp(struct ib_uobject *uobject, if (uqp->uxrcd) atomic_dec(&uqp->uxrcd->refcnt); - ib_uverbs_release_uevent(attrs->ufile, &uqp->uevent); + ib_uverbs_release_uevent(&uqp->uevent); return ret; } @@ -138,7 +138,7 @@ static int uverbs_free_wq(struct ib_uobject *uobject, if (ib_is_destroy_retryable(ret, why, uobject)) return ret; - ib_uverbs_release_uevent(attrs->ufile, &uwq->uevent); + ib_uverbs_release_uevent(&uwq->uevent); return ret; } @@ -163,7 +163,7 @@ static int uverbs_free_srq(struct ib_uobject *uobject, atomic_dec(&us->uxrcd->refcnt); } - ib_uverbs_release_uevent(attrs->ufile, uevent); + ib_uverbs_release_uevent(uevent); return ret; } diff --git a/drivers/infiniband/core/uverbs_std_types_cq.c b/drivers/infiniband/core/uverbs_std_types_cq.c index a41c758042cc..da4110a0eea2 100644 --- a/drivers/infiniband/core/uverbs_std_types_cq.c +++ b/drivers/infiniband/core/uverbs_std_types_cq.c @@ -49,7 +49,6 @@ static int uverbs_free_cq(struct ib_uobject *uobject, return ret; ib_uverbs_release_ucq( - attrs->ufile, ev_queue ? container_of(ev_queue, struct ib_uverbs_completion_event_file, ev_queue) : -- cgit v1.2.3-59-g8ed1b From d680e88e2013186e696665cbf2056fb32b781e41 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 8 Jan 2020 20:05:32 +0200 Subject: RDMA/core: Add UVERBS_METHOD_ASYNC_EVENT_ALLOC Allow the async FD to be allocated separately from the context. This is necessary to introduce the ioctl to create a context, as an ioctl should only ever create a single uobject at a time. If multiple async FDs are created then the first one is used to deliver affiliated events from any ib_uevent_object, with all subsequent ones will receive only unaffiliated events. Link: https://lore.kernel.org/r/1578506740-22188-3-git-send-email-yishaih@mellanox.com Signed-off-by: Yishai Hadas Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/uverbs_main.c | 4 +++- .../infiniband/core/uverbs_std_types_async_fd.c | 23 +++++++++++++++++++++- include/uapi/rdma/ib_user_ioctl_cmds.h | 8 ++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index 1f279b0a8e49..fb9e75257607 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -475,7 +475,9 @@ void ib_uverbs_init_async_event_file( ib_uverbs_init_event_queue(&async_file->ev_queue); - if (!WARN_ON(uverbs_file->async_file)) { + /* The first async_event_file becomes the default one for the file. */ + lockdep_assert_held(&uverbs_file->ucontext_lock); + if (!uverbs_file->async_file) { /* Pairs with the put in ib_uverbs_release_file */ uverbs_uobject_get(&async_file->uobj); smp_store_release(&uverbs_file->async_file, async_file); diff --git a/drivers/infiniband/core/uverbs_std_types_async_fd.c b/drivers/infiniband/core/uverbs_std_types_async_fd.c index 31ff96898b06..484dba136950 100644 --- a/drivers/infiniband/core/uverbs_std_types_async_fd.c +++ b/drivers/infiniband/core/uverbs_std_types_async_fd.c @@ -8,6 +8,19 @@ #include "rdma_core.h" #include "uverbs.h" +static int UVERBS_HANDLER(UVERBS_METHOD_ASYNC_EVENT_ALLOC)( + struct uverbs_attr_bundle *attrs) +{ + struct ib_uobject *uobj = + uverbs_attr_get_uobject(attrs, UVERBS_METHOD_ASYNC_EVENT_ALLOC); + + mutex_lock(&attrs->ufile->ucontext_lock); + ib_uverbs_init_async_event_file( + container_of(uobj, struct ib_uverbs_async_event_file, uobj)); + mutex_unlock(&attrs->ufile->ucontext_lock); + return 0; +} + static int uverbs_async_event_destroy_uobj(struct ib_uobject *uobj, enum rdma_remove_reason why) { @@ -19,13 +32,21 @@ static int uverbs_async_event_destroy_uobj(struct ib_uobject *uobj, return 0; } +DECLARE_UVERBS_NAMED_METHOD( + UVERBS_METHOD_ASYNC_EVENT_ALLOC, + UVERBS_ATTR_FD(UVERBS_ATTR_ASYNC_EVENT_ALLOC_FD_HANDLE, + UVERBS_OBJECT_ASYNC_EVENT, + UVERBS_ACCESS_NEW, + UA_MANDATORY)); + DECLARE_UVERBS_NAMED_OBJECT( UVERBS_OBJECT_ASYNC_EVENT, UVERBS_TYPE_ALLOC_FD(sizeof(struct ib_uverbs_async_event_file), uverbs_async_event_destroy_uobj, &uverbs_async_event_fops, "[infinibandevent]", - O_RDONLY)); + O_RDONLY), + &UVERBS_METHOD(UVERBS_METHOD_ASYNC_EVENT_ALLOC)); const struct uapi_definition uverbs_def_obj_async_fd[] = { UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_ASYNC_EVENT), diff --git a/include/uapi/rdma/ib_user_ioctl_cmds.h b/include/uapi/rdma/ib_user_ioctl_cmds.h index 9cfadb5120d9..498955c576f3 100644 --- a/include/uapi/rdma/ib_user_ioctl_cmds.h +++ b/include/uapi/rdma/ib_user_ioctl_cmds.h @@ -242,4 +242,12 @@ enum uverbs_attrs_flow_destroy_ids { UVERBS_ATTR_DESTROY_FLOW_HANDLE, }; +enum uverbs_method_async_event { + UVERBS_METHOD_ASYNC_EVENT_ALLOC, +}; + +enum uverbs_attrs_async_event_create { + UVERBS_ATTR_ASYNC_EVENT_ALLOC_FD_HANDLE, +}; + #endif -- cgit v1.2.3-59-g8ed1b From da57db25677f8566a354b0d0e4973a0fdcabcf84 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 8 Jan 2020 20:05:33 +0200 Subject: RDMA/core: Remove ucontext_lock from the uverbs_destry_ufile_hw() path This lock only serializes ucontext creation. Instead of checking the ucontext_lock during destruction hold the existing hw_destroy_rwsem during creation, which is the standard pattern for object creation. The simplification of locking is needed for the next patch. Link: https://lore.kernel.org/r/1578506740-22188-4-git-send-email-yishaih@mellanox.com Signed-off-by: Yishai Hadas Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/rdma_core.c | 21 +-------------------- drivers/infiniband/core/uverbs_cmd.c | 5 ++++- 2 files changed, 5 insertions(+), 21 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/rdma_core.c b/drivers/infiniband/core/rdma_core.c index 50251da123f7..5128cb16bb48 100644 --- a/drivers/infiniband/core/rdma_core.c +++ b/drivers/infiniband/core/rdma_core.c @@ -845,9 +845,7 @@ static int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile, } /* - * Destroy the uncontext and every uobject associated with it. If called with - * reason != RDMA_REMOVE_CLOSE this will not return until the destruction has - * been completed and ufile->ucontext is NULL. + * Destroy the uncontext and every uobject associated with it. * * This is internally locked and can be called in parallel from multiple * contexts. @@ -855,22 +853,6 @@ static int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile, void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile, enum rdma_remove_reason reason) { - if (reason == RDMA_REMOVE_CLOSE) { - /* - * During destruction we might trigger something that - * synchronously calls release on any file descriptor. For - * this reason all paths that come from file_operations - * release must use try_lock. They can progress knowing that - * there is an ongoing uverbs_destroy_ufile_hw that will clean - * up the driver resources. - */ - if (!mutex_trylock(&ufile->ucontext_lock)) - return; - - } else { - mutex_lock(&ufile->ucontext_lock); - } - down_write(&ufile->hw_destroy_rwsem); /* @@ -899,7 +881,6 @@ void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile, done: up_write(&ufile->hw_destroy_rwsem); - mutex_unlock(&ufile->ucontext_lock); } const struct uverbs_obj_type_class uverbs_fd_class = { diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index 29b1b5ad8836..d71ffe44b8ae 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -218,6 +218,8 @@ static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs) if (ret) return ret; + if (!down_read_trylock(&file->hw_destroy_rwsem)) + return -EIO; mutex_lock(&file->ucontext_lock); ib_dev = srcu_dereference(file->device->ib_dev, &file->device->disassociate_srcu); @@ -284,7 +286,7 @@ static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs) smp_store_release(&file->ucontext, ucontext); mutex_unlock(&file->ucontext_lock); - + up_read(&file->hw_destroy_rwsem); return 0; err_uobj: @@ -298,6 +300,7 @@ err_alloc: err: mutex_unlock(&file->ucontext_lock); + up_read(&file->hw_destroy_rwsem); return ret; } -- cgit v1.2.3-59-g8ed1b From a1123418ba1078037d9fecb72573ff7222dfe201 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 8 Jan 2020 20:05:34 +0200 Subject: RDMA/uverbs: Add ioctl command to get a device context Allow future extensions of the get context command through the uverbs ioctl kabi. Unlike the uverbs version this does not return an async_fd as well, that has to be done with another command. Link: https://lore.kernel.org/r/1578506740-22188-5-git-send-email-yishaih@mellanox.com Signed-off-by: Yishai Hadas Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/uverbs.h | 3 + drivers/infiniband/core/uverbs_cmd.c | 132 ++++++++++++--------- drivers/infiniband/core/uverbs_main.c | 9 +- .../infiniband/core/uverbs_std_types_async_fd.c | 2 - drivers/infiniband/core/uverbs_std_types_device.c | 30 +++++ include/uapi/rdma/ib_user_ioctl_cmds.h | 5 + 6 files changed, 119 insertions(+), 62 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h index aaa5c7550913..4d4cec46d251 100644 --- a/drivers/infiniband/core/uverbs.h +++ b/drivers/infiniband/core/uverbs.h @@ -220,6 +220,9 @@ void ib_uverbs_init_async_event_file(struct ib_uverbs_async_event_file *ev_file) void ib_uverbs_free_event_queue(struct ib_uverbs_event_queue *event_queue); void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res); +int ib_alloc_ucontext(struct uverbs_attr_bundle *attrs); +int ib_init_ucontext(struct uverbs_attr_bundle *attrs); + void ib_uverbs_release_ucq(struct ib_uverbs_completion_event_file *ev_file, struct ib_ucq_object *uobj); void ib_uverbs_release_uevent(struct ib_uevent_object *uobj); diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index d71ffe44b8ae..c8693f5231dd 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -203,104 +203,118 @@ _ib_uverbs_lookup_comp_file(s32 fd, struct uverbs_attr_bundle *attrs) #define ib_uverbs_lookup_comp_file(_fd, _ufile) \ _ib_uverbs_lookup_comp_file((_fd)*typecheck(s32, _fd), _ufile) -static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs) +int ib_alloc_ucontext(struct uverbs_attr_bundle *attrs) { - struct ib_uverbs_file *file = attrs->ufile; - struct ib_uverbs_get_context cmd; - struct ib_uverbs_get_context_resp resp; - struct ib_ucontext *ucontext; - struct ib_rdmacg_object cg_obj; + struct ib_uverbs_file *ufile = attrs->ufile; + struct ib_ucontext *ucontext; struct ib_device *ib_dev; - struct ib_uobject *uobj; - int ret; - ret = uverbs_request(attrs, &cmd, sizeof(cmd)); - if (ret) - return ret; + ib_dev = srcu_dereference(ufile->device->ib_dev, + &ufile->device->disassociate_srcu); + if (!ib_dev) + return -EIO; + + ucontext = rdma_zalloc_drv_obj(ib_dev, ib_ucontext); + if (!ucontext) + return -ENOMEM; + + ucontext->res.type = RDMA_RESTRACK_CTX; + ucontext->device = ib_dev; + ucontext->ufile = ufile; + xa_init_flags(&ucontext->mmap_xa, XA_FLAGS_ALLOC); + attrs->context = ucontext; + return 0; +} + +int ib_init_ucontext(struct uverbs_attr_bundle *attrs) +{ + struct ib_ucontext *ucontext = attrs->context; + struct ib_uverbs_file *file = attrs->ufile; + int ret; if (!down_read_trylock(&file->hw_destroy_rwsem)) return -EIO; mutex_lock(&file->ucontext_lock); - ib_dev = srcu_dereference(file->device->ib_dev, - &file->device->disassociate_srcu); - if (!ib_dev) { - ret = -EIO; - goto err; - } - if (file->ucontext) { ret = -EINVAL; goto err; } - ret = ib_rdmacg_try_charge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE); + ret = ib_rdmacg_try_charge(&ucontext->cg_obj, ucontext->device, + RDMACG_RESOURCE_HCA_HANDLE); if (ret) goto err; - ucontext = rdma_zalloc_drv_obj(ib_dev, ib_ucontext); - if (!ucontext) { - ret = -ENOMEM; - goto err_alloc; - } + ret = ucontext->device->ops.alloc_ucontext(ucontext, + &attrs->driver_udata); + if (ret) + goto err_uncharge; - attrs->context = ucontext; + rdma_restrack_uadd(&ucontext->res); - ucontext->res.type = RDMA_RESTRACK_CTX; - ucontext->device = ib_dev; - ucontext->cg_obj = cg_obj; - /* ufile is required when some objects are released */ - ucontext->ufile = file; + /* + * Make sure that ib_uverbs_get_ucontext() sees the pointer update + * only after all writes to setup the ucontext have completed + */ + smp_store_release(&file->ucontext, ucontext); + + mutex_unlock(&file->ucontext_lock); + up_read(&file->hw_destroy_rwsem); + return 0; - ucontext->closing = false; - ucontext->cleanup_retryable = false; +err_uncharge: + ib_rdmacg_uncharge(&ucontext->cg_obj, ucontext->device, + RDMACG_RESOURCE_HCA_HANDLE); +err: + mutex_unlock(&file->ucontext_lock); + up_read(&file->hw_destroy_rwsem); + return ret; +} - xa_init_flags(&ucontext->mmap_xa, XA_FLAGS_ALLOC); +static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs) +{ + struct ib_uverbs_get_context_resp resp; + struct ib_uverbs_get_context cmd; + struct ib_device *ib_dev; + struct ib_uobject *uobj; + int ret; + + ret = uverbs_request(attrs, &cmd, sizeof(cmd)); + if (ret) + return ret; + + ret = ib_alloc_ucontext(attrs); + if (ret) + return ret; uobj = uobj_alloc(UVERBS_OBJECT_ASYNC_EVENT, attrs, &ib_dev); if (IS_ERR(uobj)) { ret = PTR_ERR(uobj); - goto err_free; + goto err_ucontext; } - resp.async_fd = uobj->id; - resp.num_comp_vectors = file->device->num_comp_vectors; - + resp = (struct ib_uverbs_get_context_resp){ + .num_comp_vectors = attrs->ufile->device->num_comp_vectors, + .async_fd = uobj->id, + }; ret = uverbs_response(attrs, &resp, sizeof(resp)); if (ret) goto err_uobj; - ret = ib_dev->ops.alloc_ucontext(ucontext, &attrs->driver_udata); + ret = ib_init_ucontext(attrs); if (ret) goto err_uobj; - rdma_restrack_uadd(&ucontext->res); - ib_uverbs_init_async_event_file( container_of(uobj, struct ib_uverbs_async_event_file, uobj)); rdma_alloc_commit_uobject(uobj, attrs); - - /* - * Make sure that ib_uverbs_get_ucontext() sees the pointer update - * only after all writes to setup the ucontext have completed - */ - smp_store_release(&file->ucontext, ucontext); - - mutex_unlock(&file->ucontext_lock); - up_read(&file->hw_destroy_rwsem); return 0; err_uobj: rdma_alloc_abort_uobject(uobj, attrs); - -err_free: - kfree(ucontext); - -err_alloc: - ib_rdmacg_uncharge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE); - -err: - mutex_unlock(&file->ucontext_lock); - up_read(&file->hw_destroy_rwsem); +err_ucontext: + kfree(attrs->context); + attrs->context = NULL; return ret; } diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index fb9e75257607..2d4083bf4a04 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -150,6 +150,9 @@ void ib_uverbs_release_uevent(struct ib_uevent_object *uobj) READ_ONCE(uobj->uobject.ufile->async_file); struct ib_uverbs_event *evt, *tmp; + if (!async_file) + return; + spin_lock_irq(&async_file->ev_queue.lock); list_for_each_entry_safe(evt, tmp, &uobj->event_list, obj_list) { list_del(&evt->list); @@ -391,6 +394,9 @@ ib_uverbs_async_handler(struct ib_uverbs_async_event_file *async_file, struct ib_uverbs_event *entry; unsigned long flags; + if (!async_file) + return; + spin_lock_irqsave(&async_file->ev_queue.lock, flags); if (async_file->ev_queue.is_closed) { spin_unlock_irqrestore(&async_file->ev_queue.lock, flags); @@ -476,12 +482,13 @@ void ib_uverbs_init_async_event_file( ib_uverbs_init_event_queue(&async_file->ev_queue); /* The first async_event_file becomes the default one for the file. */ - lockdep_assert_held(&uverbs_file->ucontext_lock); + mutex_lock(&uverbs_file->ucontext_lock); if (!uverbs_file->async_file) { /* Pairs with the put in ib_uverbs_release_file */ uverbs_uobject_get(&async_file->uobj); smp_store_release(&uverbs_file->async_file, async_file); } + mutex_unlock(&uverbs_file->ucontext_lock); INIT_IB_EVENT_HANDLER(&async_file->event_handler, ib_dev, ib_uverbs_event_handler); diff --git a/drivers/infiniband/core/uverbs_std_types_async_fd.c b/drivers/infiniband/core/uverbs_std_types_async_fd.c index 484dba136950..82ec0806b34b 100644 --- a/drivers/infiniband/core/uverbs_std_types_async_fd.c +++ b/drivers/infiniband/core/uverbs_std_types_async_fd.c @@ -14,10 +14,8 @@ static int UVERBS_HANDLER(UVERBS_METHOD_ASYNC_EVENT_ALLOC)( struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, UVERBS_METHOD_ASYNC_EVENT_ALLOC); - mutex_lock(&attrs->ufile->ucontext_lock); ib_uverbs_init_async_event_file( container_of(uobj, struct ib_uverbs_async_event_file, uobj)); - mutex_unlock(&attrs->ufile->ucontext_lock); return 0; } diff --git a/drivers/infiniband/core/uverbs_std_types_device.c b/drivers/infiniband/core/uverbs_std_types_device.c index 2a3f2f01028d..2c5943548760 100644 --- a/drivers/infiniband/core/uverbs_std_types_device.c +++ b/drivers/infiniband/core/uverbs_std_types_device.c @@ -200,6 +200,35 @@ static int UVERBS_HANDLER(UVERBS_METHOD_QUERY_PORT)( &resp, sizeof(resp)); } +static int UVERBS_HANDLER(UVERBS_METHOD_GET_CONTEXT)( + struct uverbs_attr_bundle *attrs) +{ + u32 num_comp = attrs->ufile->device->num_comp_vectors; + int ret; + + ret = uverbs_copy_to(attrs, UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS, + &num_comp, sizeof(num_comp)); + if (IS_UVERBS_COPY_ERR(ret)) + return ret; + + ret = ib_alloc_ucontext(attrs); + if (ret) + return ret; + ret = ib_init_ucontext(attrs); + if (ret) { + kfree(attrs->context); + attrs->context = NULL; + return ret; + } + return 0; +} + +DECLARE_UVERBS_NAMED_METHOD( + UVERBS_METHOD_GET_CONTEXT, + UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS, + UVERBS_ATTR_TYPE(u32), UA_OPTIONAL), + UVERBS_ATTR_UHW()); + DECLARE_UVERBS_NAMED_METHOD( UVERBS_METHOD_INFO_HANDLES, /* Also includes any device specific object ids */ @@ -220,6 +249,7 @@ DECLARE_UVERBS_NAMED_METHOD( UA_MANDATORY)); DECLARE_UVERBS_GLOBAL_METHODS(UVERBS_OBJECT_DEVICE, + &UVERBS_METHOD(UVERBS_METHOD_GET_CONTEXT), &UVERBS_METHOD(UVERBS_METHOD_INVOKE_WRITE), &UVERBS_METHOD(UVERBS_METHOD_INFO_HANDLES), &UVERBS_METHOD(UVERBS_METHOD_QUERY_PORT)); diff --git a/include/uapi/rdma/ib_user_ioctl_cmds.h b/include/uapi/rdma/ib_user_ioctl_cmds.h index 498955c576f3..da6c63c8cd32 100644 --- a/include/uapi/rdma/ib_user_ioctl_cmds.h +++ b/include/uapi/rdma/ib_user_ioctl_cmds.h @@ -68,6 +68,7 @@ enum uverbs_methods_device { UVERBS_METHOD_INVOKE_WRITE, UVERBS_METHOD_INFO_HANDLES, UVERBS_METHOD_QUERY_PORT, + UVERBS_METHOD_GET_CONTEXT, }; enum uverbs_attrs_invoke_write_cmd_attr_ids { @@ -81,6 +82,10 @@ enum uverbs_attrs_query_port_cmd_attr_ids { UVERBS_ATTR_QUERY_PORT_RESP, }; +enum uverbs_attrs_get_context_attr_ids { + UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS, +}; + enum uverbs_attrs_create_cq_cmd_attr_ids { UVERBS_ATTR_CREATE_CQ_HANDLE, UVERBS_ATTR_CREATE_CQ_CQE, -- cgit v1.2.3-59-g8ed1b From 811646998e3e64af19524c8e99b790a4043c5f9b Mon Sep 17 00:00:00 2001 From: Michael Guralnik Date: Wed, 8 Jan 2020 20:05:39 +0200 Subject: RDMA/core: Add the core support field to METHOD_GET_CONTEXT Add the core support field to METHOD_GET_CONTEXT, this field should represent capabilities that are not device-specific. Return support for optional access flags for memory regions. User-space will use this capability to mask the optional access flags for unsupporting kernels. Link: https://lore.kernel.org/r/1578506740-22188-10-git-send-email-yishaih@mellanox.com Signed-off-by: Michael Guralnik Signed-off-by: Yishai Hadas Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/uverbs_std_types_device.c | 8 ++++++++ include/uapi/rdma/ib_user_ioctl_cmds.h | 1 + include/uapi/rdma/ib_user_ioctl_verbs.h | 4 ++++ 3 files changed, 13 insertions(+) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/uverbs_std_types_device.c b/drivers/infiniband/core/uverbs_std_types_device.c index 2c5943548760..ae4a59d6f9b1 100644 --- a/drivers/infiniband/core/uverbs_std_types_device.c +++ b/drivers/infiniband/core/uverbs_std_types_device.c @@ -204,6 +204,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_GET_CONTEXT)( struct uverbs_attr_bundle *attrs) { u32 num_comp = attrs->ufile->device->num_comp_vectors; + u64 core_support = IB_UVERBS_CORE_SUPPORT_OPTIONAL_MR_ACCESS; int ret; ret = uverbs_copy_to(attrs, UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS, @@ -211,6 +212,11 @@ static int UVERBS_HANDLER(UVERBS_METHOD_GET_CONTEXT)( if (IS_UVERBS_COPY_ERR(ret)) return ret; + ret = uverbs_copy_to(attrs, UVERBS_ATTR_GET_CONTEXT_CORE_SUPPORT, + &core_support, sizeof(core_support)); + if (IS_UVERBS_COPY_ERR(ret)) + return ret; + ret = ib_alloc_ucontext(attrs); if (ret) return ret; @@ -227,6 +233,8 @@ DECLARE_UVERBS_NAMED_METHOD( UVERBS_METHOD_GET_CONTEXT, UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS, UVERBS_ATTR_TYPE(u32), UA_OPTIONAL), + UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_GET_CONTEXT_CORE_SUPPORT, + UVERBS_ATTR_TYPE(u64), UA_OPTIONAL), UVERBS_ATTR_UHW()); DECLARE_UVERBS_NAMED_METHOD( diff --git a/include/uapi/rdma/ib_user_ioctl_cmds.h b/include/uapi/rdma/ib_user_ioctl_cmds.h index da6c63c8cd32..d4ddbe4e696c 100644 --- a/include/uapi/rdma/ib_user_ioctl_cmds.h +++ b/include/uapi/rdma/ib_user_ioctl_cmds.h @@ -84,6 +84,7 @@ enum uverbs_attrs_query_port_cmd_attr_ids { enum uverbs_attrs_get_context_attr_ids { UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS, + UVERBS_ATTR_GET_CONTEXT_CORE_SUPPORT, }; enum uverbs_attrs_create_cq_cmd_attr_ids { diff --git a/include/uapi/rdma/ib_user_ioctl_verbs.h b/include/uapi/rdma/ib_user_ioctl_verbs.h index 2a165f40ee38..a640bb814be0 100644 --- a/include/uapi/rdma/ib_user_ioctl_verbs.h +++ b/include/uapi/rdma/ib_user_ioctl_verbs.h @@ -44,6 +44,10 @@ #define IB_UVERBS_ACCESS_OPTIONAL_FIRST (1 << 20) #define IB_UVERBS_ACCESS_OPTIONAL_LAST (1 << 29) +enum ib_uverbs_core_support { + IB_UVERBS_CORE_SUPPORT_OPTIONAL_MR_ACCESS = 1 << 0, +}; + enum ib_uverbs_access_flags { IB_UVERBS_ACCESS_LOCAL_WRITE = 1 << 0, IB_UVERBS_ACCESS_REMOTE_WRITE = 1 << 1, -- cgit v1.2.3-59-g8ed1b From 6b3712c0246ca7b2b8fa05eab2362cf267410f7e Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 15 Jan 2020 20:20:44 +0000 Subject: RDMA/core: Ensure that rdma_user_mmap_entry_remove() is a fence The set of entry->driver_removed is missing locking, protect it with xa_lock() which is held by the only reader. Otherwise readers may continue to see driver_removed = false after rdma_user_mmap_entry_remove() returns and may continue to try and establish new mmaps. Fixes: 3411f9f01b76 ("RDMA/core: Create mmap database and cookie helper functions") Link: https://lore.kernel.org/r/20200115202041.GA17199@ziepe.ca Reviewed-by: Gal Pressman Acked-by: Michal Kalderon Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/ib_core_uverbs.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/ib_core_uverbs.c b/drivers/infiniband/core/ib_core_uverbs.c index b7cb59844ece..b51bd7087a88 100644 --- a/drivers/infiniband/core/ib_core_uverbs.c +++ b/drivers/infiniband/core/ib_core_uverbs.c @@ -232,7 +232,9 @@ void rdma_user_mmap_entry_remove(struct rdma_user_mmap_entry *entry) if (!entry) return; + xa_lock(&entry->ucontext->mmap_xa); entry->driver_removed = true; + xa_unlock(&entry->ucontext->mmap_xa); kref_put(&entry->ref, rdma_user_mmap_entry_free); } EXPORT_SYMBOL(rdma_user_mmap_entry_remove); -- cgit v1.2.3-59-g8ed1b From 792a7c1f2e74bba878a1836cfbe9611f98576d3b Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Thu, 16 Jan 2020 13:00:31 -0400 Subject: RDMA/cm: Add accessors for CM_REQ transport_type Access the two fields through wrappers, like all other fields, to make it clearer what is happening. Link: https://lore.kernel.org/r/20200116170037.30109-2-jgg@ziepe.ca Tested-by: Leon Romanovsky Reviewed-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/cm_msgs.h | 41 +++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/cm_msgs.h b/drivers/infiniband/core/cm_msgs.h index 23e3d0c6a67c..9af9a3212074 100644 --- a/drivers/infiniband/core/cm_msgs.h +++ b/drivers/infiniband/core/cm_msgs.h @@ -124,14 +124,37 @@ static inline void cm_req_set_remote_resp_timeout(struct cm_req_msg *req_msg, 0xFFFFFF07)); } +static inline u8 cm_req_get_transport_type(struct cm_req_msg *req_msg) +{ + return (u8) ((be32_to_cpu(req_msg->offset40) & 0x06) >> 1); +} + +static inline void cm_req_set_transport_type(struct cm_req_msg *req_msg, u8 val) +{ + req_msg->offset40 = + cpu_to_be32((be32_to_cpu(req_msg->offset40) & 0xFFFFFFF9) | + (val << 1)); +} + +static inline u8 cm_req_get_transport_type_ex(struct cm_req_msg *req_msg) +{ + return req_msg->offset51 & 0x7; +} + +static inline void cm_req_set_transport_type_ex(struct cm_req_msg *req_msg, + u8 val) +{ + req_msg->offset51 = (req_msg->offset51 & 0xF8) | val; +} + static inline enum ib_qp_type cm_req_get_qp_type(struct cm_req_msg *req_msg) { - u8 transport_type = (u8) (be32_to_cpu(req_msg->offset40) & 0x06) >> 1; + u8 transport_type = cm_req_get_transport_type(req_msg); switch(transport_type) { case 0: return IB_QPT_RC; case 1: return IB_QPT_UC; case 3: - switch (req_msg->offset51 & 0x7) { + switch (cm_req_get_transport_type_ex(req_msg)) { case 1: return IB_QPT_XRC_TGT; default: return 0; } @@ -144,20 +167,14 @@ static inline void cm_req_set_qp_type(struct cm_req_msg *req_msg, { switch(qp_type) { case IB_QPT_UC: - req_msg->offset40 = cpu_to_be32((be32_to_cpu( - req_msg->offset40) & - 0xFFFFFFF9) | 0x2); + cm_req_set_transport_type(req_msg, 1); break; case IB_QPT_XRC_INI: - req_msg->offset40 = cpu_to_be32((be32_to_cpu( - req_msg->offset40) & - 0xFFFFFFF9) | 0x6); - req_msg->offset51 = (req_msg->offset51 & 0xF8) | 1; + cm_req_set_transport_type(req_msg, 3); + cm_req_set_transport_type_ex(req_msg, 1); break; default: - req_msg->offset40 = cpu_to_be32(be32_to_cpu( - req_msg->offset40) & - 0xFFFFFFF9); + cm_req_set_transport_type(req_msg, 0); } } -- cgit v1.2.3-59-g8ed1b From d05d4ac4c9316a2e2ea5be434a7f7a021ec44a3c Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Thu, 16 Jan 2020 13:00:32 -0400 Subject: RDMA/cm: Add SET/GET implementations to hide IBA wire format There is no separation between RDMA-CM wire format as it is declared in IBTA and kernel logic which implements needed support. Such situation causes to many mistakes in conversion between big-endian (wire format) and CPU format used by kernel. It also mixes RDMA core code with combination of uXX and beXX variables. The idea that all accesses to IBA definitions will go through special GET/SET macros to ensure that no conversion mistakes are made. The shifting and masking required to read the value is automatically deduced using the field offset description from the tables in the IBA specification. This starts with the CM MADs described in IBTA release 1.3 volume 1. To confirm that the new macros behave the same as the old accessors a self-test is included in this patch. Each macro replacing a straightforward struct field compile-time tests that the new field has the same offsetof() and width as the old field. For the fields with accessor functions a runtime test, the 'all ones' value is placed in a dummy message and read back in several ways to confirm that both approaches give identical results. Later patches in this series delete the self test. This creates a tested table of new field name, old field name(s) and some meta information like BE coding for the functions which will be used in the next patches. Link: https://lore.kernel.org/r/20200116170037.30109-3-jgg@ziepe.ca Link: https://lore.kernel.org/r/20191212093830.316934-5-leon@kernel.org Signed-off-by: Leon Romanovsky Tested-by: Leon Romanovsky Reviewed-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/cm.c | 267 ++++++++++++++++++++++++++++++++++++++ drivers/infiniband/core/cm_msgs.h | 1 + include/rdma/iba.h | 146 +++++++++++++++++++++ include/rdma/ibta_vol1_c12.h | 208 +++++++++++++++++++++++++++++ 4 files changed, 622 insertions(+) create mode 100644 include/rdma/iba.h create mode 100644 include/rdma/ibta_vol1_c12.h (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c index d396b987b0f2..7f609979e4de 100644 --- a/drivers/infiniband/core/cm.c +++ b/drivers/infiniband/core/cm.c @@ -4253,10 +4253,277 @@ static void cm_remove_one(struct ib_device *ib_device, void *client_data) kfree(cm_dev); } +/* + * Check at compile time that the byte offset and length of field old_name in + * the struct matches the byte offset and length in the new macro. + */ +#define _IBA_CHECK_OFF(old_name, field_struct, field_offset, mask, bits) \ + static_assert(offsetof(field_struct, old_name) == (field_offset)); \ + static_assert(bits == sizeof(((field_struct *)0)->old_name) * 8) +#define IBA_CHECK_OFF(field, old_name) _IBA_CHECK_OFF(old_name, field) + +IBA_CHECK_OFF(CM_REQ_LOCAL_COMM_ID, local_comm_id); +IBA_CHECK_OFF(CM_REQ_SERVICE_ID, service_id); +IBA_CHECK_OFF(CM_REQ_LOCAL_CA_GUID, local_ca_guid); +IBA_CHECK_OFF(CM_REQ_LOCAL_Q_KEY, local_qkey); +IBA_CHECK_OFF(CM_REQ_PARTITION_KEY, pkey); +IBA_CHECK_OFF(CM_REQ_PRIMARY_LOCAL_PORT_LID, primary_local_lid); +IBA_CHECK_OFF(CM_REQ_PRIMARY_REMOTE_PORT_LID, primary_remote_lid); +IBA_CHECK_OFF(CM_REQ_PRIMARY_LOCAL_PORT_GID, primary_local_gid); +IBA_CHECK_OFF(CM_REQ_PRIMARY_REMOTE_PORT_GID, primary_remote_gid); +IBA_CHECK_OFF(CM_REQ_PRIMARY_TRAFFIC_CLASS, primary_traffic_class); +IBA_CHECK_OFF(CM_REQ_PRIMARY_HOP_LIMIT, primary_hop_limit); +IBA_CHECK_OFF(CM_REQ_ALTERNATE_LOCAL_PORT_LID, alt_local_lid); +IBA_CHECK_OFF(CM_REQ_ALTERNATE_REMOTE_PORT_LID, alt_remote_lid); +IBA_CHECK_OFF(CM_REQ_ALTERNATE_LOCAL_PORT_GID, alt_local_gid); +IBA_CHECK_OFF(CM_REQ_ALTERNATE_REMOTE_PORT_GID, alt_remote_gid); +IBA_CHECK_OFF(CM_REQ_ALTERNATE_TRAFFIC_CLASS, alt_traffic_class); +IBA_CHECK_OFF(CM_REQ_ALTERNATE_HOP_LIMIT, alt_hop_limit); +IBA_CHECK_OFF(CM_REQ_PRIVATE_DATA, private_data); +IBA_CHECK_OFF(CM_MRA_LOCAL_COMM_ID, local_comm_id); +IBA_CHECK_OFF(CM_MRA_REMOTE_COMM_ID, remote_comm_id); +IBA_CHECK_OFF(CM_MRA_PRIVATE_DATA, private_data); +IBA_CHECK_OFF(CM_REJ_LOCAL_COMM_ID, local_comm_id); +IBA_CHECK_OFF(CM_REJ_REMOTE_COMM_ID, remote_comm_id); +IBA_CHECK_OFF(CM_REJ_REASON, reason); +IBA_CHECK_OFF(CM_REJ_ARI, ari); +IBA_CHECK_OFF(CM_REJ_PRIVATE_DATA, private_data); +IBA_CHECK_OFF(CM_REP_LOCAL_COMM_ID, local_comm_id); +IBA_CHECK_OFF(CM_REP_REMOTE_COMM_ID, remote_comm_id); +IBA_CHECK_OFF(CM_REP_LOCAL_Q_KEY, local_qkey); +IBA_CHECK_OFF(CM_REP_RESPONDER_RESOURCES, resp_resources); +IBA_CHECK_OFF(CM_REP_INITIATOR_DEPTH, initiator_depth); +IBA_CHECK_OFF(CM_REP_LOCAL_CA_GUID, local_ca_guid); +IBA_CHECK_OFF(CM_REP_PRIVATE_DATA, private_data); +IBA_CHECK_OFF(CM_RTU_LOCAL_COMM_ID, local_comm_id); +IBA_CHECK_OFF(CM_RTU_REMOTE_COMM_ID, remote_comm_id); +IBA_CHECK_OFF(CM_RTU_PRIVATE_DATA, private_data); +IBA_CHECK_OFF(CM_DREQ_LOCAL_COMM_ID, local_comm_id); +IBA_CHECK_OFF(CM_DREQ_REMOTE_COMM_ID, remote_comm_id); +IBA_CHECK_OFF(CM_DREQ_PRIVATE_DATA, private_data); +IBA_CHECK_OFF(CM_DREP_LOCAL_COMM_ID, local_comm_id); +IBA_CHECK_OFF(CM_DREP_REMOTE_COMM_ID, remote_comm_id); +IBA_CHECK_OFF(CM_DREP_PRIVATE_DATA, private_data); +IBA_CHECK_OFF(CM_LAP_LOCAL_COMM_ID, local_comm_id); +IBA_CHECK_OFF(CM_LAP_REMOTE_COMM_ID, remote_comm_id); +IBA_CHECK_OFF(CM_LAP_ALTERNATE_LOCAL_PORT_LID, alt_local_lid); +IBA_CHECK_OFF(CM_LAP_ALTERNATE_REMOTE_PORT_LID, alt_remote_lid); +IBA_CHECK_OFF(CM_LAP_ALTERNATE_LOCAL_PORT_GID, alt_local_gid); +IBA_CHECK_OFF(CM_LAP_ALTERNATE_REMOTE_PORT_GID, alt_remote_gid); +IBA_CHECK_OFF(CM_LAP_ALTERNATE_HOP_LIMIT, alt_hop_limit); +IBA_CHECK_OFF(CM_LAP_PRIVATE_DATA, private_data); +IBA_CHECK_OFF(CM_APR_LOCAL_COMM_ID, local_comm_id); +IBA_CHECK_OFF(CM_APR_REMOTE_COMM_ID, remote_comm_id); +IBA_CHECK_OFF(CM_APR_ADDITIONAL_INFORMATION_LENGTH, info_length); +IBA_CHECK_OFF(CM_APR_AR_STATUS, ap_status); +IBA_CHECK_OFF(CM_APR_ADDITIONAL_INFORMATION, info); +IBA_CHECK_OFF(CM_APR_PRIVATE_DATA, private_data); +IBA_CHECK_OFF(CM_SIDR_REQ_REQUESTID, request_id); +IBA_CHECK_OFF(CM_SIDR_REQ_PARTITION_KEY, pkey); +IBA_CHECK_OFF(CM_SIDR_REQ_SERVICEID, service_id); +IBA_CHECK_OFF(CM_SIDR_REQ_PRIVATE_DATA, private_data); +IBA_CHECK_OFF(CM_SIDR_REP_REQUESTID, request_id); +IBA_CHECK_OFF(CM_SIDR_REP_STATUS, status); +IBA_CHECK_OFF(CM_SIDR_REP_ADDITIONAL_INFORMATION_LENGTH, info_length); +IBA_CHECK_OFF(CM_SIDR_REP_SERVICEID, service_id); +IBA_CHECK_OFF(CM_SIDR_REP_Q_KEY, qkey); +IBA_CHECK_OFF(CM_SIDR_REP_ADDITIONAL_INFORMATION, info); +IBA_CHECK_OFF(CM_SIDR_REP_PRIVATE_DATA, private_data); + +/* + * Check that the new macro gets the same bits as the old get function. + * - IBA_SET() IBA_GET and old get_fn all agree on the field width. + * The field width should match what IBA_SET truncates to + * - Reading from an all ones data should not return extra bits + * - Setting '1' should be the same (ie no endian problems) + */ +/* defeat builtin_constant checks */ +u64 cm_global_all_ones = 0xffffffffffffffffULL; +#define _IBA_CHECK_GET(fn, field_struct, field_offset, mask, bits) \ + ({ \ + field_struct *lmsg = (field_struct *)msg; \ + unsigned long long all_ones; \ + static_assert(sizeof(*lmsg) <= sizeof(msg)); \ + \ + bitmap_zero(msg, nbits); \ + _IBA_SET(field_struct, field_offset, mask, bits, lmsg, \ + cm_global_all_ones); \ + all_ones = (1ULL << bitmap_weight(msg, nbits)) - 1; \ + if (_IBA_GET(field_struct, field_offset, mask, bits, lmsg) != \ + all_ones) { \ + printk("Failed #1 line=%u\n", __LINE__); \ + return; \ + } \ + if (fn != all_ones) { \ + printk("Failed #2 line=%u\n", __LINE__); \ + return; \ + } \ + \ + bitmap_fill(msg, nbits); \ + if (_IBA_GET(field_struct, field_offset, mask, bits, lmsg) != \ + all_ones) { \ + printk("Failed #3 line=%u\n", __LINE__); \ + return; \ + } \ + if (fn != all_ones) { \ + printk("Failed #4 line=%u\n", __LINE__); \ + return; \ + } \ + \ + _IBA_SET(field_struct, field_offset, mask, bits, lmsg, 0); \ + if (_IBA_GET(field_struct, field_offset, mask, bits, lmsg) != \ + 0) { \ + printk("Failed #5 line=%u\n", __LINE__); \ + return; \ + } \ + if (fn != 0) { \ + printk("Failed #6 line=%u\n", __LINE__); \ + return; \ + } \ + _IBA_SET(field_struct, field_offset, mask, bits, lmsg, 1); \ + if (_IBA_GET(field_struct, field_offset, mask, bits, lmsg) != \ + 1) { \ + printk("Failed #7 line=%u\n", __LINE__); \ + return; \ + } \ + if (fn != 1) { \ + printk("Failed #8 line=%u\n", __LINE__); \ + return; \ + } \ + }) +#define IBA_CHECK_GET(field, fn_name) _IBA_CHECK_GET(fn_name(lmsg), field) +#define IBA_CHECK_GET_BE(field, fn_name) _IBA_CHECK_GET(be32_to_cpu(fn_name(lmsg)), field) + +/* + * Write the all ones value using the old setter and check that the new getter + * reads it back. + */ +#define _IBA_CHECK_SET(fn, field_struct, field_offset, mask, bits) \ + ({ \ + field_struct *lmsg = (field_struct *)msg; \ + unsigned long long all_ones; \ + static_assert(sizeof(*lmsg) <= sizeof(msg)); \ + \ + bitmap_zero(msg, nbits); \ + _IBA_SET(field_struct, field_offset, mask, bits, lmsg, \ + cm_global_all_ones); \ + all_ones = (1ULL << bitmap_weight(msg, nbits)) - 1; \ + bitmap_zero(msg, nbits); \ + fn; \ + if (_IBA_GET(field_struct, field_offset, mask, bits, lmsg) != \ + all_ones) { \ + printk("Failed #9 line=%u\n", __LINE__); \ + return; \ + } \ + all_ones = 1; \ + fn; \ + if (_IBA_GET(field_struct, field_offset, mask, bits, lmsg) != \ + 1) { \ + printk("Failed #10 line=%u\n", __LINE__); \ + return; \ + } \ + }) + +#define IBA_CHECK_SET(field, fn_name) _IBA_CHECK_SET(fn_name(lmsg, all_ones), field) +#define IBA_CHECK_SET_BE(field, fn_name) \ + _IBA_CHECK_SET(fn_name(lmsg, cpu_to_be32(all_ones)), field) + +static void self_test(void) +{ + unsigned long msg[256/4]; + const unsigned int nbits = sizeof(msg) * 8; + + printk("Running CM extractor self test\n"); + IBA_CHECK_GET_BE(CM_REQ_LOCAL_QPN, cm_req_get_local_qpn); + IBA_CHECK_SET_BE(CM_REQ_LOCAL_QPN, cm_req_set_local_qpn); + IBA_CHECK_GET(CM_REQ_RESPONDER_RESOURCES, cm_req_get_resp_res); + IBA_CHECK_SET(CM_REQ_RESPONDER_RESOURCES, cm_req_set_resp_res); + IBA_CHECK_GET(CM_REQ_INITIATOR_DEPTH, cm_req_get_init_depth); + IBA_CHECK_SET(CM_REQ_INITIATOR_DEPTH, cm_req_set_init_depth); + IBA_CHECK_GET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, cm_req_get_remote_resp_timeout); + IBA_CHECK_SET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, cm_req_set_remote_resp_timeout); + IBA_CHECK_GET(CM_REQ_TRANSPORT_SERVICE_TYPE, cm_req_get_transport_type); + IBA_CHECK_SET(CM_REQ_TRANSPORT_SERVICE_TYPE, cm_req_set_transport_type); + IBA_CHECK_GET(CM_REQ_END_TO_END_FLOW_CONTROL, cm_req_get_flow_ctrl); + IBA_CHECK_SET(CM_REQ_END_TO_END_FLOW_CONTROL, cm_req_set_flow_ctrl); + IBA_CHECK_GET_BE(CM_REQ_STARTING_PSN, cm_req_get_starting_psn); + IBA_CHECK_SET_BE(CM_REQ_STARTING_PSN, cm_req_set_starting_psn); + IBA_CHECK_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, cm_req_get_local_resp_timeout); + IBA_CHECK_SET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, cm_req_set_local_resp_timeout); + IBA_CHECK_GET(CM_REQ_RETRY_COUNT, cm_req_get_retry_count); + IBA_CHECK_SET(CM_REQ_RETRY_COUNT, cm_req_set_retry_count); + IBA_CHECK_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, cm_req_get_path_mtu); + IBA_CHECK_SET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, cm_req_set_path_mtu); + IBA_CHECK_GET(CM_REQ_RNR_RETRY_COUNT, cm_req_get_rnr_retry_count); + IBA_CHECK_SET(CM_REQ_RNR_RETRY_COUNT, cm_req_set_rnr_retry_count); + IBA_CHECK_GET(CM_REQ_MAX_CM_RETRIES, cm_req_get_max_cm_retries); + IBA_CHECK_SET(CM_REQ_MAX_CM_RETRIES, cm_req_set_max_cm_retries); + IBA_CHECK_GET(CM_REQ_SRQ, cm_req_get_srq); + IBA_CHECK_SET(CM_REQ_SRQ, cm_req_set_srq); + IBA_CHECK_GET(CM_REQ_EXTENDED_TRANSPORT_TYPE, cm_req_get_transport_type_ex); + IBA_CHECK_SET(CM_REQ_EXTENDED_TRANSPORT_TYPE, cm_req_set_transport_type_ex); + IBA_CHECK_GET_BE(CM_REQ_PRIMARY_FLOW_LABEL, cm_req_get_primary_flow_label); + IBA_CHECK_SET_BE(CM_REQ_PRIMARY_FLOW_LABEL, cm_req_set_primary_flow_label); + IBA_CHECK_GET(CM_REQ_PRIMARY_PACKET_RATE, cm_req_get_primary_packet_rate); + IBA_CHECK_SET(CM_REQ_PRIMARY_PACKET_RATE, cm_req_set_primary_packet_rate); + IBA_CHECK_GET(CM_REQ_PRIMARY_SL, cm_req_get_primary_sl); + IBA_CHECK_SET(CM_REQ_PRIMARY_SL, cm_req_set_primary_sl); + IBA_CHECK_GET(CM_REQ_PRIMARY_SUBNET_LOCAL, cm_req_get_primary_subnet_local); + IBA_CHECK_SET(CM_REQ_PRIMARY_SUBNET_LOCAL, cm_req_set_primary_subnet_local); + IBA_CHECK_GET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, cm_req_get_primary_local_ack_timeout); + IBA_CHECK_SET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, cm_req_set_primary_local_ack_timeout); + IBA_CHECK_GET_BE(CM_REQ_ALTERNATE_FLOW_LABEL, cm_req_get_alt_flow_label); + IBA_CHECK_SET_BE(CM_REQ_ALTERNATE_FLOW_LABEL, cm_req_set_alt_flow_label); + IBA_CHECK_GET(CM_REQ_ALTERNATE_PACKET_RATE, cm_req_get_alt_packet_rate); + IBA_CHECK_SET(CM_REQ_ALTERNATE_PACKET_RATE, cm_req_set_alt_packet_rate); + IBA_CHECK_GET(CM_REQ_ALTERNATE_SL, cm_req_get_alt_sl); + IBA_CHECK_SET(CM_REQ_ALTERNATE_SL, cm_req_set_alt_sl); + IBA_CHECK_GET(CM_REQ_ALTERNATE_SUBNET_LOCAL, cm_req_get_alt_subnet_local); + IBA_CHECK_SET(CM_REQ_ALTERNATE_SUBNET_LOCAL, cm_req_set_alt_subnet_local); + IBA_CHECK_GET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, cm_req_get_alt_local_ack_timeout); + IBA_CHECK_SET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, cm_req_set_alt_local_ack_timeout); + IBA_CHECK_GET(CM_MRA_MESSAGE_MRAED, cm_mra_get_msg_mraed); + IBA_CHECK_SET(CM_MRA_MESSAGE_MRAED, cm_mra_set_msg_mraed); + IBA_CHECK_GET(CM_MRA_SERVICE_TIMEOUT, cm_mra_get_service_timeout); + IBA_CHECK_SET(CM_MRA_SERVICE_TIMEOUT, cm_mra_set_service_timeout); + IBA_CHECK_GET(CM_REJ_MESSAGE_REJECTED, cm_rej_get_msg_rejected); + IBA_CHECK_SET(CM_REJ_MESSAGE_REJECTED, cm_rej_set_msg_rejected); + IBA_CHECK_GET(CM_REJ_REJECTED_INFO_LENGTH, cm_rej_get_reject_info_len); + IBA_CHECK_SET(CM_REJ_REJECTED_INFO_LENGTH, cm_rej_set_reject_info_len); + IBA_CHECK_GET_BE(CM_REP_LOCAL_QPN, cm_rep_get_local_qpn); + IBA_CHECK_SET_BE(CM_REP_LOCAL_QPN, cm_rep_set_local_qpn); + IBA_CHECK_GET_BE(CM_REP_LOCAL_EE_CONTEXT_NUMBER, cm_rep_get_local_eecn); + IBA_CHECK_SET_BE(CM_REP_LOCAL_EE_CONTEXT_NUMBER, cm_rep_set_local_eecn); + IBA_CHECK_GET_BE(CM_REP_STARTING_PSN, cm_rep_get_starting_psn); + IBA_CHECK_SET_BE(CM_REP_STARTING_PSN, cm_rep_set_starting_psn); + IBA_CHECK_GET(CM_REP_TARGET_ACK_DELAY, cm_rep_get_target_ack_delay); + IBA_CHECK_SET(CM_REP_TARGET_ACK_DELAY, cm_rep_set_target_ack_delay); + IBA_CHECK_GET(CM_REP_FAILOVER_ACCEPTED, cm_rep_get_failover); + IBA_CHECK_SET(CM_REP_FAILOVER_ACCEPTED, cm_rep_set_failover); + IBA_CHECK_GET(CM_REP_END_TO_END_FLOW_CONTROL, cm_rep_get_flow_ctrl); + IBA_CHECK_SET(CM_REP_END_TO_END_FLOW_CONTROL, cm_rep_set_flow_ctrl); + IBA_CHECK_GET(CM_REP_RNR_RETRY_COUNT, cm_rep_get_rnr_retry_count); + IBA_CHECK_SET(CM_REP_RNR_RETRY_COUNT, cm_rep_set_rnr_retry_count); + IBA_CHECK_GET(CM_REP_SRQ, cm_rep_get_srq); + IBA_CHECK_SET(CM_REP_SRQ, cm_rep_set_srq); + IBA_CHECK_GET_BE(CM_DREQ_REMOTE_QPN_EECN, cm_dreq_get_remote_qpn); + IBA_CHECK_SET_BE(CM_DREQ_REMOTE_QPN_EECN, cm_dreq_set_remote_qpn); + IBA_CHECK_GET_BE(CM_LAP_ALTERNATE_FLOW_LABEL, cm_lap_get_flow_label); + IBA_CHECK_GET(CM_LAP_ALTERNATE_TRAFFIC_CLASS, cm_lap_get_traffic_class); + IBA_CHECK_GET(CM_LAP_ALTERNATE_PACKET_RATE, cm_lap_get_packet_rate); + IBA_CHECK_GET(CM_LAP_ALTERNATE_SL, cm_lap_get_sl); + IBA_CHECK_GET(CM_LAP_ALTERNATE_LOCAL_ACK_TIMEOUT, cm_lap_get_local_ack_timeout); + IBA_CHECK_GET_BE(CM_SIDR_REP_QPN, cm_sidr_rep_get_qpn); + IBA_CHECK_SET_BE(CM_SIDR_REP_QPN, cm_sidr_rep_set_qpn); + printk("Success!\n"); +} + static int __init ib_cm_init(void) { int ret; + self_test(); + INIT_LIST_HEAD(&cm.device_list); rwlock_init(&cm.device_lock); spin_lock_init(&cm.lock); diff --git a/drivers/infiniband/core/cm_msgs.h b/drivers/infiniband/core/cm_msgs.h index 9af9a3212074..bf62461d801f 100644 --- a/drivers/infiniband/core/cm_msgs.h +++ b/drivers/infiniband/core/cm_msgs.h @@ -8,6 +8,7 @@ #ifndef CM_MSGS_H #define CM_MSGS_H +#include #include #include diff --git a/include/rdma/iba.h b/include/rdma/iba.h new file mode 100644 index 000000000000..6a1115b02a0d --- /dev/null +++ b/include/rdma/iba.h @@ -0,0 +1,146 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* + * Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. + */ +#ifndef _IBA_DEFS_H_ +#define _IBA_DEFS_H_ + +#include +#include +#include + +static inline u32 _iba_get8(const u8 *ptr) +{ + return *ptr; +} + +static inline void _iba_set8(u8 *ptr, u32 mask, u32 prep_value) +{ + *ptr = (*ptr & ~mask) | prep_value; +} + +static inline u16 _iba_get16(const __be16 *ptr) +{ + return be16_to_cpu(*ptr); +} + +static inline void _iba_set16(__be16 *ptr, u16 mask, u16 prep_value) +{ + *ptr = cpu_to_be16((be16_to_cpu(*ptr) & ~mask) | prep_value); +} + +static inline u32 _iba_get32(const __be32 *ptr) +{ + return be32_to_cpu(*ptr); +} + +static inline void _iba_set32(__be32 *ptr, u32 mask, u32 prep_value) +{ + *ptr = cpu_to_be32((be32_to_cpu(*ptr) & ~mask) | prep_value); +} + +static inline u64 _iba_get64(const __be64 *ptr) +{ + /* + * The mads are constructed so that 32 bit and smaller are naturally + * aligned, everything larger has a max alignment of 4 bytes. + */ + return be64_to_cpu(get_unaligned(ptr)); +} + +static inline void _iba_set64(__be64 *ptr, u64 mask, u64 prep_value) +{ + put_unaligned(cpu_to_be64((_iba_get64(ptr) & ~mask) | prep_value), ptr); +} + +#define _IBA_SET(field_struct, field_offset, field_mask, num_bits, ptr, value) \ + ({ \ + field_struct *_ptr = ptr; \ + _iba_set##num_bits((void *)_ptr + (field_offset), field_mask, \ + FIELD_PREP(field_mask, value)); \ + }) +#define IBA_SET(field, ptr, value) _IBA_SET(field, ptr, value) + +#define _IBA_GET_MEM_PTR(field_struct, field_offset, type, num_bits, ptr) \ + ({ \ + field_struct *_ptr = ptr; \ + (type *)((void *)_ptr + (field_offset)); \ + }) +#define IBA_GET_MEM_PTR(field, ptr) _IBA_GET_MEM_PTR(field, ptr) + +/* FIXME: A set should always set the entire field, meaning we should zero the trailing bytes */ +#define _IBA_SET_MEM(field_struct, field_offset, type, num_bits, ptr, in, \ + bytes) \ + ({ \ + const type *_in_ptr = in; \ + WARN_ON(bytes * 8 > num_bits); \ + if (in && bytes) \ + memcpy(_IBA_GET_MEM_PTR(field_struct, field_offset, \ + type, num_bits, ptr), \ + _in_ptr, bytes); \ + }) +#define IBA_SET_MEM(field, ptr, in, bytes) _IBA_SET_MEM(field, ptr, in, bytes) + +#define _IBA_GET(field_struct, field_offset, field_mask, num_bits, ptr) \ + ({ \ + const field_struct *_ptr = ptr; \ + (u##num_bits) FIELD_GET( \ + field_mask, _iba_get##num_bits((const void *)_ptr + \ + (field_offset))); \ + }) +#define IBA_GET(field, ptr) _IBA_GET(field, ptr) + +#define _IBA_GET_MEM(field_struct, field_offset, type, num_bits, ptr, out, \ + bytes) \ + ({ \ + type *_out_ptr = out; \ + WARN_ON(bytes * 8 > num_bits); \ + if (out && bytes) \ + memcpy(_out_ptr, \ + _IBA_GET_MEM_PTR(field_struct, field_offset, \ + type, num_bits, ptr), \ + bytes); \ + }) +#define IBA_GET_MEM(field, ptr, out, bytes) _IBA_GET_MEM(field, ptr, out, bytes) + +/* + * The generated list becomes the parameters to the macros, the order is: + * - struct this applies to + * - starting offset of the max + * - GENMASK or GENMASK_ULL in CPU order + * - The width of data the mask operations should work on, in bits + */ + +/* + * Extraction using a tabular description like table 106. bit_offset is from + * the Byte[Bit] notation. + */ +#define IBA_FIELD_BLOC(field_struct, byte_offset, bit_offset, num_bits) \ + field_struct, byte_offset, \ + GENMASK(7 - (bit_offset), 7 - (bit_offset) - (num_bits - 1)), \ + 8 +#define IBA_FIELD8_LOC(field_struct, byte_offset, num_bits) \ + IBA_FIELD_BLOC(field_struct, byte_offset, 0, num_bits) + +#define IBA_FIELD16_LOC(field_struct, byte_offset, num_bits) \ + field_struct, (byte_offset)&0xFFFE, \ + GENMASK(15 - (((byte_offset) % 2) * 8), \ + 15 - (((byte_offset) % 2) * 8) - (num_bits - 1)), \ + 16 + +#define IBA_FIELD32_LOC(field_struct, byte_offset, num_bits) \ + field_struct, (byte_offset)&0xFFFC, \ + GENMASK(31 - (((byte_offset) % 4) * 8), \ + 31 - (((byte_offset) % 4) * 8) - (num_bits - 1)), \ + 32 + +#define IBA_FIELD64_LOC(field_struct, byte_offset) \ + field_struct, byte_offset, GENMASK_ULL(63, 0), 64 +/* + * In IBTA spec, everything that is more than 64bits is multiple + * of bytes without leftover bits. + */ +#define IBA_FIELD_MLOC(field_struct, byte_offset, num_bits, type) \ + field_struct, byte_offset, type, num_bits + +#endif /* _IBA_DEFS_H_ */ diff --git a/include/rdma/ibta_vol1_c12.h b/include/rdma/ibta_vol1_c12.h new file mode 100644 index 000000000000..916db5c27dc5 --- /dev/null +++ b/include/rdma/ibta_vol1_c12.h @@ -0,0 +1,208 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* + * Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. + * + * This file is IBTA volume 1, chapter 12 declarations: + * CHAPTER 12: COMMUNICATION MANAGEMENT + */ +#ifndef _IBTA_VOL1_C12_H_ +#define _IBTA_VOL1_C12_H_ + +#include + +#define CM_FIELD_BLOC(field_struct, byte_offset, bits_offset, width) \ + IBA_FIELD_BLOC(field_struct, \ + (byte_offset + sizeof(struct ib_mad_hdr)), bits_offset, \ + width) +#define CM_FIELD8_LOC(field_struct, byte_offset, width) \ + IBA_FIELD8_LOC(field_struct, \ + (byte_offset + sizeof(struct ib_mad_hdr)), width) +#define CM_FIELD16_LOC(field_struct, byte_offset, width) \ + IBA_FIELD16_LOC(field_struct, \ + (byte_offset + sizeof(struct ib_mad_hdr)), width) +#define CM_FIELD32_LOC(field_struct, byte_offset, width) \ + IBA_FIELD32_LOC(field_struct, \ + (byte_offset + sizeof(struct ib_mad_hdr)), width) +#define CM_FIELD64_LOC(field_struct, byte_offset) \ + IBA_FIELD64_LOC(field_struct, (byte_offset + sizeof(struct ib_mad_hdr))) +#define CM_FIELD_MLOC(field_struct, byte_offset, width, type) \ + IBA_FIELD_MLOC(field_struct, \ + (byte_offset + sizeof(struct ib_mad_hdr)), width, type) +#define CM_STRUCT(field_struct, total_len) \ + static_assert((total_len) % 32 == 0); + +/* Table 106 REQ Message Contents */ +#define CM_REQ_LOCAL_COMM_ID CM_FIELD32_LOC(struct cm_req_msg, 0, 32) +#define CM_REQ_SERVICE_ID CM_FIELD64_LOC(struct cm_req_msg, 8) +#define CM_REQ_LOCAL_CA_GUID CM_FIELD64_LOC(struct cm_req_msg, 16) +#define CM_REQ_LOCAL_Q_KEY CM_FIELD32_LOC(struct cm_req_msg, 28, 32) +#define CM_REQ_LOCAL_QPN CM_FIELD32_LOC(struct cm_req_msg, 32, 24) +#define CM_REQ_RESPONDER_RESOURCES CM_FIELD8_LOC(struct cm_req_msg, 35, 8) +#define CM_REQ_LOCAL_EECN CM_FIELD32_LOC(struct cm_req_msg, 36, 24) +#define CM_REQ_INITIATOR_DEPTH CM_FIELD8_LOC(struct cm_req_msg, 39, 8) +#define CM_REQ_REMOTE_EECN CM_FIELD32_LOC(struct cm_req_msg, 40, 24) +#define CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT \ + CM_FIELD8_LOC(struct cm_req_msg, 43, 5) +#define CM_REQ_TRANSPORT_SERVICE_TYPE CM_FIELD_BLOC(struct cm_req_msg, 43, 5, 2) +#define CM_REQ_END_TO_END_FLOW_CONTROL \ + CM_FIELD_BLOC(struct cm_req_msg, 43, 7, 1) +#define CM_REQ_STARTING_PSN CM_FIELD32_LOC(struct cm_req_msg, 44, 24) +#define CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT CM_FIELD8_LOC(struct cm_req_msg, 47, 5) +#define CM_REQ_RETRY_COUNT CM_FIELD_BLOC(struct cm_req_msg, 47, 5, 3) +#define CM_REQ_PARTITION_KEY CM_FIELD16_LOC(struct cm_req_msg, 48, 16) +#define CM_REQ_PATH_PACKET_PAYLOAD_MTU CM_FIELD8_LOC(struct cm_req_msg, 50, 4) +#define CM_REQ_RDC_EXISTS CM_FIELD_BLOC(struct cm_req_msg, 50, 4, 1) +#define CM_REQ_RNR_RETRY_COUNT CM_FIELD_BLOC(struct cm_req_msg, 50, 5, 3) +#define CM_REQ_MAX_CM_RETRIES CM_FIELD8_LOC(struct cm_req_msg, 51, 4) +#define CM_REQ_SRQ CM_FIELD_BLOC(struct cm_req_msg, 51, 4, 1) +#define CM_REQ_EXTENDED_TRANSPORT_TYPE \ + CM_FIELD_BLOC(struct cm_req_msg, 51, 5, 3) +#define CM_REQ_PRIMARY_LOCAL_PORT_LID CM_FIELD16_LOC(struct cm_req_msg, 52, 16) +#define CM_REQ_PRIMARY_REMOTE_PORT_LID CM_FIELD16_LOC(struct cm_req_msg, 54, 16) +#define CM_REQ_PRIMARY_LOCAL_PORT_GID \ + CM_FIELD_MLOC(struct cm_req_msg, 56, 128, union ib_gid) +#define CM_REQ_PRIMARY_REMOTE_PORT_GID \ + CM_FIELD_MLOC(struct cm_req_msg, 72, 128, union ib_gid) +#define CM_REQ_PRIMARY_FLOW_LABEL CM_FIELD32_LOC(struct cm_req_msg, 88, 20) +#define CM_REQ_PRIMARY_PACKET_RATE CM_FIELD_BLOC(struct cm_req_msg, 91, 2, 6) +#define CM_REQ_PRIMARY_TRAFFIC_CLASS CM_FIELD8_LOC(struct cm_req_msg, 92, 8) +#define CM_REQ_PRIMARY_HOP_LIMIT CM_FIELD8_LOC(struct cm_req_msg, 93, 8) +#define CM_REQ_PRIMARY_SL CM_FIELD8_LOC(struct cm_req_msg, 94, 4) +#define CM_REQ_PRIMARY_SUBNET_LOCAL CM_FIELD_BLOC(struct cm_req_msg, 94, 4, 1) +#define CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT CM_FIELD8_LOC(struct cm_req_msg, 95, 5) +#define CM_REQ_ALTERNATE_LOCAL_PORT_LID \ + CM_FIELD16_LOC(struct cm_req_msg, 96, 16) +#define CM_REQ_ALTERNATE_REMOTE_PORT_LID \ + CM_FIELD16_LOC(struct cm_req_msg, 98, 16) +#define CM_REQ_ALTERNATE_LOCAL_PORT_GID \ + CM_FIELD_MLOC(struct cm_req_msg, 100, 128, union ib_gid) +#define CM_REQ_ALTERNATE_REMOTE_PORT_GID \ + CM_FIELD_MLOC(struct cm_req_msg, 116, 128, union ib_gid) +#define CM_REQ_ALTERNATE_FLOW_LABEL CM_FIELD32_LOC(struct cm_req_msg, 132, 20) +#define CM_REQ_ALTERNATE_PACKET_RATE CM_FIELD_BLOC(struct cm_req_msg, 135, 2, 6) +#define CM_REQ_ALTERNATE_TRAFFIC_CLASS CM_FIELD8_LOC(struct cm_req_msg, 136, 8) +#define CM_REQ_ALTERNATE_HOP_LIMIT CM_FIELD8_LOC(struct cm_req_msg, 137, 8) +#define CM_REQ_ALTERNATE_SL CM_FIELD8_LOC(struct cm_req_msg, 138, 4) +#define CM_REQ_ALTERNATE_SUBNET_LOCAL \ + CM_FIELD_BLOC(struct cm_req_msg, 138, 4, 1) +#define CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT \ + CM_FIELD8_LOC(struct cm_req_msg, 139, 5) +#define CM_REQ_SAP_SUPPORTED CM_FIELD_BLOC(struct cm_req_msg, 139, 5, 1) +#define CM_REQ_PRIVATE_DATA CM_FIELD_MLOC(struct cm_req_msg, 140, 736, void) +CM_STRUCT(struct cm_req_msg, 140 * 8 + 736); + +/* Table 107 MRA Message Contents */ +#define CM_MRA_LOCAL_COMM_ID CM_FIELD32_LOC(struct cm_mra_msg, 0, 32) +#define CM_MRA_REMOTE_COMM_ID CM_FIELD32_LOC(struct cm_mra_msg, 4, 32) +#define CM_MRA_MESSAGE_MRAED CM_FIELD8_LOC(struct cm_mra_msg, 8, 2) +#define CM_MRA_SERVICE_TIMEOUT CM_FIELD8_LOC(struct cm_mra_msg, 9, 5) +#define CM_MRA_PRIVATE_DATA CM_FIELD_MLOC(struct cm_mra_msg, 10, 1776, void) +CM_STRUCT(struct cm_mra_msg, 10 * 8 + 1776); + +/* Table 108 REJ Message Contents */ +#define CM_REJ_LOCAL_COMM_ID CM_FIELD32_LOC(struct cm_rej_msg, 0, 32) +#define CM_REJ_REMOTE_COMM_ID CM_FIELD32_LOC(struct cm_rej_msg, 4, 32) +#define CM_REJ_MESSAGE_REJECTED CM_FIELD8_LOC(struct cm_rej_msg, 8, 2) +#define CM_REJ_REJECTED_INFO_LENGTH CM_FIELD8_LOC(struct cm_rej_msg, 9, 7) +#define CM_REJ_REASON CM_FIELD16_LOC(struct cm_rej_msg, 10, 16) +#define CM_REJ_ARI CM_FIELD_MLOC(struct cm_rej_msg, 12, 576, void) +#define CM_REJ_PRIVATE_DATA CM_FIELD_MLOC(struct cm_rej_msg, 84, 1184, void) +CM_STRUCT(struct cm_rej_msg, 84 * 8 + 1184); + +/* Table 110 REP Message Contents */ +#define CM_REP_LOCAL_COMM_ID CM_FIELD32_LOC(struct cm_rep_msg, 0, 32) +#define CM_REP_REMOTE_COMM_ID CM_FIELD32_LOC(struct cm_rep_msg, 4, 32) +#define CM_REP_LOCAL_Q_KEY CM_FIELD32_LOC(struct cm_rep_msg, 8, 32) +#define CM_REP_LOCAL_QPN CM_FIELD32_LOC(struct cm_rep_msg, 12, 24) +#define CM_REP_LOCAL_EE_CONTEXT_NUMBER CM_FIELD32_LOC(struct cm_rep_msg, 16, 24) +#define CM_REP_STARTING_PSN CM_FIELD32_LOC(struct cm_rep_msg, 20, 24) +#define CM_REP_RESPONDER_RESOURCES CM_FIELD8_LOC(struct cm_rep_msg, 24, 8) +#define CM_REP_INITIATOR_DEPTH CM_FIELD8_LOC(struct cm_rep_msg, 25, 8) +#define CM_REP_TARGET_ACK_DELAY CM_FIELD8_LOC(struct cm_rep_msg, 26, 5) +#define CM_REP_FAILOVER_ACCEPTED CM_FIELD_BLOC(struct cm_rep_msg, 26, 5, 2) +#define CM_REP_END_TO_END_FLOW_CONTROL \ + CM_FIELD_BLOC(struct cm_rep_msg, 26, 7, 1) +#define CM_REP_RNR_RETRY_COUNT CM_FIELD8_LOC(struct cm_rep_msg, 27, 3) +#define CM_REP_SRQ CM_FIELD_BLOC(struct cm_rep_msg, 27, 3, 1) +#define CM_REP_LOCAL_CA_GUID CM_FIELD64_LOC(struct cm_rep_msg, 28) +#define CM_REP_PRIVATE_DATA CM_FIELD_MLOC(struct cm_rep_msg, 36, 1568, void) +CM_STRUCT(struct cm_rep_msg, 36 * 8 + 1568); + +/* Table 111 RTU Message Contents */ +#define CM_RTU_LOCAL_COMM_ID CM_FIELD32_LOC(struct cm_rtu_msg, 0, 32) +#define CM_RTU_REMOTE_COMM_ID CM_FIELD32_LOC(struct cm_rtu_msg, 4, 32) +#define CM_RTU_PRIVATE_DATA CM_FIELD_MLOC(struct cm_rtu_msg, 8, 1792, void) +CM_STRUCT(struct cm_rtu_msg, 8 * 8 + 1792); + +/* Table 112 DREQ Message Contents */ +#define CM_DREQ_LOCAL_COMM_ID CM_FIELD32_LOC(struct cm_dreq_msg, 0, 32) +#define CM_DREQ_REMOTE_COMM_ID CM_FIELD32_LOC(struct cm_dreq_msg, 4, 32) +#define CM_DREQ_REMOTE_QPN_EECN CM_FIELD32_LOC(struct cm_dreq_msg, 8, 24) +#define CM_DREQ_PRIVATE_DATA CM_FIELD_MLOC(struct cm_dreq_msg, 12, 1760, void) +CM_STRUCT(struct cm_dreq_msg, 12 * 8 + 1760); + +/* Table 113 DREP Message Contents */ +#define CM_DREP_LOCAL_COMM_ID CM_FIELD32_LOC(struct cm_drep_msg, 0, 32) +#define CM_DREP_REMOTE_COMM_ID CM_FIELD32_LOC(struct cm_drep_msg, 4, 32) +#define CM_DREP_PRIVATE_DATA CM_FIELD_MLOC(struct cm_drep_msg, 8, 1792, void) +CM_STRUCT(struct cm_drep_msg, 8 * 8 + 1792); + +/* Table 115 LAP Message Contents */ +#define CM_LAP_LOCAL_COMM_ID CM_FIELD32_LOC(struct cm_lap_msg, 0, 32) +#define CM_LAP_REMOTE_COMM_ID CM_FIELD32_LOC(struct cm_lap_msg, 4, 32) +#define CM_LAP_REMOTE_QPN_EECN CM_FIELD32_LOC(struct cm_lap_msg, 12, 24) +#define CM_LAP_REMOTE_CM_RESPONSE_TIMEOUT \ + CM_FIELD8_LOC(struct cm_lap_msg, 15, 5) +#define CM_LAP_ALTERNATE_LOCAL_PORT_LID \ + CM_FIELD16_LOC(struct cm_lap_msg, 20, 16) +#define CM_LAP_ALTERNATE_REMOTE_PORT_LID \ + CM_FIELD16_LOC(struct cm_lap_msg, 22, 16) +#define CM_LAP_ALTERNATE_LOCAL_PORT_GID \ + CM_FIELD_MLOC(struct cm_lap_msg, 24, 128, union ib_gid) +#define CM_LAP_ALTERNATE_REMOTE_PORT_GID \ + CM_FIELD_MLOC(struct cm_lap_msg, 40, 128, union ib_gid) +#define CM_LAP_ALTERNATE_FLOW_LABEL CM_FIELD32_LOC(struct cm_lap_msg, 56, 20) +#define CM_LAP_ALTERNATE_TRAFFIC_CLASS CM_FIELD8_LOC(struct cm_lap_msg, 59, 8) +#define CM_LAP_ALTERNATE_HOP_LIMIT CM_FIELD8_LOC(struct cm_lap_msg, 60, 8) +#define CM_LAP_ALTERNATE_PACKET_RATE CM_FIELD_BLOC(struct cm_lap_msg, 61, 2, 6) +#define CM_LAP_ALTERNATE_SL CM_FIELD8_LOC(struct cm_lap_msg, 62, 4) +#define CM_LAP_ALTERNATE_SUBNET_LOCAL CM_FIELD_BLOC(struct cm_lap_msg, 62, 4, 1) +#define CM_LAP_ALTERNATE_LOCAL_ACK_TIMEOUT \ + CM_FIELD8_LOC(struct cm_lap_msg, 63, 5) +#define CM_LAP_PRIVATE_DATA CM_FIELD_MLOC(struct cm_lap_msg, 64, 1344, void) +CM_STRUCT(struct cm_lap_msg, 64 * 8 + 1344); + +/* Table 116 APR Message Contents */ +#define CM_APR_LOCAL_COMM_ID CM_FIELD32_LOC(struct cm_apr_msg, 0, 32) +#define CM_APR_REMOTE_COMM_ID CM_FIELD32_LOC(struct cm_apr_msg, 4, 32) +#define CM_APR_ADDITIONAL_INFORMATION_LENGTH \ + CM_FIELD8_LOC(struct cm_apr_msg, 8, 8) +#define CM_APR_AR_STATUS CM_FIELD8_LOC(struct cm_apr_msg, 9, 8) +#define CM_APR_ADDITIONAL_INFORMATION \ + CM_FIELD_MLOC(struct cm_apr_msg, 12, 576, void) +#define CM_APR_PRIVATE_DATA CM_FIELD_MLOC(struct cm_apr_msg, 84, 1184, void) +CM_STRUCT(struct cm_apr_msg, 84 * 8 + 1184); + +/* Table 119 SIDR_REQ Message Contents */ +#define CM_SIDR_REQ_REQUESTID CM_FIELD32_LOC(struct cm_sidr_req_msg, 0, 32) +#define CM_SIDR_REQ_PARTITION_KEY CM_FIELD16_LOC(struct cm_sidr_req_msg, 4, 16) +#define CM_SIDR_REQ_SERVICEID CM_FIELD64_LOC(struct cm_sidr_req_msg, 8) +#define CM_SIDR_REQ_PRIVATE_DATA \ + CM_FIELD_MLOC(struct cm_sidr_req_msg, 16, 1728, void) +CM_STRUCT(struct cm_sidr_req_msg, 16 * 8 + 1728); + +/* Table 120 SIDR_REP Message Contents */ +#define CM_SIDR_REP_REQUESTID CM_FIELD32_LOC(struct cm_sidr_rep_msg, 0, 32) +#define CM_SIDR_REP_STATUS CM_FIELD8_LOC(struct cm_sidr_rep_msg, 4, 8) +#define CM_SIDR_REP_ADDITIONAL_INFORMATION_LENGTH \ + CM_FIELD8_LOC(struct cm_sidr_rep_msg, 5, 8) +#define CM_SIDR_REP_QPN CM_FIELD32_LOC(struct cm_sidr_rep_msg, 8, 24) +#define CM_SIDR_REP_SERVICEID CM_FIELD64_LOC(struct cm_sidr_rep_msg, 12) +#define CM_SIDR_REP_Q_KEY CM_FIELD32_LOC(struct cm_sidr_rep_msg, 20, 32) +#define CM_SIDR_REP_ADDITIONAL_INFORMATION \ + CM_FIELD_MLOC(struct cm_sidr_rep_msg, 24, 576, void) +#define CM_SIDR_REP_PRIVATE_DATA \ + CM_FIELD_MLOC(struct cm_sidr_rep_msg, 96, 1088, void) +CM_STRUCT(struct cm_sidr_rep_msg, 96 * 8 + 1088); + +#endif /* _IBTA_VOL1_C12_H_ */ -- cgit v1.2.3-59-g8ed1b From b6bbee6889300372e8d84373de0afbd980155f90 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Thu, 16 Jan 2020 13:00:33 -0400 Subject: RDMA/cm: Use IBA functions for simple get/set acessors Use a Coccinelle spatch to replace CM helper functions with IBA_GET/SET versions. Applied with $ spatch --sp-file edits.sp --in-place drivers/infiniband/core/cm.c The spatch file was generated using the template pattern: @@ expression val; {struct} *msg; @@ - {old_setter} + IBA_SET({new_name}, msg, val) @@ {struct} *msg; @@ - {old_getter} + IBA_GET({new_name}, msg) Iterated for every IBA_CHECK_GET()/IBA_CHECK_GET() pairing. Touched up with clang-format after. Link: https://lore.kernel.org/r/20200116170037.30109-4-jgg@ziepe.ca Tested-by: Leon Romanovsky Reviewed-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/cm.c | 242 ++++++++++--------------- drivers/infiniband/core/cm_msgs.h | 371 +------------------------------------- 2 files changed, 104 insertions(+), 509 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c index 7f609979e4de..a1a968160132 100644 --- a/drivers/infiniband/core/cm.c +++ b/drivers/infiniband/core/cm.c @@ -1255,23 +1255,26 @@ static void cm_format_req(struct cm_req_msg *req_msg, req_msg->service_id = param->service_id; req_msg->local_ca_guid = cm_id_priv->id.device->node_guid; cm_req_set_local_qpn(req_msg, cpu_to_be32(param->qp_num)); - cm_req_set_init_depth(req_msg, param->initiator_depth); - cm_req_set_remote_resp_timeout(req_msg, - param->remote_cm_response_timeout); + IBA_SET(CM_REQ_INITIATOR_DEPTH, req_msg, param->initiator_depth); + IBA_SET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg, + param->remote_cm_response_timeout); cm_req_set_qp_type(req_msg, param->qp_type); - cm_req_set_flow_ctrl(req_msg, param->flow_control); + IBA_SET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg, param->flow_control); cm_req_set_starting_psn(req_msg, cpu_to_be32(param->starting_psn)); - cm_req_set_local_resp_timeout(req_msg, - param->local_cm_response_timeout); + IBA_SET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg, + param->local_cm_response_timeout); req_msg->pkey = param->primary_path->pkey; - cm_req_set_path_mtu(req_msg, param->primary_path->mtu); - cm_req_set_max_cm_retries(req_msg, param->max_cm_retries); + IBA_SET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg, + param->primary_path->mtu); + IBA_SET(CM_REQ_MAX_CM_RETRIES, req_msg, param->max_cm_retries); if (param->qp_type != IB_QPT_XRC_INI) { - cm_req_set_resp_res(req_msg, param->responder_resources); - cm_req_set_retry_count(req_msg, param->retry_count); - cm_req_set_rnr_retry_count(req_msg, param->rnr_retry_count); - cm_req_set_srq(req_msg, param->srq); + IBA_SET(CM_REQ_RESPONDER_RESOURCES, req_msg, + param->responder_resources); + IBA_SET(CM_REQ_RETRY_COUNT, req_msg, param->retry_count); + IBA_SET(CM_REQ_RNR_RETRY_COUNT, req_msg, + param->rnr_retry_count); + IBA_SET(CM_REQ_SRQ, req_msg, param->srq); } req_msg->primary_local_gid = pri_path->sgid; @@ -1293,12 +1296,13 @@ static void cm_format_req(struct cm_req_msg *req_msg, req_msg->primary_remote_lid = IB_LID_PERMISSIVE; } cm_req_set_primary_flow_label(req_msg, pri_path->flow_label); - cm_req_set_primary_packet_rate(req_msg, pri_path->rate); + IBA_SET(CM_REQ_PRIMARY_PACKET_RATE, req_msg, pri_path->rate); req_msg->primary_traffic_class = pri_path->traffic_class; req_msg->primary_hop_limit = pri_path->hop_limit; - cm_req_set_primary_sl(req_msg, pri_path->sl); - cm_req_set_primary_subnet_local(req_msg, (pri_path->hop_limit <= 1)); - cm_req_set_primary_local_ack_timeout(req_msg, + IBA_SET(CM_REQ_PRIMARY_SL, req_msg, pri_path->sl); + IBA_SET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg, + (pri_path->hop_limit <= 1)); + IBA_SET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg, cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay, pri_path->packet_life_time)); @@ -1328,12 +1332,13 @@ static void cm_format_req(struct cm_req_msg *req_msg, } cm_req_set_alt_flow_label(req_msg, alt_path->flow_label); - cm_req_set_alt_packet_rate(req_msg, alt_path->rate); + IBA_SET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg, alt_path->rate); req_msg->alt_traffic_class = alt_path->traffic_class; req_msg->alt_hop_limit = alt_path->hop_limit; - cm_req_set_alt_sl(req_msg, alt_path->sl); - cm_req_set_alt_subnet_local(req_msg, (alt_path->hop_limit <= 1)); - cm_req_set_alt_local_ack_timeout(req_msg, + IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, alt_path->sl); + IBA_SET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg, + (alt_path->hop_limit <= 1)); + IBA_SET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg, cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay, alt_path->packet_life_time)); } @@ -1473,11 +1478,11 @@ static int cm_issue_rej(struct cm_port *port, cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, rcv_msg->hdr.tid); rej_msg->remote_comm_id = rcv_msg->local_comm_id; rej_msg->local_comm_id = rcv_msg->remote_comm_id; - cm_rej_set_msg_rejected(rej_msg, msg_rejected); + IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, msg_rejected); rej_msg->reason = cpu_to_be16(reason); if (ari && ari_length) { - cm_rej_set_reject_info_len(rej_msg, ari_length); + IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length); memcpy(rej_msg->ari, ari, ari_length); } @@ -1548,14 +1553,14 @@ static void cm_format_paths_from_req(struct cm_req_msg *req_msg, primary_path->traffic_class = req_msg->primary_traffic_class; primary_path->reversible = 1; primary_path->pkey = req_msg->pkey; - primary_path->sl = cm_req_get_primary_sl(req_msg); + primary_path->sl = IBA_GET(CM_REQ_PRIMARY_SL, req_msg); primary_path->mtu_selector = IB_SA_EQ; - primary_path->mtu = cm_req_get_path_mtu(req_msg); + primary_path->mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg); primary_path->rate_selector = IB_SA_EQ; - primary_path->rate = cm_req_get_primary_packet_rate(req_msg); + primary_path->rate = IBA_GET(CM_REQ_PRIMARY_PACKET_RATE, req_msg); primary_path->packet_life_time_selector = IB_SA_EQ; primary_path->packet_life_time = - cm_req_get_primary_local_ack_timeout(req_msg); + IBA_GET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg); primary_path->packet_life_time -= (primary_path->packet_life_time > 0); primary_path->service_id = req_msg->service_id; if (sa_path_is_roce(primary_path)) @@ -1569,14 +1574,15 @@ static void cm_format_paths_from_req(struct cm_req_msg *req_msg, alt_path->traffic_class = req_msg->alt_traffic_class; alt_path->reversible = 1; alt_path->pkey = req_msg->pkey; - alt_path->sl = cm_req_get_alt_sl(req_msg); + alt_path->sl = IBA_GET(CM_REQ_ALTERNATE_SL, req_msg); alt_path->mtu_selector = IB_SA_EQ; - alt_path->mtu = cm_req_get_path_mtu(req_msg); + alt_path->mtu = + IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg); alt_path->rate_selector = IB_SA_EQ; - alt_path->rate = cm_req_get_alt_packet_rate(req_msg); + alt_path->rate = IBA_GET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg); alt_path->packet_life_time_selector = IB_SA_EQ; alt_path->packet_life_time = - cm_req_get_alt_local_ack_timeout(req_msg); + IBA_GET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg); alt_path->packet_life_time -= (alt_path->packet_life_time > 0); alt_path->service_id = req_msg->service_id; @@ -1658,16 +1664,16 @@ static void cm_format_req_event(struct cm_work *work, param->remote_qpn = be32_to_cpu(cm_req_get_local_qpn(req_msg)); param->qp_type = cm_req_get_qp_type(req_msg); param->starting_psn = be32_to_cpu(cm_req_get_starting_psn(req_msg)); - param->responder_resources = cm_req_get_init_depth(req_msg); - param->initiator_depth = cm_req_get_resp_res(req_msg); + param->responder_resources = IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg); + param->initiator_depth = IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg); param->local_cm_response_timeout = - cm_req_get_remote_resp_timeout(req_msg); - param->flow_control = cm_req_get_flow_ctrl(req_msg); + IBA_GET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg); + param->flow_control = IBA_GET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg); param->remote_cm_response_timeout = - cm_req_get_local_resp_timeout(req_msg); - param->retry_count = cm_req_get_retry_count(req_msg); - param->rnr_retry_count = cm_req_get_rnr_retry_count(req_msg); - param->srq = cm_req_get_srq(req_msg); + IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg); + param->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg); + param->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg); + param->srq = IBA_GET(CM_REQ_SRQ, req_msg); param->ppath_sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr; work->cm_event.private_data = &req_msg->private_data; } @@ -1703,10 +1709,10 @@ static void cm_format_mra(struct cm_mra_msg *mra_msg, const void *private_data, u8 private_data_len) { cm_format_mad_hdr(&mra_msg->hdr, CM_MRA_ATTR_ID, cm_id_priv->tid); - cm_mra_set_msg_mraed(mra_msg, msg_mraed); + IBA_SET(CM_MRA_MESSAGE_MRAED, mra_msg, msg_mraed); mra_msg->local_comm_id = cm_id_priv->id.local_id; mra_msg->remote_comm_id = cm_id_priv->id.remote_id; - cm_mra_set_service_timeout(mra_msg, service_timeout); + IBA_SET(CM_MRA_SERVICE_TIMEOUT, mra_msg, service_timeout); if (private_data && private_data_len) memcpy(mra_msg->private_data, private_data, private_data_len); @@ -1726,26 +1732,27 @@ static void cm_format_rej(struct cm_rej_msg *rej_msg, switch(cm_id_priv->id.state) { case IB_CM_REQ_RCVD: rej_msg->local_comm_id = 0; - cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REQ); + IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ); break; case IB_CM_MRA_REQ_SENT: rej_msg->local_comm_id = cm_id_priv->id.local_id; - cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REQ); + IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ); break; case IB_CM_REP_RCVD: case IB_CM_MRA_REP_SENT: rej_msg->local_comm_id = cm_id_priv->id.local_id; - cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REP); + IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REP); break; default: rej_msg->local_comm_id = cm_id_priv->id.local_id; - cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_OTHER); + IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, + CM_MSG_RESPONSE_OTHER); break; } rej_msg->reason = cpu_to_be16(reason); if (ari && ari_length) { - cm_rej_set_reject_info_len(rej_msg, ari_length); + IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length); memcpy(rej_msg->ari, ari, ari_length); } @@ -1866,20 +1873,20 @@ out: */ static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc) { - if (!cm_req_get_primary_subnet_local(req_msg)) { + if (!IBA_GET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg)) { if (req_msg->primary_local_lid == IB_LID_PERMISSIVE) { req_msg->primary_local_lid = ib_lid_be16(wc->slid); - cm_req_set_primary_sl(req_msg, wc->sl); + IBA_SET(CM_REQ_PRIMARY_SL, req_msg, wc->sl); } if (req_msg->primary_remote_lid == IB_LID_PERMISSIVE) req_msg->primary_remote_lid = cpu_to_be16(wc->dlid_path_bits); } - if (!cm_req_get_alt_subnet_local(req_msg)) { + if (!IBA_GET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg)) { if (req_msg->alt_local_lid == IB_LID_PERMISSIVE) { req_msg->alt_local_lid = ib_lid_be16(wc->slid); - cm_req_set_alt_sl(req_msg, wc->sl); + IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, wc->sl); } if (req_msg->alt_remote_lid == IB_LID_PERMISSIVE) @@ -1989,16 +1996,18 @@ static int cm_req_handler(struct cm_work *work) } cm_id_priv->tid = req_msg->hdr.tid; cm_id_priv->timeout_ms = cm_convert_to_ms( - cm_req_get_local_resp_timeout(req_msg)); - cm_id_priv->max_cm_retries = cm_req_get_max_cm_retries(req_msg); + IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg)); + cm_id_priv->max_cm_retries = IBA_GET(CM_REQ_MAX_CM_RETRIES, req_msg); cm_id_priv->remote_qpn = cm_req_get_local_qpn(req_msg); - cm_id_priv->initiator_depth = cm_req_get_resp_res(req_msg); - cm_id_priv->responder_resources = cm_req_get_init_depth(req_msg); - cm_id_priv->path_mtu = cm_req_get_path_mtu(req_msg); + cm_id_priv->initiator_depth = + IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg); + cm_id_priv->responder_resources = + IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg); + cm_id_priv->path_mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg); cm_id_priv->pkey = req_msg->pkey; cm_id_priv->sq_psn = cm_req_get_starting_psn(req_msg); - cm_id_priv->retry_count = cm_req_get_retry_count(req_msg); - cm_id_priv->rnr_retry_count = cm_req_get_rnr_retry_count(req_msg); + cm_id_priv->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg); + cm_id_priv->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg); cm_id_priv->qp_type = cm_req_get_qp_type(req_msg); cm_format_req_event(work, cm_id_priv, &listen_cm_id_priv->id); @@ -2025,19 +2034,20 @@ static void cm_format_rep(struct cm_rep_msg *rep_msg, rep_msg->remote_comm_id = cm_id_priv->id.remote_id; cm_rep_set_starting_psn(rep_msg, cpu_to_be32(param->starting_psn)); rep_msg->resp_resources = param->responder_resources; - cm_rep_set_target_ack_delay(rep_msg, - cm_id_priv->av.port->cm_dev->ack_delay); - cm_rep_set_failover(rep_msg, param->failover_accepted); - cm_rep_set_rnr_retry_count(rep_msg, param->rnr_retry_count); + IBA_SET(CM_REP_TARGET_ACK_DELAY, rep_msg, + cm_id_priv->av.port->cm_dev->ack_delay); + IBA_SET(CM_REP_FAILOVER_ACCEPTED, rep_msg, param->failover_accepted); + IBA_SET(CM_REP_RNR_RETRY_COUNT, rep_msg, param->rnr_retry_count); rep_msg->local_ca_guid = cm_id_priv->id.device->node_guid; if (cm_id_priv->qp_type != IB_QPT_XRC_TGT) { rep_msg->initiator_depth = param->initiator_depth; - cm_rep_set_flow_ctrl(rep_msg, param->flow_control); - cm_rep_set_srq(rep_msg, param->srq); + IBA_SET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg, + param->flow_control); + IBA_SET(CM_REP_SRQ, rep_msg, param->srq); cm_rep_set_local_qpn(rep_msg, cpu_to_be32(param->qp_num)); } else { - cm_rep_set_srq(rep_msg, 1); + IBA_SET(CM_REP_SRQ, rep_msg, 1); cm_rep_set_local_eecn(rep_msg, cpu_to_be32(param->qp_num)); } @@ -2176,11 +2186,11 @@ static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type) param->starting_psn = be32_to_cpu(cm_rep_get_starting_psn(rep_msg)); param->responder_resources = rep_msg->initiator_depth; param->initiator_depth = rep_msg->resp_resources; - param->target_ack_delay = cm_rep_get_target_ack_delay(rep_msg); - param->failover_accepted = cm_rep_get_failover(rep_msg); - param->flow_control = cm_rep_get_flow_ctrl(rep_msg); - param->rnr_retry_count = cm_rep_get_rnr_retry_count(rep_msg); - param->srq = cm_rep_get_srq(rep_msg); + param->target_ack_delay = IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg); + param->failover_accepted = IBA_GET(CM_REP_FAILOVER_ACCEPTED, rep_msg); + param->flow_control = IBA_GET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg); + param->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg); + param->srq = IBA_GET(CM_REP_SRQ, rep_msg); work->cm_event.private_data = &rep_msg->private_data; } @@ -2311,8 +2321,9 @@ static int cm_rep_handler(struct cm_work *work) cm_id_priv->initiator_depth = rep_msg->resp_resources; cm_id_priv->responder_resources = rep_msg->initiator_depth; cm_id_priv->sq_psn = cm_rep_get_starting_psn(rep_msg); - cm_id_priv->rnr_retry_count = cm_rep_get_rnr_retry_count(rep_msg); - cm_id_priv->target_ack_delay = cm_rep_get_target_ack_delay(rep_msg); + cm_id_priv->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg); + cm_id_priv->target_ack_delay = + IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg); cm_id_priv->av.timeout = cm_ack_timeout(cm_id_priv->target_ack_delay, cm_id_priv->av.timeout - 1); @@ -2756,7 +2767,7 @@ static void cm_format_rej_event(struct cm_work *work) rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad; param = &work->cm_event.param.rej_rcvd; param->ari = rej_msg->ari; - param->ari_length = cm_rej_get_reject_info_len(rej_msg); + param->ari_length = IBA_GET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg); param->reason = __be16_to_cpu(rej_msg->reason); work->cm_event.private_data = &rej_msg->private_data; } @@ -2780,7 +2791,8 @@ static struct cm_id_private * cm_acquire_rejected_id(struct cm_rej_msg *rej_msg) cm_id_priv = cm_acquire_id(timewait_info->work.local_id, remote_id); spin_unlock_irq(&cm.lock); - } else if (cm_rej_get_msg_rejected(rej_msg) == CM_MSG_RESPONSE_REQ) + } else if (IBA_GET(CM_REJ_MESSAGE_REJECTED, rej_msg) == + CM_MSG_RESPONSE_REQ) cm_id_priv = cm_acquire_id(rej_msg->remote_comm_id, 0); else cm_id_priv = cm_acquire_id(rej_msg->remote_comm_id, remote_id); @@ -2941,7 +2953,7 @@ EXPORT_SYMBOL(ib_send_cm_mra); static struct cm_id_private * cm_acquire_mraed_id(struct cm_mra_msg *mra_msg) { - switch (cm_mra_get_msg_mraed(mra_msg)) { + switch (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg)) { case CM_MSG_RESPONSE_REQ: return cm_acquire_id(mra_msg->remote_comm_id, 0); case CM_MSG_RESPONSE_REP: @@ -2966,28 +2978,31 @@ static int cm_mra_handler(struct cm_work *work) work->cm_event.private_data = &mra_msg->private_data; work->cm_event.param.mra_rcvd.service_timeout = - cm_mra_get_service_timeout(mra_msg); - timeout = cm_convert_to_ms(cm_mra_get_service_timeout(mra_msg)) + + IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg); + timeout = cm_convert_to_ms(IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg)) + cm_convert_to_ms(cm_id_priv->av.timeout); spin_lock_irq(&cm_id_priv->lock); switch (cm_id_priv->id.state) { case IB_CM_REQ_SENT: - if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_REQ || + if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) != + CM_MSG_RESPONSE_REQ || ib_modify_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg, timeout)) goto out; cm_id_priv->id.state = IB_CM_MRA_REQ_RCVD; break; case IB_CM_REP_SENT: - if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_REP || + if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) != + CM_MSG_RESPONSE_REP || ib_modify_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg, timeout)) goto out; cm_id_priv->id.state = IB_CM_MRA_REP_RCVD; break; case IB_CM_ESTABLISHED: - if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_OTHER || + if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) != + CM_MSG_RESPONSE_OTHER || cm_id_priv->id.lap_state != IB_CM_LAP_SENT || ib_modify_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg, timeout)) { @@ -3054,16 +3069,17 @@ static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv, path->sgid = lap_msg->alt_remote_gid; path->flow_label = cm_lap_get_flow_label(lap_msg); path->hop_limit = lap_msg->alt_hop_limit; - path->traffic_class = cm_lap_get_traffic_class(lap_msg); + path->traffic_class = IBA_GET(CM_LAP_ALTERNATE_TRAFFIC_CLASS, lap_msg); path->reversible = 1; path->pkey = cm_id_priv->pkey; - path->sl = cm_lap_get_sl(lap_msg); + path->sl = IBA_GET(CM_LAP_ALTERNATE_SL, lap_msg); path->mtu_selector = IB_SA_EQ; path->mtu = cm_id_priv->path_mtu; path->rate_selector = IB_SA_EQ; - path->rate = cm_lap_get_packet_rate(lap_msg); + path->rate = IBA_GET(CM_LAP_ALTERNATE_PACKET_RATE, lap_msg); path->packet_life_time_selector = IB_SA_EQ; - path->packet_life_time = cm_lap_get_local_ack_timeout(lap_msg); + path->packet_life_time = + IBA_GET(CM_LAP_ALTERNATE_LOCAL_ACK_TIMEOUT, lap_msg); path->packet_life_time -= (path->packet_life_time > 0); cm_format_path_lid_from_lap(lap_msg, path); } @@ -4436,83 +4452,21 @@ static void self_test(void) printk("Running CM extractor self test\n"); IBA_CHECK_GET_BE(CM_REQ_LOCAL_QPN, cm_req_get_local_qpn); IBA_CHECK_SET_BE(CM_REQ_LOCAL_QPN, cm_req_set_local_qpn); - IBA_CHECK_GET(CM_REQ_RESPONDER_RESOURCES, cm_req_get_resp_res); - IBA_CHECK_SET(CM_REQ_RESPONDER_RESOURCES, cm_req_set_resp_res); - IBA_CHECK_GET(CM_REQ_INITIATOR_DEPTH, cm_req_get_init_depth); - IBA_CHECK_SET(CM_REQ_INITIATOR_DEPTH, cm_req_set_init_depth); - IBA_CHECK_GET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, cm_req_get_remote_resp_timeout); - IBA_CHECK_SET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, cm_req_set_remote_resp_timeout); - IBA_CHECK_GET(CM_REQ_TRANSPORT_SERVICE_TYPE, cm_req_get_transport_type); - IBA_CHECK_SET(CM_REQ_TRANSPORT_SERVICE_TYPE, cm_req_set_transport_type); - IBA_CHECK_GET(CM_REQ_END_TO_END_FLOW_CONTROL, cm_req_get_flow_ctrl); - IBA_CHECK_SET(CM_REQ_END_TO_END_FLOW_CONTROL, cm_req_set_flow_ctrl); IBA_CHECK_GET_BE(CM_REQ_STARTING_PSN, cm_req_get_starting_psn); IBA_CHECK_SET_BE(CM_REQ_STARTING_PSN, cm_req_set_starting_psn); - IBA_CHECK_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, cm_req_get_local_resp_timeout); - IBA_CHECK_SET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, cm_req_set_local_resp_timeout); - IBA_CHECK_GET(CM_REQ_RETRY_COUNT, cm_req_get_retry_count); - IBA_CHECK_SET(CM_REQ_RETRY_COUNT, cm_req_set_retry_count); - IBA_CHECK_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, cm_req_get_path_mtu); - IBA_CHECK_SET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, cm_req_set_path_mtu); - IBA_CHECK_GET(CM_REQ_RNR_RETRY_COUNT, cm_req_get_rnr_retry_count); - IBA_CHECK_SET(CM_REQ_RNR_RETRY_COUNT, cm_req_set_rnr_retry_count); - IBA_CHECK_GET(CM_REQ_MAX_CM_RETRIES, cm_req_get_max_cm_retries); - IBA_CHECK_SET(CM_REQ_MAX_CM_RETRIES, cm_req_set_max_cm_retries); - IBA_CHECK_GET(CM_REQ_SRQ, cm_req_get_srq); - IBA_CHECK_SET(CM_REQ_SRQ, cm_req_set_srq); - IBA_CHECK_GET(CM_REQ_EXTENDED_TRANSPORT_TYPE, cm_req_get_transport_type_ex); - IBA_CHECK_SET(CM_REQ_EXTENDED_TRANSPORT_TYPE, cm_req_set_transport_type_ex); IBA_CHECK_GET_BE(CM_REQ_PRIMARY_FLOW_LABEL, cm_req_get_primary_flow_label); IBA_CHECK_SET_BE(CM_REQ_PRIMARY_FLOW_LABEL, cm_req_set_primary_flow_label); - IBA_CHECK_GET(CM_REQ_PRIMARY_PACKET_RATE, cm_req_get_primary_packet_rate); - IBA_CHECK_SET(CM_REQ_PRIMARY_PACKET_RATE, cm_req_set_primary_packet_rate); - IBA_CHECK_GET(CM_REQ_PRIMARY_SL, cm_req_get_primary_sl); - IBA_CHECK_SET(CM_REQ_PRIMARY_SL, cm_req_set_primary_sl); - IBA_CHECK_GET(CM_REQ_PRIMARY_SUBNET_LOCAL, cm_req_get_primary_subnet_local); - IBA_CHECK_SET(CM_REQ_PRIMARY_SUBNET_LOCAL, cm_req_set_primary_subnet_local); - IBA_CHECK_GET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, cm_req_get_primary_local_ack_timeout); - IBA_CHECK_SET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, cm_req_set_primary_local_ack_timeout); IBA_CHECK_GET_BE(CM_REQ_ALTERNATE_FLOW_LABEL, cm_req_get_alt_flow_label); IBA_CHECK_SET_BE(CM_REQ_ALTERNATE_FLOW_LABEL, cm_req_set_alt_flow_label); - IBA_CHECK_GET(CM_REQ_ALTERNATE_PACKET_RATE, cm_req_get_alt_packet_rate); - IBA_CHECK_SET(CM_REQ_ALTERNATE_PACKET_RATE, cm_req_set_alt_packet_rate); - IBA_CHECK_GET(CM_REQ_ALTERNATE_SL, cm_req_get_alt_sl); - IBA_CHECK_SET(CM_REQ_ALTERNATE_SL, cm_req_set_alt_sl); - IBA_CHECK_GET(CM_REQ_ALTERNATE_SUBNET_LOCAL, cm_req_get_alt_subnet_local); - IBA_CHECK_SET(CM_REQ_ALTERNATE_SUBNET_LOCAL, cm_req_set_alt_subnet_local); - IBA_CHECK_GET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, cm_req_get_alt_local_ack_timeout); - IBA_CHECK_SET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, cm_req_set_alt_local_ack_timeout); - IBA_CHECK_GET(CM_MRA_MESSAGE_MRAED, cm_mra_get_msg_mraed); - IBA_CHECK_SET(CM_MRA_MESSAGE_MRAED, cm_mra_set_msg_mraed); - IBA_CHECK_GET(CM_MRA_SERVICE_TIMEOUT, cm_mra_get_service_timeout); - IBA_CHECK_SET(CM_MRA_SERVICE_TIMEOUT, cm_mra_set_service_timeout); - IBA_CHECK_GET(CM_REJ_MESSAGE_REJECTED, cm_rej_get_msg_rejected); - IBA_CHECK_SET(CM_REJ_MESSAGE_REJECTED, cm_rej_set_msg_rejected); - IBA_CHECK_GET(CM_REJ_REJECTED_INFO_LENGTH, cm_rej_get_reject_info_len); - IBA_CHECK_SET(CM_REJ_REJECTED_INFO_LENGTH, cm_rej_set_reject_info_len); IBA_CHECK_GET_BE(CM_REP_LOCAL_QPN, cm_rep_get_local_qpn); IBA_CHECK_SET_BE(CM_REP_LOCAL_QPN, cm_rep_set_local_qpn); IBA_CHECK_GET_BE(CM_REP_LOCAL_EE_CONTEXT_NUMBER, cm_rep_get_local_eecn); IBA_CHECK_SET_BE(CM_REP_LOCAL_EE_CONTEXT_NUMBER, cm_rep_set_local_eecn); IBA_CHECK_GET_BE(CM_REP_STARTING_PSN, cm_rep_get_starting_psn); IBA_CHECK_SET_BE(CM_REP_STARTING_PSN, cm_rep_set_starting_psn); - IBA_CHECK_GET(CM_REP_TARGET_ACK_DELAY, cm_rep_get_target_ack_delay); - IBA_CHECK_SET(CM_REP_TARGET_ACK_DELAY, cm_rep_set_target_ack_delay); - IBA_CHECK_GET(CM_REP_FAILOVER_ACCEPTED, cm_rep_get_failover); - IBA_CHECK_SET(CM_REP_FAILOVER_ACCEPTED, cm_rep_set_failover); - IBA_CHECK_GET(CM_REP_END_TO_END_FLOW_CONTROL, cm_rep_get_flow_ctrl); - IBA_CHECK_SET(CM_REP_END_TO_END_FLOW_CONTROL, cm_rep_set_flow_ctrl); - IBA_CHECK_GET(CM_REP_RNR_RETRY_COUNT, cm_rep_get_rnr_retry_count); - IBA_CHECK_SET(CM_REP_RNR_RETRY_COUNT, cm_rep_set_rnr_retry_count); - IBA_CHECK_GET(CM_REP_SRQ, cm_rep_get_srq); - IBA_CHECK_SET(CM_REP_SRQ, cm_rep_set_srq); IBA_CHECK_GET_BE(CM_DREQ_REMOTE_QPN_EECN, cm_dreq_get_remote_qpn); IBA_CHECK_SET_BE(CM_DREQ_REMOTE_QPN_EECN, cm_dreq_set_remote_qpn); IBA_CHECK_GET_BE(CM_LAP_ALTERNATE_FLOW_LABEL, cm_lap_get_flow_label); - IBA_CHECK_GET(CM_LAP_ALTERNATE_TRAFFIC_CLASS, cm_lap_get_traffic_class); - IBA_CHECK_GET(CM_LAP_ALTERNATE_PACKET_RATE, cm_lap_get_packet_rate); - IBA_CHECK_GET(CM_LAP_ALTERNATE_SL, cm_lap_get_sl); - IBA_CHECK_GET(CM_LAP_ALTERNATE_LOCAL_ACK_TIMEOUT, cm_lap_get_local_ack_timeout); IBA_CHECK_GET_BE(CM_SIDR_REP_QPN, cm_sidr_rep_get_qpn); IBA_CHECK_SET_BE(CM_SIDR_REP_QPN, cm_sidr_rep_set_qpn); printk("Success!\n"); diff --git a/drivers/infiniband/core/cm_msgs.h b/drivers/infiniband/core/cm_msgs.h index bf62461d801f..d30586b1b8a4 100644 --- a/drivers/infiniband/core/cm_msgs.h +++ b/drivers/infiniband/core/cm_msgs.h @@ -87,75 +87,14 @@ static inline void cm_req_set_local_qpn(struct cm_req_msg *req_msg, __be32 qpn) 0x000000FF)); } -static inline u8 cm_req_get_resp_res(struct cm_req_msg *req_msg) -{ - return (u8) be32_to_cpu(req_msg->offset32); -} - -static inline void cm_req_set_resp_res(struct cm_req_msg *req_msg, u8 resp_res) -{ - req_msg->offset32 = cpu_to_be32(resp_res | - (be32_to_cpu(req_msg->offset32) & - 0xFFFFFF00)); -} - -static inline u8 cm_req_get_init_depth(struct cm_req_msg *req_msg) -{ - return (u8) be32_to_cpu(req_msg->offset36); -} - -static inline void cm_req_set_init_depth(struct cm_req_msg *req_msg, - u8 init_depth) -{ - req_msg->offset36 = cpu_to_be32(init_depth | - (be32_to_cpu(req_msg->offset36) & - 0xFFFFFF00)); -} - -static inline u8 cm_req_get_remote_resp_timeout(struct cm_req_msg *req_msg) -{ - return (u8) ((be32_to_cpu(req_msg->offset40) & 0xF8) >> 3); -} - -static inline void cm_req_set_remote_resp_timeout(struct cm_req_msg *req_msg, - u8 resp_timeout) -{ - req_msg->offset40 = cpu_to_be32((resp_timeout << 3) | - (be32_to_cpu(req_msg->offset40) & - 0xFFFFFF07)); -} - -static inline u8 cm_req_get_transport_type(struct cm_req_msg *req_msg) -{ - return (u8) ((be32_to_cpu(req_msg->offset40) & 0x06) >> 1); -} - -static inline void cm_req_set_transport_type(struct cm_req_msg *req_msg, u8 val) -{ - req_msg->offset40 = - cpu_to_be32((be32_to_cpu(req_msg->offset40) & 0xFFFFFFF9) | - (val << 1)); -} - -static inline u8 cm_req_get_transport_type_ex(struct cm_req_msg *req_msg) -{ - return req_msg->offset51 & 0x7; -} - -static inline void cm_req_set_transport_type_ex(struct cm_req_msg *req_msg, - u8 val) -{ - req_msg->offset51 = (req_msg->offset51 & 0xF8) | val; -} - static inline enum ib_qp_type cm_req_get_qp_type(struct cm_req_msg *req_msg) { - u8 transport_type = cm_req_get_transport_type(req_msg); + u8 transport_type = IBA_GET(CM_REQ_TRANSPORT_SERVICE_TYPE, req_msg); switch(transport_type) { case 0: return IB_QPT_RC; case 1: return IB_QPT_UC; case 3: - switch (cm_req_get_transport_type_ex(req_msg)) { + switch (IBA_GET(CM_REQ_EXTENDED_TRANSPORT_TYPE, req_msg)) { case 1: return IB_QPT_XRC_TGT; default: return 0; } @@ -168,30 +107,17 @@ static inline void cm_req_set_qp_type(struct cm_req_msg *req_msg, { switch(qp_type) { case IB_QPT_UC: - cm_req_set_transport_type(req_msg, 1); + IBA_SET(CM_REQ_TRANSPORT_SERVICE_TYPE, req_msg, 1); break; case IB_QPT_XRC_INI: - cm_req_set_transport_type(req_msg, 3); - cm_req_set_transport_type_ex(req_msg, 1); + IBA_SET(CM_REQ_TRANSPORT_SERVICE_TYPE, req_msg, 3); + IBA_SET(CM_REQ_EXTENDED_TRANSPORT_TYPE, req_msg, 1); break; default: - cm_req_set_transport_type(req_msg, 0); + IBA_SET(CM_REQ_TRANSPORT_SERVICE_TYPE, req_msg, 0); } } -static inline u8 cm_req_get_flow_ctrl(struct cm_req_msg *req_msg) -{ - return be32_to_cpu(req_msg->offset40) & 0x1; -} - -static inline void cm_req_set_flow_ctrl(struct cm_req_msg *req_msg, - u8 flow_ctrl) -{ - req_msg->offset40 = cpu_to_be32((flow_ctrl & 0x1) | - (be32_to_cpu(req_msg->offset40) & - 0xFFFFFFFE)); -} - static inline __be32 cm_req_get_starting_psn(struct cm_req_msg *req_msg) { return cpu_to_be32(be32_to_cpu(req_msg->offset44) >> 8); @@ -204,74 +130,6 @@ static inline void cm_req_set_starting_psn(struct cm_req_msg *req_msg, (be32_to_cpu(req_msg->offset44) & 0x000000FF)); } -static inline u8 cm_req_get_local_resp_timeout(struct cm_req_msg *req_msg) -{ - return (u8) ((be32_to_cpu(req_msg->offset44) & 0xF8) >> 3); -} - -static inline void cm_req_set_local_resp_timeout(struct cm_req_msg *req_msg, - u8 resp_timeout) -{ - req_msg->offset44 = cpu_to_be32((resp_timeout << 3) | - (be32_to_cpu(req_msg->offset44) & 0xFFFFFF07)); -} - -static inline u8 cm_req_get_retry_count(struct cm_req_msg *req_msg) -{ - return (u8) (be32_to_cpu(req_msg->offset44) & 0x7); -} - -static inline void cm_req_set_retry_count(struct cm_req_msg *req_msg, - u8 retry_count) -{ - req_msg->offset44 = cpu_to_be32((retry_count & 0x7) | - (be32_to_cpu(req_msg->offset44) & 0xFFFFFFF8)); -} - -static inline u8 cm_req_get_path_mtu(struct cm_req_msg *req_msg) -{ - return req_msg->offset50 >> 4; -} - -static inline void cm_req_set_path_mtu(struct cm_req_msg *req_msg, u8 path_mtu) -{ - req_msg->offset50 = (u8) ((req_msg->offset50 & 0xF) | (path_mtu << 4)); -} - -static inline u8 cm_req_get_rnr_retry_count(struct cm_req_msg *req_msg) -{ - return req_msg->offset50 & 0x7; -} - -static inline void cm_req_set_rnr_retry_count(struct cm_req_msg *req_msg, - u8 rnr_retry_count) -{ - req_msg->offset50 = (u8) ((req_msg->offset50 & 0xF8) | - (rnr_retry_count & 0x7)); -} - -static inline u8 cm_req_get_max_cm_retries(struct cm_req_msg *req_msg) -{ - return req_msg->offset51 >> 4; -} - -static inline void cm_req_set_max_cm_retries(struct cm_req_msg *req_msg, - u8 retries) -{ - req_msg->offset51 = (u8) ((req_msg->offset51 & 0xF) | (retries << 4)); -} - -static inline u8 cm_req_get_srq(struct cm_req_msg *req_msg) -{ - return (req_msg->offset51 & 0x8) >> 3; -} - -static inline void cm_req_set_srq(struct cm_req_msg *req_msg, u8 srq) -{ - req_msg->offset51 = (u8) ((req_msg->offset51 & 0xF7) | - ((srq & 0x1) << 3)); -} - static inline __be32 cm_req_get_primary_flow_label(struct cm_req_msg *req_msg) { return cpu_to_be32(be32_to_cpu(req_msg->primary_offset88) >> 12); @@ -286,54 +144,6 @@ static inline void cm_req_set_primary_flow_label(struct cm_req_msg *req_msg, (be32_to_cpu(flow_label) << 12)); } -static inline u8 cm_req_get_primary_packet_rate(struct cm_req_msg *req_msg) -{ - return (u8) (be32_to_cpu(req_msg->primary_offset88) & 0x3F); -} - -static inline void cm_req_set_primary_packet_rate(struct cm_req_msg *req_msg, - u8 rate) -{ - req_msg->primary_offset88 = cpu_to_be32( - (be32_to_cpu(req_msg->primary_offset88) & - 0xFFFFFFC0) | (rate & 0x3F)); -} - -static inline u8 cm_req_get_primary_sl(struct cm_req_msg *req_msg) -{ - return (u8) (req_msg->primary_offset94 >> 4); -} - -static inline void cm_req_set_primary_sl(struct cm_req_msg *req_msg, u8 sl) -{ - req_msg->primary_offset94 = (u8) ((req_msg->primary_offset94 & 0x0F) | - (sl << 4)); -} - -static inline u8 cm_req_get_primary_subnet_local(struct cm_req_msg *req_msg) -{ - return (u8) ((req_msg->primary_offset94 & 0x08) >> 3); -} - -static inline void cm_req_set_primary_subnet_local(struct cm_req_msg *req_msg, - u8 subnet_local) -{ - req_msg->primary_offset94 = (u8) ((req_msg->primary_offset94 & 0xF7) | - ((subnet_local & 0x1) << 3)); -} - -static inline u8 cm_req_get_primary_local_ack_timeout(struct cm_req_msg *req_msg) -{ - return (u8) (req_msg->primary_offset95 >> 3); -} - -static inline void cm_req_set_primary_local_ack_timeout(struct cm_req_msg *req_msg, - u8 local_ack_timeout) -{ - req_msg->primary_offset95 = (u8) ((req_msg->primary_offset95 & 0x07) | - (local_ack_timeout << 3)); -} - static inline __be32 cm_req_get_alt_flow_label(struct cm_req_msg *req_msg) { return cpu_to_be32(be32_to_cpu(req_msg->alt_offset132) >> 12); @@ -348,54 +158,6 @@ static inline void cm_req_set_alt_flow_label(struct cm_req_msg *req_msg, (be32_to_cpu(flow_label) << 12)); } -static inline u8 cm_req_get_alt_packet_rate(struct cm_req_msg *req_msg) -{ - return (u8) (be32_to_cpu(req_msg->alt_offset132) & 0x3F); -} - -static inline void cm_req_set_alt_packet_rate(struct cm_req_msg *req_msg, - u8 rate) -{ - req_msg->alt_offset132 = cpu_to_be32( - (be32_to_cpu(req_msg->alt_offset132) & - 0xFFFFFFC0) | (rate & 0x3F)); -} - -static inline u8 cm_req_get_alt_sl(struct cm_req_msg *req_msg) -{ - return (u8) (req_msg->alt_offset138 >> 4); -} - -static inline void cm_req_set_alt_sl(struct cm_req_msg *req_msg, u8 sl) -{ - req_msg->alt_offset138 = (u8) ((req_msg->alt_offset138 & 0x0F) | - (sl << 4)); -} - -static inline u8 cm_req_get_alt_subnet_local(struct cm_req_msg *req_msg) -{ - return (u8) ((req_msg->alt_offset138 & 0x08) >> 3); -} - -static inline void cm_req_set_alt_subnet_local(struct cm_req_msg *req_msg, - u8 subnet_local) -{ - req_msg->alt_offset138 = (u8) ((req_msg->alt_offset138 & 0xF7) | - ((subnet_local & 0x1) << 3)); -} - -static inline u8 cm_req_get_alt_local_ack_timeout(struct cm_req_msg *req_msg) -{ - return (u8) (req_msg->alt_offset139 >> 3); -} - -static inline void cm_req_set_alt_local_ack_timeout(struct cm_req_msg *req_msg, - u8 local_ack_timeout) -{ - req_msg->alt_offset139 = (u8) ((req_msg->alt_offset139 & 0x07) | - (local_ack_timeout << 3)); -} - /* Message REJected or MRAed */ enum cm_msg_response { CM_MSG_RESPONSE_REQ = 0x0, @@ -417,28 +179,6 @@ enum cm_msg_response { } __packed; -static inline u8 cm_mra_get_msg_mraed(struct cm_mra_msg *mra_msg) -{ - return (u8) (mra_msg->offset8 >> 6); -} - -static inline void cm_mra_set_msg_mraed(struct cm_mra_msg *mra_msg, u8 msg) -{ - mra_msg->offset8 = (u8) ((mra_msg->offset8 & 0x3F) | (msg << 6)); -} - -static inline u8 cm_mra_get_service_timeout(struct cm_mra_msg *mra_msg) -{ - return (u8) (mra_msg->offset9 >> 3); -} - -static inline void cm_mra_set_service_timeout(struct cm_mra_msg *mra_msg, - u8 service_timeout) -{ - mra_msg->offset9 = (u8) ((mra_msg->offset9 & 0x07) | - (service_timeout << 3)); -} - struct cm_rej_msg { struct ib_mad_hdr hdr; @@ -455,27 +195,6 @@ struct cm_rej_msg { } __packed; -static inline u8 cm_rej_get_msg_rejected(struct cm_rej_msg *rej_msg) -{ - return (u8) (rej_msg->offset8 >> 6); -} - -static inline void cm_rej_set_msg_rejected(struct cm_rej_msg *rej_msg, u8 msg) -{ - rej_msg->offset8 = (u8) ((rej_msg->offset8 & 0x3F) | (msg << 6)); -} - -static inline u8 cm_rej_get_reject_info_len(struct cm_rej_msg *rej_msg) -{ - return (u8) (rej_msg->offset9 >> 1); -} - -static inline void cm_rej_set_reject_info_len(struct cm_rej_msg *rej_msg, - u8 len) -{ - rej_msg->offset9 = (u8) ((rej_msg->offset9 & 0x1) | (len << 1)); -} - struct cm_rep_msg { struct ib_mad_hdr hdr; @@ -540,64 +259,6 @@ static inline void cm_rep_set_starting_psn(struct cm_rep_msg *rep_msg, (be32_to_cpu(rep_msg->offset20) & 0x000000FF)); } -static inline u8 cm_rep_get_target_ack_delay(struct cm_rep_msg *rep_msg) -{ - return (u8) (rep_msg->offset26 >> 3); -} - -static inline void cm_rep_set_target_ack_delay(struct cm_rep_msg *rep_msg, - u8 target_ack_delay) -{ - rep_msg->offset26 = (u8) ((rep_msg->offset26 & 0x07) | - (target_ack_delay << 3)); -} - -static inline u8 cm_rep_get_failover(struct cm_rep_msg *rep_msg) -{ - return (u8) ((rep_msg->offset26 & 0x06) >> 1); -} - -static inline void cm_rep_set_failover(struct cm_rep_msg *rep_msg, u8 failover) -{ - rep_msg->offset26 = (u8) ((rep_msg->offset26 & 0xF9) | - ((failover & 0x3) << 1)); -} - -static inline u8 cm_rep_get_flow_ctrl(struct cm_rep_msg *rep_msg) -{ - return (u8) (rep_msg->offset26 & 0x01); -} - -static inline void cm_rep_set_flow_ctrl(struct cm_rep_msg *rep_msg, - u8 flow_ctrl) -{ - rep_msg->offset26 = (u8) ((rep_msg->offset26 & 0xFE) | - (flow_ctrl & 0x1)); -} - -static inline u8 cm_rep_get_rnr_retry_count(struct cm_rep_msg *rep_msg) -{ - return (u8) (rep_msg->offset27 >> 5); -} - -static inline void cm_rep_set_rnr_retry_count(struct cm_rep_msg *rep_msg, - u8 rnr_retry_count) -{ - rep_msg->offset27 = (u8) ((rep_msg->offset27 & 0x1F) | - (rnr_retry_count << 5)); -} - -static inline u8 cm_rep_get_srq(struct cm_rep_msg *rep_msg) -{ - return (u8) ((rep_msg->offset27 >> 4) & 0x1); -} - -static inline void cm_rep_set_srq(struct cm_rep_msg *rep_msg, u8 srq) -{ - rep_msg->offset27 = (u8) ((rep_msg->offset27 & 0xEF) | - ((srq & 0x1) << 4)); -} - struct cm_rtu_msg { struct ib_mad_hdr hdr; @@ -674,26 +335,6 @@ static inline __be32 cm_lap_get_flow_label(struct cm_lap_msg *lap_msg) return cpu_to_be32(be32_to_cpu(lap_msg->offset56) >> 12); } -static inline u8 cm_lap_get_traffic_class(struct cm_lap_msg *lap_msg) -{ - return (u8) be32_to_cpu(lap_msg->offset56); -} - -static inline u8 cm_lap_get_packet_rate(struct cm_lap_msg *lap_msg) -{ - return lap_msg->offset61 & 0x3F; -} - -static inline u8 cm_lap_get_sl(struct cm_lap_msg *lap_msg) -{ - return lap_msg->offset62 >> 4; -} - -static inline u8 cm_lap_get_local_ack_timeout(struct cm_lap_msg *lap_msg) -{ - return lap_msg->offset63 >> 3; -} - struct cm_apr_msg { struct ib_mad_hdr hdr; -- cgit v1.2.3-59-g8ed1b From 01adb7f46fe455975d9aec07966e72958bc20f8e Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Thu, 16 Jan 2020 13:00:34 -0400 Subject: RDMA/cm: Use IBA functions for swapping get/set acessors Use a Coccinelle spatch script to replace CM helper functions that return/accept BE values with IBA_GET/SET versions. Applied with $ spatch --sp-file edits.sp --in-place drivers/infiniband/core/cm.c The spatch file was generated using the template pattern: @@ expression val; {struct} *msg; @@ - {old_setter}(msg, val) + IBA_SET({new_name}, msg, be{bits}_to_cpu(val)) @@ {struct} *msg; @@ - {old_getter}(msg) + cpu_to_be{bits}(IBA_GET({new_name}, msg)) Iterated for every IBA_CHECK_GET_BE()/IBA_CHECK_SET_BE() pairing. And the below iterated over all byte sizes to remove doubled byte swaps: @@ expression val; @@ -be{bits}_to_cpu(cpu_to_be{bits}(val)) +val (and __be_to_cpu and ntoh varients) Touched up with clang-format after. Link: https://lore.kernel.org/r/20200116170037.30109-5-jgg@ziepe.ca Tested-by: Leon Romanovsky Reviewed-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/cm.c | 186 +++++++------------------------------- drivers/infiniband/core/cm_msgs.h | 119 +----------------------- 2 files changed, 36 insertions(+), 269 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c index a1a968160132..976bb85b6aa6 100644 --- a/drivers/infiniband/core/cm.c +++ b/drivers/infiniband/core/cm.c @@ -1254,13 +1254,13 @@ static void cm_format_req(struct cm_req_msg *req_msg, req_msg->local_comm_id = cm_id_priv->id.local_id; req_msg->service_id = param->service_id; req_msg->local_ca_guid = cm_id_priv->id.device->node_guid; - cm_req_set_local_qpn(req_msg, cpu_to_be32(param->qp_num)); + IBA_SET(CM_REQ_LOCAL_QPN, req_msg, param->qp_num); IBA_SET(CM_REQ_INITIATOR_DEPTH, req_msg, param->initiator_depth); IBA_SET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg, param->remote_cm_response_timeout); cm_req_set_qp_type(req_msg, param->qp_type); IBA_SET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg, param->flow_control); - cm_req_set_starting_psn(req_msg, cpu_to_be32(param->starting_psn)); + IBA_SET(CM_REQ_STARTING_PSN, req_msg, param->starting_psn); IBA_SET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg, param->local_cm_response_timeout); req_msg->pkey = param->primary_path->pkey; @@ -1295,7 +1295,8 @@ static void cm_format_req(struct cm_req_msg *req_msg, req_msg->primary_local_lid = IB_LID_PERMISSIVE; req_msg->primary_remote_lid = IB_LID_PERMISSIVE; } - cm_req_set_primary_flow_label(req_msg, pri_path->flow_label); + IBA_SET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg, + be32_to_cpu(pri_path->flow_label)); IBA_SET(CM_REQ_PRIMARY_PACKET_RATE, req_msg, pri_path->rate); req_msg->primary_traffic_class = pri_path->traffic_class; req_msg->primary_hop_limit = pri_path->hop_limit; @@ -1330,8 +1331,8 @@ static void cm_format_req(struct cm_req_msg *req_msg, req_msg->alt_local_lid = IB_LID_PERMISSIVE; req_msg->alt_remote_lid = IB_LID_PERMISSIVE; } - cm_req_set_alt_flow_label(req_msg, - alt_path->flow_label); + IBA_SET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg, + be32_to_cpu(alt_path->flow_label)); IBA_SET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg, alt_path->rate); req_msg->alt_traffic_class = alt_path->traffic_class; req_msg->alt_hop_limit = alt_path->hop_limit; @@ -1437,8 +1438,8 @@ int ib_send_cm_req(struct ib_cm_id *cm_id, cm_id_priv->msg->timeout_ms = cm_id_priv->timeout_ms; cm_id_priv->msg->context[1] = (void *) (unsigned long) IB_CM_REQ_SENT; - cm_id_priv->local_qpn = cm_req_get_local_qpn(req_msg); - cm_id_priv->rq_psn = cm_req_get_starting_psn(req_msg); + cm_id_priv->local_qpn = cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg)); + cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg)); spin_lock_irqsave(&cm_id_priv->lock, flags); ret = ib_post_send_mad(cm_id_priv->msg, NULL); @@ -1548,7 +1549,8 @@ static void cm_format_paths_from_req(struct cm_req_msg *req_msg, { primary_path->dgid = req_msg->primary_local_gid; primary_path->sgid = req_msg->primary_remote_gid; - primary_path->flow_label = cm_req_get_primary_flow_label(req_msg); + primary_path->flow_label = + cpu_to_be32(IBA_GET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg)); primary_path->hop_limit = req_msg->primary_hop_limit; primary_path->traffic_class = req_msg->primary_traffic_class; primary_path->reversible = 1; @@ -1569,7 +1571,8 @@ static void cm_format_paths_from_req(struct cm_req_msg *req_msg, if (cm_req_has_alt_path(req_msg)) { alt_path->dgid = req_msg->alt_local_gid; alt_path->sgid = req_msg->alt_remote_gid; - alt_path->flow_label = cm_req_get_alt_flow_label(req_msg); + alt_path->flow_label = cpu_to_be32( + IBA_GET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg)); alt_path->hop_limit = req_msg->alt_hop_limit; alt_path->traffic_class = req_msg->alt_traffic_class; alt_path->reversible = 1; @@ -1661,9 +1664,9 @@ static void cm_format_req_event(struct cm_work *work, } param->remote_ca_guid = req_msg->local_ca_guid; param->remote_qkey = be32_to_cpu(req_msg->local_qkey); - param->remote_qpn = be32_to_cpu(cm_req_get_local_qpn(req_msg)); + param->remote_qpn = IBA_GET(CM_REQ_LOCAL_QPN, req_msg); param->qp_type = cm_req_get_qp_type(req_msg); - param->starting_psn = be32_to_cpu(cm_req_get_starting_psn(req_msg)); + param->starting_psn = IBA_GET(CM_REQ_STARTING_PSN, req_msg); param->responder_resources = IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg); param->initiator_depth = IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg); param->local_cm_response_timeout = @@ -1924,7 +1927,8 @@ static int cm_req_handler(struct cm_work *work) } cm_id_priv->timewait_info->work.remote_id = req_msg->local_comm_id; cm_id_priv->timewait_info->remote_ca_guid = req_msg->local_ca_guid; - cm_id_priv->timewait_info->remote_qpn = cm_req_get_local_qpn(req_msg); + cm_id_priv->timewait_info->remote_qpn = + cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg)); listen_cm_id_priv = cm_match_req(work, cm_id_priv); if (!listen_cm_id_priv) { @@ -1998,14 +2002,15 @@ static int cm_req_handler(struct cm_work *work) cm_id_priv->timeout_ms = cm_convert_to_ms( IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg)); cm_id_priv->max_cm_retries = IBA_GET(CM_REQ_MAX_CM_RETRIES, req_msg); - cm_id_priv->remote_qpn = cm_req_get_local_qpn(req_msg); + cm_id_priv->remote_qpn = + cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg)); cm_id_priv->initiator_depth = IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg); cm_id_priv->responder_resources = IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg); cm_id_priv->path_mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg); cm_id_priv->pkey = req_msg->pkey; - cm_id_priv->sq_psn = cm_req_get_starting_psn(req_msg); + cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg)); cm_id_priv->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg); cm_id_priv->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg); cm_id_priv->qp_type = cm_req_get_qp_type(req_msg); @@ -2032,7 +2037,7 @@ static void cm_format_rep(struct cm_rep_msg *rep_msg, cm_format_mad_hdr(&rep_msg->hdr, CM_REP_ATTR_ID, cm_id_priv->tid); rep_msg->local_comm_id = cm_id_priv->id.local_id; rep_msg->remote_comm_id = cm_id_priv->id.remote_id; - cm_rep_set_starting_psn(rep_msg, cpu_to_be32(param->starting_psn)); + IBA_SET(CM_REP_STARTING_PSN, rep_msg, param->starting_psn); rep_msg->resp_resources = param->responder_resources; IBA_SET(CM_REP_TARGET_ACK_DELAY, rep_msg, cm_id_priv->av.port->cm_dev->ack_delay); @@ -2045,10 +2050,10 @@ static void cm_format_rep(struct cm_rep_msg *rep_msg, IBA_SET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg, param->flow_control); IBA_SET(CM_REP_SRQ, rep_msg, param->srq); - cm_rep_set_local_qpn(rep_msg, cpu_to_be32(param->qp_num)); + IBA_SET(CM_REP_LOCAL_QPN, rep_msg, param->qp_num); } else { IBA_SET(CM_REP_SRQ, rep_msg, 1); - cm_rep_set_local_eecn(rep_msg, cpu_to_be32(param->qp_num)); + IBA_SET(CM_REP_LOCAL_EE_CONTEXT_NUMBER, rep_msg, param->qp_num); } if (param->private_data && param->private_data_len) @@ -2099,7 +2104,7 @@ int ib_send_cm_rep(struct ib_cm_id *cm_id, cm_id_priv->msg = msg; cm_id_priv->initiator_depth = param->initiator_depth; cm_id_priv->responder_resources = param->responder_resources; - cm_id_priv->rq_psn = cm_rep_get_starting_psn(rep_msg); + cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg)); cm_id_priv->local_qpn = cpu_to_be32(param->qp_num & 0xFFFFFF); out: spin_unlock_irqrestore(&cm_id_priv->lock, flags); @@ -2183,7 +2188,7 @@ static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type) param->remote_ca_guid = rep_msg->local_ca_guid; param->remote_qkey = be32_to_cpu(rep_msg->local_qkey); param->remote_qpn = be32_to_cpu(cm_rep_get_qpn(rep_msg, qp_type)); - param->starting_psn = be32_to_cpu(cm_rep_get_starting_psn(rep_msg)); + param->starting_psn = IBA_GET(CM_REP_STARTING_PSN, rep_msg); param->responder_resources = rep_msg->initiator_depth; param->initiator_depth = rep_msg->resp_resources; param->target_ack_delay = IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg); @@ -2320,7 +2325,7 @@ static int cm_rep_handler(struct cm_work *work) cm_id_priv->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type); cm_id_priv->initiator_depth = rep_msg->resp_resources; cm_id_priv->responder_resources = rep_msg->initiator_depth; - cm_id_priv->sq_psn = cm_rep_get_starting_psn(rep_msg); + cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg)); cm_id_priv->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg); cm_id_priv->target_ack_delay = IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg); @@ -2431,7 +2436,8 @@ static void cm_format_dreq(struct cm_dreq_msg *dreq_msg, cm_form_tid(cm_id_priv)); dreq_msg->local_comm_id = cm_id_priv->id.local_id; dreq_msg->remote_comm_id = cm_id_priv->id.remote_id; - cm_dreq_set_remote_qpn(dreq_msg, cm_id_priv->remote_qpn); + IBA_SET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg, + be32_to_cpu(cm_id_priv->remote_qpn)); if (private_data && private_data_len) memcpy(dreq_msg->private_data, private_data, private_data_len); @@ -2599,7 +2605,8 @@ static int cm_dreq_handler(struct cm_work *work) work->cm_event.private_data = &dreq_msg->private_data; spin_lock_irq(&cm_id_priv->lock); - if (cm_id_priv->local_qpn != cm_dreq_get_remote_qpn(dreq_msg)) + if (cm_id_priv->local_qpn != + cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg))) goto unlock; switch (cm_id_priv->id.state) { @@ -3067,7 +3074,8 @@ static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv, { path->dgid = lap_msg->alt_local_gid; path->sgid = lap_msg->alt_remote_gid; - path->flow_label = cm_lap_get_flow_label(lap_msg); + path->flow_label = + cpu_to_be32(IBA_GET(CM_LAP_ALTERNATE_FLOW_LABEL, lap_msg)); path->hop_limit = lap_msg->alt_hop_limit; path->traffic_class = IBA_GET(CM_LAP_ALTERNATE_TRAFFIC_CLASS, lap_msg); path->reversible = 1; @@ -3423,7 +3431,7 @@ static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg, cm_id_priv->tid); sidr_rep_msg->request_id = cm_id_priv->id.remote_id; sidr_rep_msg->status = param->status; - cm_sidr_rep_set_qpn(sidr_rep_msg, cpu_to_be32(param->qp_num)); + IBA_SET(CM_SIDR_REP_QPN, sidr_rep_msg, param->qp_num); sidr_rep_msg->service_id = cm_id_priv->id.service_id; sidr_rep_msg->qkey = cpu_to_be32(param->qkey); @@ -3494,7 +3502,7 @@ static void cm_format_sidr_rep_event(struct cm_work *work, param = &work->cm_event.param.sidr_rep_rcvd; param->status = sidr_rep_msg->status; param->qkey = be32_to_cpu(sidr_rep_msg->qkey); - param->qpn = be32_to_cpu(cm_sidr_rep_get_qpn(sidr_rep_msg)); + param->qpn = IBA_GET(CM_SIDR_REP_QPN, sidr_rep_msg); param->info = &sidr_rep_msg->info; param->info_len = sidr_rep_msg->info_length; param->sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr; @@ -4346,138 +4354,10 @@ IBA_CHECK_OFF(CM_SIDR_REP_Q_KEY, qkey); IBA_CHECK_OFF(CM_SIDR_REP_ADDITIONAL_INFORMATION, info); IBA_CHECK_OFF(CM_SIDR_REP_PRIVATE_DATA, private_data); -/* - * Check that the new macro gets the same bits as the old get function. - * - IBA_SET() IBA_GET and old get_fn all agree on the field width. - * The field width should match what IBA_SET truncates to - * - Reading from an all ones data should not return extra bits - * - Setting '1' should be the same (ie no endian problems) - */ -/* defeat builtin_constant checks */ -u64 cm_global_all_ones = 0xffffffffffffffffULL; -#define _IBA_CHECK_GET(fn, field_struct, field_offset, mask, bits) \ - ({ \ - field_struct *lmsg = (field_struct *)msg; \ - unsigned long long all_ones; \ - static_assert(sizeof(*lmsg) <= sizeof(msg)); \ - \ - bitmap_zero(msg, nbits); \ - _IBA_SET(field_struct, field_offset, mask, bits, lmsg, \ - cm_global_all_ones); \ - all_ones = (1ULL << bitmap_weight(msg, nbits)) - 1; \ - if (_IBA_GET(field_struct, field_offset, mask, bits, lmsg) != \ - all_ones) { \ - printk("Failed #1 line=%u\n", __LINE__); \ - return; \ - } \ - if (fn != all_ones) { \ - printk("Failed #2 line=%u\n", __LINE__); \ - return; \ - } \ - \ - bitmap_fill(msg, nbits); \ - if (_IBA_GET(field_struct, field_offset, mask, bits, lmsg) != \ - all_ones) { \ - printk("Failed #3 line=%u\n", __LINE__); \ - return; \ - } \ - if (fn != all_ones) { \ - printk("Failed #4 line=%u\n", __LINE__); \ - return; \ - } \ - \ - _IBA_SET(field_struct, field_offset, mask, bits, lmsg, 0); \ - if (_IBA_GET(field_struct, field_offset, mask, bits, lmsg) != \ - 0) { \ - printk("Failed #5 line=%u\n", __LINE__); \ - return; \ - } \ - if (fn != 0) { \ - printk("Failed #6 line=%u\n", __LINE__); \ - return; \ - } \ - _IBA_SET(field_struct, field_offset, mask, bits, lmsg, 1); \ - if (_IBA_GET(field_struct, field_offset, mask, bits, lmsg) != \ - 1) { \ - printk("Failed #7 line=%u\n", __LINE__); \ - return; \ - } \ - if (fn != 1) { \ - printk("Failed #8 line=%u\n", __LINE__); \ - return; \ - } \ - }) -#define IBA_CHECK_GET(field, fn_name) _IBA_CHECK_GET(fn_name(lmsg), field) -#define IBA_CHECK_GET_BE(field, fn_name) _IBA_CHECK_GET(be32_to_cpu(fn_name(lmsg)), field) - -/* - * Write the all ones value using the old setter and check that the new getter - * reads it back. - */ -#define _IBA_CHECK_SET(fn, field_struct, field_offset, mask, bits) \ - ({ \ - field_struct *lmsg = (field_struct *)msg; \ - unsigned long long all_ones; \ - static_assert(sizeof(*lmsg) <= sizeof(msg)); \ - \ - bitmap_zero(msg, nbits); \ - _IBA_SET(field_struct, field_offset, mask, bits, lmsg, \ - cm_global_all_ones); \ - all_ones = (1ULL << bitmap_weight(msg, nbits)) - 1; \ - bitmap_zero(msg, nbits); \ - fn; \ - if (_IBA_GET(field_struct, field_offset, mask, bits, lmsg) != \ - all_ones) { \ - printk("Failed #9 line=%u\n", __LINE__); \ - return; \ - } \ - all_ones = 1; \ - fn; \ - if (_IBA_GET(field_struct, field_offset, mask, bits, lmsg) != \ - 1) { \ - printk("Failed #10 line=%u\n", __LINE__); \ - return; \ - } \ - }) - -#define IBA_CHECK_SET(field, fn_name) _IBA_CHECK_SET(fn_name(lmsg, all_ones), field) -#define IBA_CHECK_SET_BE(field, fn_name) \ - _IBA_CHECK_SET(fn_name(lmsg, cpu_to_be32(all_ones)), field) - -static void self_test(void) -{ - unsigned long msg[256/4]; - const unsigned int nbits = sizeof(msg) * 8; - - printk("Running CM extractor self test\n"); - IBA_CHECK_GET_BE(CM_REQ_LOCAL_QPN, cm_req_get_local_qpn); - IBA_CHECK_SET_BE(CM_REQ_LOCAL_QPN, cm_req_set_local_qpn); - IBA_CHECK_GET_BE(CM_REQ_STARTING_PSN, cm_req_get_starting_psn); - IBA_CHECK_SET_BE(CM_REQ_STARTING_PSN, cm_req_set_starting_psn); - IBA_CHECK_GET_BE(CM_REQ_PRIMARY_FLOW_LABEL, cm_req_get_primary_flow_label); - IBA_CHECK_SET_BE(CM_REQ_PRIMARY_FLOW_LABEL, cm_req_set_primary_flow_label); - IBA_CHECK_GET_BE(CM_REQ_ALTERNATE_FLOW_LABEL, cm_req_get_alt_flow_label); - IBA_CHECK_SET_BE(CM_REQ_ALTERNATE_FLOW_LABEL, cm_req_set_alt_flow_label); - IBA_CHECK_GET_BE(CM_REP_LOCAL_QPN, cm_rep_get_local_qpn); - IBA_CHECK_SET_BE(CM_REP_LOCAL_QPN, cm_rep_set_local_qpn); - IBA_CHECK_GET_BE(CM_REP_LOCAL_EE_CONTEXT_NUMBER, cm_rep_get_local_eecn); - IBA_CHECK_SET_BE(CM_REP_LOCAL_EE_CONTEXT_NUMBER, cm_rep_set_local_eecn); - IBA_CHECK_GET_BE(CM_REP_STARTING_PSN, cm_rep_get_starting_psn); - IBA_CHECK_SET_BE(CM_REP_STARTING_PSN, cm_rep_set_starting_psn); - IBA_CHECK_GET_BE(CM_DREQ_REMOTE_QPN_EECN, cm_dreq_get_remote_qpn); - IBA_CHECK_SET_BE(CM_DREQ_REMOTE_QPN_EECN, cm_dreq_set_remote_qpn); - IBA_CHECK_GET_BE(CM_LAP_ALTERNATE_FLOW_LABEL, cm_lap_get_flow_label); - IBA_CHECK_GET_BE(CM_SIDR_REP_QPN, cm_sidr_rep_get_qpn); - IBA_CHECK_SET_BE(CM_SIDR_REP_QPN, cm_sidr_rep_set_qpn); - printk("Success!\n"); -} - static int __init ib_cm_init(void) { int ret; - self_test(); - INIT_LIST_HEAD(&cm.device_list); rwlock_init(&cm.device_lock); spin_lock_init(&cm.lock); diff --git a/drivers/infiniband/core/cm_msgs.h b/drivers/infiniband/core/cm_msgs.h index d30586b1b8a4..86ab6952d5d8 100644 --- a/drivers/infiniband/core/cm_msgs.h +++ b/drivers/infiniband/core/cm_msgs.h @@ -75,18 +75,6 @@ struct cm_req_msg { } __packed; -static inline __be32 cm_req_get_local_qpn(struct cm_req_msg *req_msg) -{ - return cpu_to_be32(be32_to_cpu(req_msg->offset32) >> 8); -} - -static inline void cm_req_set_local_qpn(struct cm_req_msg *req_msg, __be32 qpn) -{ - req_msg->offset32 = cpu_to_be32((be32_to_cpu(qpn) << 8) | - (be32_to_cpu(req_msg->offset32) & - 0x000000FF)); -} - static inline enum ib_qp_type cm_req_get_qp_type(struct cm_req_msg *req_msg) { u8 transport_type = IBA_GET(CM_REQ_TRANSPORT_SERVICE_TYPE, req_msg); @@ -118,46 +106,6 @@ static inline void cm_req_set_qp_type(struct cm_req_msg *req_msg, } } -static inline __be32 cm_req_get_starting_psn(struct cm_req_msg *req_msg) -{ - return cpu_to_be32(be32_to_cpu(req_msg->offset44) >> 8); -} - -static inline void cm_req_set_starting_psn(struct cm_req_msg *req_msg, - __be32 starting_psn) -{ - req_msg->offset44 = cpu_to_be32((be32_to_cpu(starting_psn) << 8) | - (be32_to_cpu(req_msg->offset44) & 0x000000FF)); -} - -static inline __be32 cm_req_get_primary_flow_label(struct cm_req_msg *req_msg) -{ - return cpu_to_be32(be32_to_cpu(req_msg->primary_offset88) >> 12); -} - -static inline void cm_req_set_primary_flow_label(struct cm_req_msg *req_msg, - __be32 flow_label) -{ - req_msg->primary_offset88 = cpu_to_be32( - (be32_to_cpu(req_msg->primary_offset88) & - 0x00000FFF) | - (be32_to_cpu(flow_label) << 12)); -} - -static inline __be32 cm_req_get_alt_flow_label(struct cm_req_msg *req_msg) -{ - return cpu_to_be32(be32_to_cpu(req_msg->alt_offset132) >> 12); -} - -static inline void cm_req_set_alt_flow_label(struct cm_req_msg *req_msg, - __be32 flow_label) -{ - req_msg->alt_offset132 = cpu_to_be32( - (be32_to_cpu(req_msg->alt_offset132) & - 0x00000FFF) | - (be32_to_cpu(flow_label) << 12)); -} - /* Message REJected or MRAed */ enum cm_msg_response { CM_MSG_RESPONSE_REQ = 0x0, @@ -219,44 +167,12 @@ struct cm_rep_msg { } __packed; -static inline __be32 cm_rep_get_local_qpn(struct cm_rep_msg *rep_msg) -{ - return cpu_to_be32(be32_to_cpu(rep_msg->offset12) >> 8); -} - -static inline void cm_rep_set_local_qpn(struct cm_rep_msg *rep_msg, __be32 qpn) -{ - rep_msg->offset12 = cpu_to_be32((be32_to_cpu(qpn) << 8) | - (be32_to_cpu(rep_msg->offset12) & 0x000000FF)); -} - -static inline __be32 cm_rep_get_local_eecn(struct cm_rep_msg *rep_msg) -{ - return cpu_to_be32(be32_to_cpu(rep_msg->offset16) >> 8); -} - -static inline void cm_rep_set_local_eecn(struct cm_rep_msg *rep_msg, __be32 eecn) -{ - rep_msg->offset16 = cpu_to_be32((be32_to_cpu(eecn) << 8) | - (be32_to_cpu(rep_msg->offset16) & 0x000000FF)); -} - static inline __be32 cm_rep_get_qpn(struct cm_rep_msg *rep_msg, enum ib_qp_type qp_type) { return (qp_type == IB_QPT_XRC_INI) ? - cm_rep_get_local_eecn(rep_msg) : cm_rep_get_local_qpn(rep_msg); -} - -static inline __be32 cm_rep_get_starting_psn(struct cm_rep_msg *rep_msg) -{ - return cpu_to_be32(be32_to_cpu(rep_msg->offset20) >> 8); -} - -static inline void cm_rep_set_starting_psn(struct cm_rep_msg *rep_msg, - __be32 starting_psn) -{ - rep_msg->offset20 = cpu_to_be32((be32_to_cpu(starting_psn) << 8) | - (be32_to_cpu(rep_msg->offset20) & 0x000000FF)); + cpu_to_be32(IBA_GET(CM_REP_LOCAL_EE_CONTEXT_NUMBER, + rep_msg)) : + cpu_to_be32(IBA_GET(CM_REP_LOCAL_QPN, rep_msg)); } struct cm_rtu_msg { @@ -281,17 +197,6 @@ struct cm_dreq_msg { } __packed; -static inline __be32 cm_dreq_get_remote_qpn(struct cm_dreq_msg *dreq_msg) -{ - return cpu_to_be32(be32_to_cpu(dreq_msg->offset8) >> 8); -} - -static inline void cm_dreq_set_remote_qpn(struct cm_dreq_msg *dreq_msg, __be32 qpn) -{ - dreq_msg->offset8 = cpu_to_be32((be32_to_cpu(qpn) << 8) | - (be32_to_cpu(dreq_msg->offset8) & 0x000000FF)); -} - struct cm_drep_msg { struct ib_mad_hdr hdr; @@ -330,11 +235,6 @@ struct cm_lap_msg { u8 private_data[IB_CM_LAP_PRIVATE_DATA_SIZE]; } __packed; -static inline __be32 cm_lap_get_flow_label(struct cm_lap_msg *lap_msg) -{ - return cpu_to_be32(be32_to_cpu(lap_msg->offset56) >> 12); -} - struct cm_apr_msg { struct ib_mad_hdr hdr; @@ -376,17 +276,4 @@ struct cm_sidr_rep_msg { u8 private_data[IB_CM_SIDR_REP_PRIVATE_DATA_SIZE]; } __packed; -static inline __be32 cm_sidr_rep_get_qpn(struct cm_sidr_rep_msg *sidr_rep_msg) -{ - return cpu_to_be32(be32_to_cpu(sidr_rep_msg->offset8) >> 8); -} - -static inline void cm_sidr_rep_set_qpn(struct cm_sidr_rep_msg *sidr_rep_msg, - __be32 qpn) -{ - sidr_rep_msg->offset8 = cpu_to_be32((be32_to_cpu(qpn) << 8) | - (be32_to_cpu(sidr_rep_msg->offset8) & - 0x000000FF)); -} - #endif /* CM_MSGS_H */ -- cgit v1.2.3-59-g8ed1b From 91b60a7128d96244794beb9b324eb39273872da2 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Thu, 16 Jan 2020 13:00:35 -0400 Subject: RDMA/cm: Use IBA functions for simple structure members Use a Coccinelle spatch script to replace use of simple CM structure members with IBA_GET/SET versions. Applied with $ spatch --sp-file edits.sp --in-place drivers/infiniband/core/cm.c The spatch file was generated using the template pattern: @@ expression val; {struct} *msg; @@ - msg->{old_name} = val + IBA_SET({new_name}, msg, be{bits}_to_cpu(val)) @@ {struct} *msg; @@ - msg->{old_name} + cpu_to_be{bits}(IBA_GET({new_name}, msg)) Iterated for every IBA_CHECK_OFF that isn't a CM_FIELD_MLOC. And the below iterated over all byte sizes to remove doubled byte swaps: @@ expression val; @@ -be{bits}_to_cpu(cpu_to_be{bits}(val)) +val (and __be_to_cpu and ntoh varients) Touched up with clang-format after. Link: https://lore.kernel.org/r/20200116170037.30109-6-jgg@ziepe.ca Tested-by: Leon Romanovsky Reviewed-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/cm.c | 384 +++++++++++++++++++++++++++---------------- 1 file changed, 245 insertions(+), 139 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c index 976bb85b6aa6..047ee560046e 100644 --- a/drivers/infiniband/core/cm.c +++ b/drivers/infiniband/core/cm.c @@ -1251,9 +1251,11 @@ static void cm_format_req(struct cm_req_msg *req_msg, cm_format_mad_hdr(&req_msg->hdr, CM_REQ_ATTR_ID, cm_form_tid(cm_id_priv)); - req_msg->local_comm_id = cm_id_priv->id.local_id; - req_msg->service_id = param->service_id; - req_msg->local_ca_guid = cm_id_priv->id.device->node_guid; + IBA_SET(CM_REQ_LOCAL_COMM_ID, req_msg, + be32_to_cpu(cm_id_priv->id.local_id)); + IBA_SET(CM_REQ_SERVICE_ID, req_msg, be64_to_cpu(param->service_id)); + IBA_SET(CM_REQ_LOCAL_CA_GUID, req_msg, + be64_to_cpu(cm_id_priv->id.device->node_guid)); IBA_SET(CM_REQ_LOCAL_QPN, req_msg, param->qp_num); IBA_SET(CM_REQ_INITIATOR_DEPTH, req_msg, param->initiator_depth); IBA_SET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg, @@ -1263,7 +1265,8 @@ static void cm_format_req(struct cm_req_msg *req_msg, IBA_SET(CM_REQ_STARTING_PSN, req_msg, param->starting_psn); IBA_SET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg, param->local_cm_response_timeout); - req_msg->pkey = param->primary_path->pkey; + IBA_SET(CM_REQ_PARTITION_KEY, req_msg, + be16_to_cpu(param->primary_path->pkey)); IBA_SET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg, param->primary_path->mtu); IBA_SET(CM_REQ_MAX_CM_RETRIES, req_msg, param->max_cm_retries); @@ -1286,20 +1289,26 @@ static void cm_format_req(struct cm_req_msg *req_msg, = OPA_MAKE_ID(be32_to_cpu(pri_path->opa.dlid)); } if (pri_path->hop_limit <= 1) { - req_msg->primary_local_lid = pri_ext ? 0 : - htons(ntohl(sa_path_get_slid(pri_path))); - req_msg->primary_remote_lid = pri_ext ? 0 : - htons(ntohl(sa_path_get_dlid(pri_path))); + IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg, + be16_to_cpu(pri_ext ? 0 : + htons(ntohl(sa_path_get_slid( + pri_path))))); + IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg, + be16_to_cpu(pri_ext ? 0 : + htons(ntohl(sa_path_get_dlid( + pri_path))))); } else { /* Work-around until there's a way to obtain remote LID info */ - req_msg->primary_local_lid = IB_LID_PERMISSIVE; - req_msg->primary_remote_lid = IB_LID_PERMISSIVE; + IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg, + be16_to_cpu(IB_LID_PERMISSIVE)); + IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg, + be16_to_cpu(IB_LID_PERMISSIVE)); } IBA_SET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg, be32_to_cpu(pri_path->flow_label)); IBA_SET(CM_REQ_PRIMARY_PACKET_RATE, req_msg, pri_path->rate); - req_msg->primary_traffic_class = pri_path->traffic_class; - req_msg->primary_hop_limit = pri_path->hop_limit; + IBA_SET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg, pri_path->traffic_class); + IBA_SET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg, pri_path->hop_limit); IBA_SET(CM_REQ_PRIMARY_SL, req_msg, pri_path->sl); IBA_SET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg, (pri_path->hop_limit <= 1)); @@ -1323,19 +1332,29 @@ static void cm_format_req(struct cm_req_msg *req_msg, = OPA_MAKE_ID(be32_to_cpu(alt_path->opa.dlid)); } if (alt_path->hop_limit <= 1) { - req_msg->alt_local_lid = alt_ext ? 0 : - htons(ntohl(sa_path_get_slid(alt_path))); - req_msg->alt_remote_lid = alt_ext ? 0 : - htons(ntohl(sa_path_get_dlid(alt_path))); + IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg, + be16_to_cpu( + alt_ext ? 0 : + htons(ntohl(sa_path_get_slid( + alt_path))))); + IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg, + be16_to_cpu( + alt_ext ? 0 : + htons(ntohl(sa_path_get_dlid( + alt_path))))); } else { - req_msg->alt_local_lid = IB_LID_PERMISSIVE; - req_msg->alt_remote_lid = IB_LID_PERMISSIVE; + IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg, + be16_to_cpu(IB_LID_PERMISSIVE)); + IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg, + be16_to_cpu(IB_LID_PERMISSIVE)); } IBA_SET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg, be32_to_cpu(alt_path->flow_label)); IBA_SET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg, alt_path->rate); - req_msg->alt_traffic_class = alt_path->traffic_class; - req_msg->alt_hop_limit = alt_path->hop_limit; + IBA_SET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg, + alt_path->traffic_class); + IBA_SET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg, + alt_path->hop_limit); IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, alt_path->sl); IBA_SET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg, (alt_path->hop_limit <= 1)); @@ -1477,10 +1496,12 @@ static int cm_issue_rej(struct cm_port *port, rej_msg = (struct cm_rej_msg *) msg->mad; cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, rcv_msg->hdr.tid); - rej_msg->remote_comm_id = rcv_msg->local_comm_id; - rej_msg->local_comm_id = rcv_msg->remote_comm_id; + IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg, + IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg)); + IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, + IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg)); IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, msg_rejected); - rej_msg->reason = cpu_to_be16(reason); + IBA_SET(CM_REJ_REASON, rej_msg, reason); if (ari && ari_length) { IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length); @@ -1496,7 +1517,8 @@ static int cm_issue_rej(struct cm_port *port, static bool cm_req_has_alt_path(struct cm_req_msg *req_msg) { - return ((req_msg->alt_local_lid) || + return ((cpu_to_be16( + IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg))) || (ib_is_opa_gid(&req_msg->alt_local_gid))); } @@ -1517,9 +1539,11 @@ static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg, if (primary_path->rec_type != SA_PATH_REC_TYPE_OPA) { sa_path_set_dlid(primary_path, - ntohs(req_msg->primary_local_lid)); + IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID, + req_msg)); sa_path_set_slid(primary_path, - ntohs(req_msg->primary_remote_lid)); + IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID, + req_msg)); } else { lid = opa_get_lid_from_gid(&req_msg->primary_local_gid); sa_path_set_dlid(primary_path, lid); @@ -1532,8 +1556,12 @@ static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg, return; if (alt_path->rec_type != SA_PATH_REC_TYPE_OPA) { - sa_path_set_dlid(alt_path, ntohs(req_msg->alt_local_lid)); - sa_path_set_slid(alt_path, ntohs(req_msg->alt_remote_lid)); + sa_path_set_dlid(alt_path, + IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, + req_msg)); + sa_path_set_slid(alt_path, + IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, + req_msg)); } else { lid = opa_get_lid_from_gid(&req_msg->alt_local_gid); sa_path_set_dlid(alt_path, lid); @@ -1551,10 +1579,12 @@ static void cm_format_paths_from_req(struct cm_req_msg *req_msg, primary_path->sgid = req_msg->primary_remote_gid; primary_path->flow_label = cpu_to_be32(IBA_GET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg)); - primary_path->hop_limit = req_msg->primary_hop_limit; - primary_path->traffic_class = req_msg->primary_traffic_class; + primary_path->hop_limit = IBA_GET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg); + primary_path->traffic_class = + IBA_GET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg); primary_path->reversible = 1; - primary_path->pkey = req_msg->pkey; + primary_path->pkey = + cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg)); primary_path->sl = IBA_GET(CM_REQ_PRIMARY_SL, req_msg); primary_path->mtu_selector = IB_SA_EQ; primary_path->mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg); @@ -1564,7 +1594,8 @@ static void cm_format_paths_from_req(struct cm_req_msg *req_msg, primary_path->packet_life_time = IBA_GET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg); primary_path->packet_life_time -= (primary_path->packet_life_time > 0); - primary_path->service_id = req_msg->service_id; + primary_path->service_id = + cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)); if (sa_path_is_roce(primary_path)) primary_path->roce.route_resolved = false; @@ -1573,10 +1604,13 @@ static void cm_format_paths_from_req(struct cm_req_msg *req_msg, alt_path->sgid = req_msg->alt_remote_gid; alt_path->flow_label = cpu_to_be32( IBA_GET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg)); - alt_path->hop_limit = req_msg->alt_hop_limit; - alt_path->traffic_class = req_msg->alt_traffic_class; + alt_path->hop_limit = + IBA_GET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg); + alt_path->traffic_class = + IBA_GET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg); alt_path->reversible = 1; - alt_path->pkey = req_msg->pkey; + alt_path->pkey = + cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg)); alt_path->sl = IBA_GET(CM_REQ_ALTERNATE_SL, req_msg); alt_path->mtu_selector = IB_SA_EQ; alt_path->mtu = @@ -1587,7 +1621,8 @@ static void cm_format_paths_from_req(struct cm_req_msg *req_msg, alt_path->packet_life_time = IBA_GET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg); alt_path->packet_life_time -= (alt_path->packet_life_time > 0); - alt_path->service_id = req_msg->service_id; + alt_path->service_id = + cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)); if (sa_path_is_roce(alt_path)) alt_path->roce.route_resolved = false; @@ -1662,8 +1697,9 @@ static void cm_format_req_event(struct cm_work *work, } else { param->alternate_path = NULL; } - param->remote_ca_guid = req_msg->local_ca_guid; - param->remote_qkey = be32_to_cpu(req_msg->local_qkey); + param->remote_ca_guid = + cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg)); + param->remote_qkey = IBA_GET(CM_REQ_LOCAL_Q_KEY, req_msg); param->remote_qpn = IBA_GET(CM_REQ_LOCAL_QPN, req_msg); param->qp_type = cm_req_get_qp_type(req_msg); param->starting_psn = IBA_GET(CM_REQ_STARTING_PSN, req_msg); @@ -1713,8 +1749,10 @@ static void cm_format_mra(struct cm_mra_msg *mra_msg, { cm_format_mad_hdr(&mra_msg->hdr, CM_MRA_ATTR_ID, cm_id_priv->tid); IBA_SET(CM_MRA_MESSAGE_MRAED, mra_msg, msg_mraed); - mra_msg->local_comm_id = cm_id_priv->id.local_id; - mra_msg->remote_comm_id = cm_id_priv->id.remote_id; + IBA_SET(CM_MRA_LOCAL_COMM_ID, mra_msg, + be32_to_cpu(cm_id_priv->id.local_id)); + IBA_SET(CM_MRA_REMOTE_COMM_ID, mra_msg, + be32_to_cpu(cm_id_priv->id.remote_id)); IBA_SET(CM_MRA_SERVICE_TIMEOUT, mra_msg, service_timeout); if (private_data && private_data_len) @@ -1730,30 +1768,34 @@ static void cm_format_rej(struct cm_rej_msg *rej_msg, u8 private_data_len) { cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, cm_id_priv->tid); - rej_msg->remote_comm_id = cm_id_priv->id.remote_id; + IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg, + be32_to_cpu(cm_id_priv->id.remote_id)); switch(cm_id_priv->id.state) { case IB_CM_REQ_RCVD: - rej_msg->local_comm_id = 0; + IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, be32_to_cpu(0)); IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ); break; case IB_CM_MRA_REQ_SENT: - rej_msg->local_comm_id = cm_id_priv->id.local_id; + IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, + be32_to_cpu(cm_id_priv->id.local_id)); IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ); break; case IB_CM_REP_RCVD: case IB_CM_MRA_REP_SENT: - rej_msg->local_comm_id = cm_id_priv->id.local_id; + IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, + be32_to_cpu(cm_id_priv->id.local_id)); IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REP); break; default: - rej_msg->local_comm_id = cm_id_priv->id.local_id; + IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, + be32_to_cpu(cm_id_priv->id.local_id)); IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_OTHER); break; } - rej_msg->reason = cpu_to_be16(reason); + IBA_SET(CM_REJ_REASON, rej_msg, reason); if (ari && ari_length) { IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length); memcpy(rej_msg->ari, ari, ari_length); @@ -1850,8 +1892,9 @@ static struct cm_id_private * cm_match_req(struct cm_work *work, } /* Find matching listen request. */ - listen_cm_id_priv = cm_find_listen(cm_id_priv->id.device, - req_msg->service_id); + listen_cm_id_priv = cm_find_listen( + cm_id_priv->id.device, + cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg))); if (!listen_cm_id_priv) { cm_cleanup_timewait(cm_id_priv->timewait_info); spin_unlock_irq(&cm.lock); @@ -1877,23 +1920,31 @@ out: static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc) { if (!IBA_GET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg)) { - if (req_msg->primary_local_lid == IB_LID_PERMISSIVE) { - req_msg->primary_local_lid = ib_lid_be16(wc->slid); + if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID, + req_msg)) == IB_LID_PERMISSIVE) { + IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg, + be16_to_cpu(ib_lid_be16(wc->slid))); IBA_SET(CM_REQ_PRIMARY_SL, req_msg, wc->sl); } - if (req_msg->primary_remote_lid == IB_LID_PERMISSIVE) - req_msg->primary_remote_lid = cpu_to_be16(wc->dlid_path_bits); + if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID, + req_msg)) == IB_LID_PERMISSIVE) + IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg, + wc->dlid_path_bits); } if (!IBA_GET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg)) { - if (req_msg->alt_local_lid == IB_LID_PERMISSIVE) { - req_msg->alt_local_lid = ib_lid_be16(wc->slid); + if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, + req_msg)) == IB_LID_PERMISSIVE) { + IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg, + be16_to_cpu(ib_lid_be16(wc->slid))); IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, wc->sl); } - if (req_msg->alt_remote_lid == IB_LID_PERMISSIVE) - req_msg->alt_remote_lid = cpu_to_be16(wc->dlid_path_bits); + if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, + req_msg)) == IB_LID_PERMISSIVE) + IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg, + wc->dlid_path_bits); } } @@ -1913,7 +1964,8 @@ static int cm_req_handler(struct cm_work *work) return PTR_ERR(cm_id); cm_id_priv = container_of(cm_id, struct cm_id_private, id); - cm_id_priv->id.remote_id = req_msg->local_comm_id; + cm_id_priv->id.remote_id = + cpu_to_be32(IBA_GET(CM_REQ_LOCAL_COMM_ID, req_msg)); ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc, work->mad_recv_wc->recv_buf.grh, &cm_id_priv->av); @@ -1925,8 +1977,10 @@ static int cm_req_handler(struct cm_work *work) ret = PTR_ERR(cm_id_priv->timewait_info); goto destroy; } - cm_id_priv->timewait_info->work.remote_id = req_msg->local_comm_id; - cm_id_priv->timewait_info->remote_ca_guid = req_msg->local_ca_guid; + cm_id_priv->timewait_info->work.remote_id = + cpu_to_be32(IBA_GET(CM_REQ_LOCAL_COMM_ID, req_msg)); + cm_id_priv->timewait_info->remote_ca_guid = + cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg)); cm_id_priv->timewait_info->remote_qpn = cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg)); @@ -1940,7 +1994,8 @@ static int cm_req_handler(struct cm_work *work) cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler; cm_id_priv->id.context = listen_cm_id_priv->id.context; - cm_id_priv->id.service_id = req_msg->service_id; + cm_id_priv->id.service_id = + cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)); cm_id_priv->id.service_mask = ~cpu_to_be64(0); cm_process_routed_req(req_msg, work->mad_recv_wc->wc); @@ -2009,7 +2064,7 @@ static int cm_req_handler(struct cm_work *work) cm_id_priv->responder_resources = IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg); cm_id_priv->path_mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg); - cm_id_priv->pkey = req_msg->pkey; + cm_id_priv->pkey = cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg)); cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg)); cm_id_priv->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg); cm_id_priv->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg); @@ -2035,18 +2090,23 @@ static void cm_format_rep(struct cm_rep_msg *rep_msg, struct ib_cm_rep_param *param) { cm_format_mad_hdr(&rep_msg->hdr, CM_REP_ATTR_ID, cm_id_priv->tid); - rep_msg->local_comm_id = cm_id_priv->id.local_id; - rep_msg->remote_comm_id = cm_id_priv->id.remote_id; + IBA_SET(CM_REP_LOCAL_COMM_ID, rep_msg, + be32_to_cpu(cm_id_priv->id.local_id)); + IBA_SET(CM_REP_REMOTE_COMM_ID, rep_msg, + be32_to_cpu(cm_id_priv->id.remote_id)); IBA_SET(CM_REP_STARTING_PSN, rep_msg, param->starting_psn); - rep_msg->resp_resources = param->responder_resources; + IBA_SET(CM_REP_RESPONDER_RESOURCES, rep_msg, + param->responder_resources); IBA_SET(CM_REP_TARGET_ACK_DELAY, rep_msg, cm_id_priv->av.port->cm_dev->ack_delay); IBA_SET(CM_REP_FAILOVER_ACCEPTED, rep_msg, param->failover_accepted); IBA_SET(CM_REP_RNR_RETRY_COUNT, rep_msg, param->rnr_retry_count); - rep_msg->local_ca_guid = cm_id_priv->id.device->node_guid; + IBA_SET(CM_REP_LOCAL_CA_GUID, rep_msg, + be64_to_cpu(cm_id_priv->id.device->node_guid)); if (cm_id_priv->qp_type != IB_QPT_XRC_TGT) { - rep_msg->initiator_depth = param->initiator_depth; + IBA_SET(CM_REP_INITIATOR_DEPTH, rep_msg, + param->initiator_depth); IBA_SET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg, param->flow_control); IBA_SET(CM_REP_SRQ, rep_msg, param->srq); @@ -2118,8 +2178,10 @@ static void cm_format_rtu(struct cm_rtu_msg *rtu_msg, u8 private_data_len) { cm_format_mad_hdr(&rtu_msg->hdr, CM_RTU_ATTR_ID, cm_id_priv->tid); - rtu_msg->local_comm_id = cm_id_priv->id.local_id; - rtu_msg->remote_comm_id = cm_id_priv->id.remote_id; + IBA_SET(CM_RTU_LOCAL_COMM_ID, rtu_msg, + be32_to_cpu(cm_id_priv->id.local_id)); + IBA_SET(CM_RTU_REMOTE_COMM_ID, rtu_msg, + be32_to_cpu(cm_id_priv->id.remote_id)); if (private_data && private_data_len) memcpy(rtu_msg->private_data, private_data, private_data_len); @@ -2185,12 +2247,13 @@ static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type) rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad; param = &work->cm_event.param.rep_rcvd; - param->remote_ca_guid = rep_msg->local_ca_guid; - param->remote_qkey = be32_to_cpu(rep_msg->local_qkey); + param->remote_ca_guid = + cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg)); + param->remote_qkey = IBA_GET(CM_REP_LOCAL_Q_KEY, rep_msg); param->remote_qpn = be32_to_cpu(cm_rep_get_qpn(rep_msg, qp_type)); param->starting_psn = IBA_GET(CM_REP_STARTING_PSN, rep_msg); - param->responder_resources = rep_msg->initiator_depth; - param->initiator_depth = rep_msg->resp_resources; + param->responder_resources = IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg); + param->initiator_depth = IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg); param->target_ack_delay = IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg); param->failover_accepted = IBA_GET(CM_REP_FAILOVER_ACCEPTED, rep_msg); param->flow_control = IBA_GET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg); @@ -2207,8 +2270,9 @@ static void cm_dup_rep_handler(struct cm_work *work) int ret; rep_msg = (struct cm_rep_msg *) work->mad_recv_wc->recv_buf.mad; - cm_id_priv = cm_acquire_id(rep_msg->remote_comm_id, - rep_msg->local_comm_id); + cm_id_priv = cm_acquire_id( + cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)), + cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg))); if (!cm_id_priv) return; @@ -2252,11 +2316,12 @@ static int cm_rep_handler(struct cm_work *work) struct cm_timewait_info *timewait_info; rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad; - cm_id_priv = cm_acquire_id(rep_msg->remote_comm_id, 0); + cm_id_priv = cm_acquire_id( + cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)), 0); if (!cm_id_priv) { cm_dup_rep_handler(work); pr_debug("%s: remote_comm_id %d, no cm_id_priv\n", __func__, - be32_to_cpu(rep_msg->remote_comm_id)); + IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)); return -EINVAL; } @@ -2270,15 +2335,18 @@ static int cm_rep_handler(struct cm_work *work) default: spin_unlock_irq(&cm_id_priv->lock); ret = -EINVAL; - pr_debug("%s: cm_id_priv->id.state: %d, local_comm_id %d, remote_comm_id %d\n", - __func__, cm_id_priv->id.state, - be32_to_cpu(rep_msg->local_comm_id), - be32_to_cpu(rep_msg->remote_comm_id)); + pr_debug( + "%s: cm_id_priv->id.state: %d, local_comm_id %d, remote_comm_id %d\n", + __func__, cm_id_priv->id.state, + IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg), + IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)); goto error; } - cm_id_priv->timewait_info->work.remote_id = rep_msg->local_comm_id; - cm_id_priv->timewait_info->remote_ca_guid = rep_msg->local_ca_guid; + cm_id_priv->timewait_info->work.remote_id = + cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg)); + cm_id_priv->timewait_info->remote_ca_guid = + cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg)); cm_id_priv->timewait_info->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type); spin_lock(&cm.lock); @@ -2288,7 +2356,7 @@ static int cm_rep_handler(struct cm_work *work) spin_unlock_irq(&cm_id_priv->lock); ret = -EINVAL; pr_debug("%s: Failed to insert remote id %d\n", __func__, - be32_to_cpu(rep_msg->remote_comm_id)); + IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)); goto error; } /* Check for a stale connection. */ @@ -2306,9 +2374,10 @@ static int cm_rep_handler(struct cm_work *work) IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REP, NULL, 0); ret = -EINVAL; - pr_debug("%s: Stale connection. local_comm_id %d, remote_comm_id %d\n", - __func__, be32_to_cpu(rep_msg->local_comm_id), - be32_to_cpu(rep_msg->remote_comm_id)); + pr_debug( + "%s: Stale connection. local_comm_id %d, remote_comm_id %d\n", + __func__, IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg), + IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)); if (cur_cm_id_priv) { cm_id = &cur_cm_id_priv->id; @@ -2321,10 +2390,13 @@ static int cm_rep_handler(struct cm_work *work) spin_unlock(&cm.lock); cm_id_priv->id.state = IB_CM_REP_RCVD; - cm_id_priv->id.remote_id = rep_msg->local_comm_id; + cm_id_priv->id.remote_id = + cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg)); cm_id_priv->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type); - cm_id_priv->initiator_depth = rep_msg->resp_resources; - cm_id_priv->responder_resources = rep_msg->initiator_depth; + cm_id_priv->initiator_depth = + IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg); + cm_id_priv->responder_resources = + IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg); cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg)); cm_id_priv->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg); cm_id_priv->target_ack_delay = @@ -2394,8 +2466,9 @@ static int cm_rtu_handler(struct cm_work *work) int ret; rtu_msg = (struct cm_rtu_msg *)work->mad_recv_wc->recv_buf.mad; - cm_id_priv = cm_acquire_id(rtu_msg->remote_comm_id, - rtu_msg->local_comm_id); + cm_id_priv = cm_acquire_id( + cpu_to_be32(IBA_GET(CM_RTU_REMOTE_COMM_ID, rtu_msg)), + cpu_to_be32(IBA_GET(CM_RTU_LOCAL_COMM_ID, rtu_msg))); if (!cm_id_priv) return -EINVAL; @@ -2434,8 +2507,10 @@ static void cm_format_dreq(struct cm_dreq_msg *dreq_msg, { cm_format_mad_hdr(&dreq_msg->hdr, CM_DREQ_ATTR_ID, cm_form_tid(cm_id_priv)); - dreq_msg->local_comm_id = cm_id_priv->id.local_id; - dreq_msg->remote_comm_id = cm_id_priv->id.remote_id; + IBA_SET(CM_DREQ_LOCAL_COMM_ID, dreq_msg, + be32_to_cpu(cm_id_priv->id.local_id)); + IBA_SET(CM_DREQ_REMOTE_COMM_ID, dreq_msg, + be32_to_cpu(cm_id_priv->id.remote_id)); IBA_SET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg, be32_to_cpu(cm_id_priv->remote_qpn)); @@ -2500,8 +2575,10 @@ static void cm_format_drep(struct cm_drep_msg *drep_msg, u8 private_data_len) { cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, cm_id_priv->tid); - drep_msg->local_comm_id = cm_id_priv->id.local_id; - drep_msg->remote_comm_id = cm_id_priv->id.remote_id; + IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg, + be32_to_cpu(cm_id_priv->id.local_id)); + IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg, + be32_to_cpu(cm_id_priv->id.remote_id)); if (private_data && private_data_len) memcpy(drep_msg->private_data, private_data, private_data_len); @@ -2572,8 +2649,10 @@ static int cm_issue_drep(struct cm_port *port, drep_msg = (struct cm_drep_msg *) msg->mad; cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, dreq_msg->hdr.tid); - drep_msg->remote_comm_id = dreq_msg->local_comm_id; - drep_msg->local_comm_id = dreq_msg->remote_comm_id; + IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg, + IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg)); + IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg, + IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)); ret = ib_post_send_mad(msg, NULL); if (ret) @@ -2590,15 +2669,17 @@ static int cm_dreq_handler(struct cm_work *work) int ret; dreq_msg = (struct cm_dreq_msg *)work->mad_recv_wc->recv_buf.mad; - cm_id_priv = cm_acquire_id(dreq_msg->remote_comm_id, - dreq_msg->local_comm_id); + cm_id_priv = cm_acquire_id( + cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)), + cpu_to_be32(IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg))); if (!cm_id_priv) { atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES]. counter[CM_DREQ_COUNTER]); cm_issue_drep(work->port, work->mad_recv_wc); - pr_debug("%s: no cm_id_priv, local_comm_id %d, remote_comm_id %d\n", - __func__, be32_to_cpu(dreq_msg->local_comm_id), - be32_to_cpu(dreq_msg->remote_comm_id)); + pr_debug( + "%s: no cm_id_priv, local_comm_id %d, remote_comm_id %d\n", + __func__, IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg), + IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)); return -EINVAL; } @@ -2672,8 +2753,9 @@ static int cm_drep_handler(struct cm_work *work) int ret; drep_msg = (struct cm_drep_msg *)work->mad_recv_wc->recv_buf.mad; - cm_id_priv = cm_acquire_id(drep_msg->remote_comm_id, - drep_msg->local_comm_id); + cm_id_priv = cm_acquire_id( + cpu_to_be32(IBA_GET(CM_DREP_REMOTE_COMM_ID, drep_msg)), + cpu_to_be32(IBA_GET(CM_DREP_LOCAL_COMM_ID, drep_msg))); if (!cm_id_priv) return -EINVAL; @@ -2775,7 +2857,7 @@ static void cm_format_rej_event(struct cm_work *work) param = &work->cm_event.param.rej_rcvd; param->ari = rej_msg->ari; param->ari_length = IBA_GET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg); - param->reason = __be16_to_cpu(rej_msg->reason); + param->reason = IBA_GET(CM_REJ_REASON, rej_msg); work->cm_event.private_data = &rej_msg->private_data; } @@ -2785,9 +2867,9 @@ static struct cm_id_private * cm_acquire_rejected_id(struct cm_rej_msg *rej_msg) struct cm_id_private *cm_id_priv; __be32 remote_id; - remote_id = rej_msg->local_comm_id; + remote_id = cpu_to_be32(IBA_GET(CM_REJ_LOCAL_COMM_ID, rej_msg)); - if (__be16_to_cpu(rej_msg->reason) == IB_CM_REJ_TIMEOUT) { + if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_TIMEOUT) { spin_lock_irq(&cm.lock); timewait_info = cm_find_remote_id( *((__be64 *) rej_msg->ari), remote_id); @@ -2800,9 +2882,13 @@ static struct cm_id_private * cm_acquire_rejected_id(struct cm_rej_msg *rej_msg) spin_unlock_irq(&cm.lock); } else if (IBA_GET(CM_REJ_MESSAGE_REJECTED, rej_msg) == CM_MSG_RESPONSE_REQ) - cm_id_priv = cm_acquire_id(rej_msg->remote_comm_id, 0); + cm_id_priv = cm_acquire_id( + cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)), + 0); else - cm_id_priv = cm_acquire_id(rej_msg->remote_comm_id, remote_id); + cm_id_priv = cm_acquire_id( + cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)), + remote_id); return cm_id_priv; } @@ -2830,7 +2916,7 @@ static int cm_rej_handler(struct cm_work *work) /* fall through */ case IB_CM_REQ_RCVD: case IB_CM_MRA_REQ_SENT: - if (__be16_to_cpu(rej_msg->reason) == IB_CM_REJ_STALE_CONN) + if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_STALE_CONN) cm_enter_timewait(cm_id_priv); else cm_reset_to_idle(cm_id_priv); @@ -2962,11 +3048,14 @@ static struct cm_id_private * cm_acquire_mraed_id(struct cm_mra_msg *mra_msg) { switch (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg)) { case CM_MSG_RESPONSE_REQ: - return cm_acquire_id(mra_msg->remote_comm_id, 0); + return cm_acquire_id( + cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)), + 0); case CM_MSG_RESPONSE_REP: case CM_MSG_RESPONSE_OTHER: - return cm_acquire_id(mra_msg->remote_comm_id, - mra_msg->local_comm_id); + return cm_acquire_id( + cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)), + cpu_to_be32(IBA_GET(CM_MRA_LOCAL_COMM_ID, mra_msg))); default: return NULL; } @@ -3057,8 +3146,10 @@ static void cm_format_path_lid_from_lap(struct cm_lap_msg *lap_msg, u32 lid; if (path->rec_type != SA_PATH_REC_TYPE_OPA) { - sa_path_set_dlid(path, ntohs(lap_msg->alt_local_lid)); - sa_path_set_slid(path, ntohs(lap_msg->alt_remote_lid)); + sa_path_set_dlid(path, IBA_GET(CM_LAP_ALTERNATE_LOCAL_PORT_LID, + lap_msg)); + sa_path_set_slid(path, IBA_GET(CM_LAP_ALTERNATE_REMOTE_PORT_LID, + lap_msg)); } else { lid = opa_get_lid_from_gid(&lap_msg->alt_local_gid); sa_path_set_dlid(path, lid); @@ -3076,7 +3167,7 @@ static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv, path->sgid = lap_msg->alt_remote_gid; path->flow_label = cpu_to_be32(IBA_GET(CM_LAP_ALTERNATE_FLOW_LABEL, lap_msg)); - path->hop_limit = lap_msg->alt_hop_limit; + path->hop_limit = IBA_GET(CM_LAP_ALTERNATE_HOP_LIMIT, lap_msg); path->traffic_class = IBA_GET(CM_LAP_ALTERNATE_TRAFFIC_CLASS, lap_msg); path->reversible = 1; path->pkey = cm_id_priv->pkey; @@ -3109,8 +3200,9 @@ static int cm_lap_handler(struct cm_work *work) /* todo: verify LAP request and send reject APR if invalid. */ lap_msg = (struct cm_lap_msg *)work->mad_recv_wc->recv_buf.mad; - cm_id_priv = cm_acquire_id(lap_msg->remote_comm_id, - lap_msg->local_comm_id); + cm_id_priv = cm_acquire_id( + cpu_to_be32(IBA_GET(CM_LAP_REMOTE_COMM_ID, lap_msg)), + cpu_to_be32(IBA_GET(CM_LAP_LOCAL_COMM_ID, lap_msg))); if (!cm_id_priv) return -EINVAL; @@ -3201,14 +3293,17 @@ static int cm_apr_handler(struct cm_work *work) return -EINVAL; apr_msg = (struct cm_apr_msg *)work->mad_recv_wc->recv_buf.mad; - cm_id_priv = cm_acquire_id(apr_msg->remote_comm_id, - apr_msg->local_comm_id); + cm_id_priv = cm_acquire_id( + cpu_to_be32(IBA_GET(CM_APR_REMOTE_COMM_ID, apr_msg)), + cpu_to_be32(IBA_GET(CM_APR_LOCAL_COMM_ID, apr_msg))); if (!cm_id_priv) return -EINVAL; /* Unmatched reply. */ - work->cm_event.param.apr_rcvd.ap_status = apr_msg->ap_status; + work->cm_event.param.apr_rcvd.ap_status = + IBA_GET(CM_APR_AR_STATUS, apr_msg); work->cm_event.param.apr_rcvd.apr_info = &apr_msg->info; - work->cm_event.param.apr_rcvd.info_len = apr_msg->info_length; + work->cm_event.param.apr_rcvd.info_len = + IBA_GET(CM_APR_ADDITIONAL_INFORMATION_LENGTH, apr_msg); work->cm_event.private_data = &apr_msg->private_data; spin_lock_irq(&cm_id_priv->lock); @@ -3281,9 +3376,12 @@ static void cm_format_sidr_req(struct cm_sidr_req_msg *sidr_req_msg, { cm_format_mad_hdr(&sidr_req_msg->hdr, CM_SIDR_REQ_ATTR_ID, cm_form_tid(cm_id_priv)); - sidr_req_msg->request_id = cm_id_priv->id.local_id; - sidr_req_msg->pkey = param->path->pkey; - sidr_req_msg->service_id = param->service_id; + IBA_SET(CM_SIDR_REQ_REQUESTID, sidr_req_msg, + be32_to_cpu(cm_id_priv->id.local_id)); + IBA_SET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg, + be16_to_cpu(param->path->pkey)); + IBA_SET(CM_SIDR_REQ_SERVICEID, sidr_req_msg, + be64_to_cpu(param->service_id)); if (param->private_data && param->private_data_len) memcpy(sidr_req_msg->private_data, param->private_data, @@ -3351,9 +3449,10 @@ static void cm_format_sidr_req_event(struct cm_work *work, sidr_req_msg = (struct cm_sidr_req_msg *) work->mad_recv_wc->recv_buf.mad; param = &work->cm_event.param.sidr_req_rcvd; - param->pkey = __be16_to_cpu(sidr_req_msg->pkey); + param->pkey = IBA_GET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg); param->listen_id = listen_id; - param->service_id = sidr_req_msg->service_id; + param->service_id = + cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg)); param->bth_pkey = cm_get_bth_pkey(work); param->port = work->port->port_num; param->sgid_attr = rx_cm_id->av.ah_attr.grh.sgid_attr; @@ -3385,7 +3484,8 @@ static int cm_sidr_req_handler(struct cm_work *work) if (ret) goto out; - cm_id_priv->id.remote_id = sidr_req_msg->request_id; + cm_id_priv->id.remote_id = + cpu_to_be32(IBA_GET(CM_SIDR_REQ_REQUESTID, sidr_req_msg)); cm_id_priv->tid = sidr_req_msg->hdr.tid; atomic_inc(&cm_id_priv->work_count); @@ -3398,8 +3498,9 @@ static int cm_sidr_req_handler(struct cm_work *work) goto out; /* Duplicate message. */ } cm_id_priv->id.state = IB_CM_SIDR_REQ_RCVD; - cur_cm_id_priv = cm_find_listen(cm_id->device, - sidr_req_msg->service_id); + cur_cm_id_priv = cm_find_listen( + cm_id->device, + cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg))); if (!cur_cm_id_priv) { spin_unlock_irq(&cm.lock); cm_reject_sidr_req(cm_id_priv, IB_SIDR_UNSUPPORTED); @@ -3411,7 +3512,8 @@ static int cm_sidr_req_handler(struct cm_work *work) cm_id_priv->id.cm_handler = cur_cm_id_priv->id.cm_handler; cm_id_priv->id.context = cur_cm_id_priv->id.context; - cm_id_priv->id.service_id = sidr_req_msg->service_id; + cm_id_priv->id.service_id = + cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg)); cm_id_priv->id.service_mask = ~cpu_to_be64(0); cm_format_sidr_req_event(work, cm_id_priv, &cur_cm_id_priv->id); @@ -3429,11 +3531,13 @@ static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg, { cm_format_mad_hdr(&sidr_rep_msg->hdr, CM_SIDR_REP_ATTR_ID, cm_id_priv->tid); - sidr_rep_msg->request_id = cm_id_priv->id.remote_id; - sidr_rep_msg->status = param->status; + IBA_SET(CM_SIDR_REP_REQUESTID, sidr_rep_msg, + be32_to_cpu(cm_id_priv->id.remote_id)); + IBA_SET(CM_SIDR_REP_STATUS, sidr_rep_msg, param->status); IBA_SET(CM_SIDR_REP_QPN, sidr_rep_msg, param->qp_num); - sidr_rep_msg->service_id = cm_id_priv->id.service_id; - sidr_rep_msg->qkey = cpu_to_be32(param->qkey); + IBA_SET(CM_SIDR_REP_SERVICEID, sidr_rep_msg, + be64_to_cpu(cm_id_priv->id.service_id)); + IBA_SET(CM_SIDR_REP_Q_KEY, sidr_rep_msg, param->qkey); if (param->info && param->info_length) memcpy(sidr_rep_msg->info, param->info, param->info_length); @@ -3500,11 +3604,12 @@ static void cm_format_sidr_rep_event(struct cm_work *work, sidr_rep_msg = (struct cm_sidr_rep_msg *) work->mad_recv_wc->recv_buf.mad; param = &work->cm_event.param.sidr_rep_rcvd; - param->status = sidr_rep_msg->status; - param->qkey = be32_to_cpu(sidr_rep_msg->qkey); + param->status = IBA_GET(CM_SIDR_REP_STATUS, sidr_rep_msg); + param->qkey = IBA_GET(CM_SIDR_REP_Q_KEY, sidr_rep_msg); param->qpn = IBA_GET(CM_SIDR_REP_QPN, sidr_rep_msg); param->info = &sidr_rep_msg->info; - param->info_len = sidr_rep_msg->info_length; + param->info_len = IBA_GET(CM_SIDR_REP_ADDITIONAL_INFORMATION_LENGTH, + sidr_rep_msg); param->sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr; work->cm_event.private_data = &sidr_rep_msg->private_data; } @@ -3516,7 +3621,8 @@ static int cm_sidr_rep_handler(struct cm_work *work) sidr_rep_msg = (struct cm_sidr_rep_msg *) work->mad_recv_wc->recv_buf.mad; - cm_id_priv = cm_acquire_id(sidr_rep_msg->request_id, 0); + cm_id_priv = cm_acquire_id( + cpu_to_be32(IBA_GET(CM_SIDR_REP_REQUESTID, sidr_rep_msg)), 0); if (!cm_id_priv) return -EINVAL; /* Unmatched reply. */ -- cgit v1.2.3-59-g8ed1b From 4ca662a30a3c9c8a320549f7ef3066921e36f0b4 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Thu, 16 Jan 2020 13:00:36 -0400 Subject: RDMA/cm: Use IBA functions for complex structure members Use a Coccinelle spatch to replace CM structure members used as structures, arrays, or pointers with IBA_GET/SET versions. Applied with $ spatch --sp-file edits.sp --in-place drivers/infiniband/core/cm.c The spatch file was generated using the template pattern: @@ expression src; expression len; {struct} *msg; @@ - memcpy(msg->{old_name}, src, len) + IBA_SET_MEM({new_name}, msg, src, len) @@ {struct} *msg; identifier x; @@ - msg->{old_name}.x + IBA_GET_MEM_PTR({new_name}, msg)->x @@ {struct} *msg; @@ - &msg->{old_name} + IBA_GET_MEM_PTR({new_name}, msg) For GIDs: @@ {struct} *msg; @@ - msg->{old_name} + *IBA_GET_MEM_PTR({new_name}, msg) For non-GIDs: @@ {struct} *msg; @@ - msg->{old_name} + IBA_GET_MEM_PTR({new_name}, msg) Iterated for every remaining IBA_CHECK_OFF()/IBA_CHECK_GET() pairing. Touched up with clang-format after. Link: https://lore.kernel.org/r/20200116170037.30109-7-jgg@ziepe.ca Tested-by: Leon Romanovsky Reviewed-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/cm.c | 171 +++++++++++++++++++++++++++---------------- 1 file changed, 107 insertions(+), 64 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c index 047ee560046e..5ccd59f1ebb8 100644 --- a/drivers/infiniband/core/cm.c +++ b/drivers/infiniband/core/cm.c @@ -1280,13 +1280,17 @@ static void cm_format_req(struct cm_req_msg *req_msg, IBA_SET(CM_REQ_SRQ, req_msg, param->srq); } - req_msg->primary_local_gid = pri_path->sgid; - req_msg->primary_remote_gid = pri_path->dgid; + *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg) = + pri_path->sgid; + *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg) = + pri_path->dgid; if (pri_ext) { - req_msg->primary_local_gid.global.interface_id - = OPA_MAKE_ID(be32_to_cpu(pri_path->opa.slid)); - req_msg->primary_remote_gid.global.interface_id - = OPA_MAKE_ID(be32_to_cpu(pri_path->opa.dlid)); + IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg) + ->global.interface_id = + OPA_MAKE_ID(be32_to_cpu(pri_path->opa.slid)); + IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg) + ->global.interface_id = + OPA_MAKE_ID(be32_to_cpu(pri_path->opa.dlid)); } if (pri_path->hop_limit <= 1) { IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg, @@ -1323,13 +1327,19 @@ static void cm_format_req(struct cm_req_msg *req_msg, alt_ext = opa_is_extended_lid(alt_path->opa.dlid, alt_path->opa.slid); - req_msg->alt_local_gid = alt_path->sgid; - req_msg->alt_remote_gid = alt_path->dgid; + *IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg) = + alt_path->sgid; + *IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg) = + alt_path->dgid; if (alt_ext) { - req_msg->alt_local_gid.global.interface_id - = OPA_MAKE_ID(be32_to_cpu(alt_path->opa.slid)); - req_msg->alt_remote_gid.global.interface_id - = OPA_MAKE_ID(be32_to_cpu(alt_path->opa.dlid)); + IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID, + req_msg) + ->global.interface_id = + OPA_MAKE_ID(be32_to_cpu(alt_path->opa.slid)); + IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID, + req_msg) + ->global.interface_id = + OPA_MAKE_ID(be32_to_cpu(alt_path->opa.dlid)); } if (alt_path->hop_limit <= 1) { IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg, @@ -1364,8 +1374,8 @@ static void cm_format_req(struct cm_req_msg *req_msg, } if (param->private_data && param->private_data_len) - memcpy(req_msg->private_data, param->private_data, - param->private_data_len); + IBA_SET_MEM(CM_REQ_PRIVATE_DATA, req_msg, param->private_data, + param->private_data_len); } static int cm_validate_req_param(struct ib_cm_req_param *param) @@ -1505,7 +1515,7 @@ static int cm_issue_rej(struct cm_port *port, if (ari && ari_length) { IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length); - memcpy(rej_msg->ari, ari, ari_length); + IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length); } ret = ib_post_send_mad(msg, NULL); @@ -1519,7 +1529,8 @@ static bool cm_req_has_alt_path(struct cm_req_msg *req_msg) { return ((cpu_to_be16( IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg))) || - (ib_is_opa_gid(&req_msg->alt_local_gid))); + (ib_is_opa_gid(IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID, + req_msg)))); } static void cm_path_set_rec_type(struct ib_device *ib_device, u8 port_num, @@ -1545,10 +1556,12 @@ static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg, IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg)); } else { - lid = opa_get_lid_from_gid(&req_msg->primary_local_gid); + lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR( + CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg)); sa_path_set_dlid(primary_path, lid); - lid = opa_get_lid_from_gid(&req_msg->primary_remote_gid); + lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR( + CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg)); sa_path_set_slid(primary_path, lid); } @@ -1563,10 +1576,12 @@ static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg, IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg)); } else { - lid = opa_get_lid_from_gid(&req_msg->alt_local_gid); + lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR( + CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg)); sa_path_set_dlid(alt_path, lid); - lid = opa_get_lid_from_gid(&req_msg->alt_remote_gid); + lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR( + CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg)); sa_path_set_slid(alt_path, lid); } } @@ -1575,8 +1590,10 @@ static void cm_format_paths_from_req(struct cm_req_msg *req_msg, struct sa_path_rec *primary_path, struct sa_path_rec *alt_path) { - primary_path->dgid = req_msg->primary_local_gid; - primary_path->sgid = req_msg->primary_remote_gid; + primary_path->dgid = + *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg); + primary_path->sgid = + *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg); primary_path->flow_label = cpu_to_be32(IBA_GET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg)); primary_path->hop_limit = IBA_GET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg); @@ -1600,8 +1617,10 @@ static void cm_format_paths_from_req(struct cm_req_msg *req_msg, primary_path->roce.route_resolved = false; if (cm_req_has_alt_path(req_msg)) { - alt_path->dgid = req_msg->alt_local_gid; - alt_path->sgid = req_msg->alt_remote_gid; + alt_path->dgid = *IBA_GET_MEM_PTR( + CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg); + alt_path->sgid = *IBA_GET_MEM_PTR( + CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg); alt_path->flow_label = cpu_to_be32( IBA_GET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg)); alt_path->hop_limit = @@ -1714,7 +1733,8 @@ static void cm_format_req_event(struct cm_work *work, param->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg); param->srq = IBA_GET(CM_REQ_SRQ, req_msg); param->ppath_sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr; - work->cm_event.private_data = &req_msg->private_data; + work->cm_event.private_data = + IBA_GET_MEM_PTR(CM_REQ_PRIVATE_DATA, req_msg); } static void cm_process_work(struct cm_id_private *cm_id_priv, @@ -1756,7 +1776,8 @@ static void cm_format_mra(struct cm_mra_msg *mra_msg, IBA_SET(CM_MRA_SERVICE_TIMEOUT, mra_msg, service_timeout); if (private_data && private_data_len) - memcpy(mra_msg->private_data, private_data, private_data_len); + IBA_SET_MEM(CM_MRA_PRIVATE_DATA, mra_msg, private_data, + private_data_len); } static void cm_format_rej(struct cm_rej_msg *rej_msg, @@ -1798,11 +1819,12 @@ static void cm_format_rej(struct cm_rej_msg *rej_msg, IBA_SET(CM_REJ_REASON, rej_msg, reason); if (ari && ari_length) { IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length); - memcpy(rej_msg->ari, ari, ari_length); + IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length); } if (private_data && private_data_len) - memcpy(rej_msg->private_data, private_data, private_data_len); + IBA_SET_MEM(CM_REJ_PRIVATE_DATA, rej_msg, private_data, + private_data_len); } static void cm_dup_req_handler(struct cm_work *work, @@ -2012,10 +2034,11 @@ static int cm_req_handler(struct cm_work *work) work->path[0].rec_type = sa_conv_gid_to_pathrec_type(gid_attr->gid_type); } else { - cm_path_set_rec_type(work->port->cm_dev->ib_device, - work->port->port_num, - &work->path[0], - &req_msg->primary_local_gid); + cm_path_set_rec_type( + work->port->cm_dev->ib_device, work->port->port_num, + &work->path[0], + IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, + req_msg)); } if (cm_req_has_alt_path(req_msg)) work->path[1].rec_type = work->path[0].rec_type; @@ -2117,8 +2140,8 @@ static void cm_format_rep(struct cm_rep_msg *rep_msg, } if (param->private_data && param->private_data_len) - memcpy(rep_msg->private_data, param->private_data, - param->private_data_len); + IBA_SET_MEM(CM_REP_PRIVATE_DATA, rep_msg, param->private_data, + param->private_data_len); } int ib_send_cm_rep(struct ib_cm_id *cm_id, @@ -2184,7 +2207,8 @@ static void cm_format_rtu(struct cm_rtu_msg *rtu_msg, be32_to_cpu(cm_id_priv->id.remote_id)); if (private_data && private_data_len) - memcpy(rtu_msg->private_data, private_data, private_data_len); + IBA_SET_MEM(CM_RTU_PRIVATE_DATA, rtu_msg, private_data, + private_data_len); } int ib_send_cm_rtu(struct ib_cm_id *cm_id, @@ -2259,7 +2283,8 @@ static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type) param->flow_control = IBA_GET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg); param->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg); param->srq = IBA_GET(CM_REP_SRQ, rep_msg); - work->cm_event.private_data = &rep_msg->private_data; + work->cm_event.private_data = + IBA_GET_MEM_PTR(CM_REP_PRIVATE_DATA, rep_msg); } static void cm_dup_rep_handler(struct cm_work *work) @@ -2472,7 +2497,8 @@ static int cm_rtu_handler(struct cm_work *work) if (!cm_id_priv) return -EINVAL; - work->cm_event.private_data = &rtu_msg->private_data; + work->cm_event.private_data = + IBA_GET_MEM_PTR(CM_RTU_PRIVATE_DATA, rtu_msg); spin_lock_irq(&cm_id_priv->lock); if (cm_id_priv->id.state != IB_CM_REP_SENT && @@ -2515,7 +2541,8 @@ static void cm_format_dreq(struct cm_dreq_msg *dreq_msg, be32_to_cpu(cm_id_priv->remote_qpn)); if (private_data && private_data_len) - memcpy(dreq_msg->private_data, private_data, private_data_len); + IBA_SET_MEM(CM_DREQ_PRIVATE_DATA, dreq_msg, private_data, + private_data_len); } int ib_send_cm_dreq(struct ib_cm_id *cm_id, @@ -2581,7 +2608,8 @@ static void cm_format_drep(struct cm_drep_msg *drep_msg, be32_to_cpu(cm_id_priv->id.remote_id)); if (private_data && private_data_len) - memcpy(drep_msg->private_data, private_data, private_data_len); + IBA_SET_MEM(CM_DREP_PRIVATE_DATA, drep_msg, private_data, + private_data_len); } int ib_send_cm_drep(struct ib_cm_id *cm_id, @@ -2683,7 +2711,8 @@ static int cm_dreq_handler(struct cm_work *work) return -EINVAL; } - work->cm_event.private_data = &dreq_msg->private_data; + work->cm_event.private_data = + IBA_GET_MEM_PTR(CM_DREQ_PRIVATE_DATA, dreq_msg); spin_lock_irq(&cm_id_priv->lock); if (cm_id_priv->local_qpn != @@ -2759,7 +2788,8 @@ static int cm_drep_handler(struct cm_work *work) if (!cm_id_priv) return -EINVAL; - work->cm_event.private_data = &drep_msg->private_data; + work->cm_event.private_data = + IBA_GET_MEM_PTR(CM_DREP_PRIVATE_DATA, drep_msg); spin_lock_irq(&cm_id_priv->lock); if (cm_id_priv->id.state != IB_CM_DREQ_SENT && @@ -2855,10 +2885,11 @@ static void cm_format_rej_event(struct cm_work *work) rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad; param = &work->cm_event.param.rej_rcvd; - param->ari = rej_msg->ari; + param->ari = IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg); param->ari_length = IBA_GET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg); param->reason = IBA_GET(CM_REJ_REASON, rej_msg); - work->cm_event.private_data = &rej_msg->private_data; + work->cm_event.private_data = + IBA_GET_MEM_PTR(CM_REJ_PRIVATE_DATA, rej_msg); } static struct cm_id_private * cm_acquire_rejected_id(struct cm_rej_msg *rej_msg) @@ -2871,8 +2902,9 @@ static struct cm_id_private * cm_acquire_rejected_id(struct cm_rej_msg *rej_msg) if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_TIMEOUT) { spin_lock_irq(&cm.lock); - timewait_info = cm_find_remote_id( *((__be64 *) rej_msg->ari), - remote_id); + timewait_info = cm_find_remote_id( + *((__be64 *)IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg)), + remote_id); if (!timewait_info) { spin_unlock_irq(&cm.lock); return NULL; @@ -3072,7 +3104,8 @@ static int cm_mra_handler(struct cm_work *work) if (!cm_id_priv) return -EINVAL; - work->cm_event.private_data = &mra_msg->private_data; + work->cm_event.private_data = + IBA_GET_MEM_PTR(CM_MRA_PRIVATE_DATA, mra_msg); work->cm_event.param.mra_rcvd.service_timeout = IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg); timeout = cm_convert_to_ms(IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg)) + @@ -3151,10 +3184,12 @@ static void cm_format_path_lid_from_lap(struct cm_lap_msg *lap_msg, sa_path_set_slid(path, IBA_GET(CM_LAP_ALTERNATE_REMOTE_PORT_LID, lap_msg)); } else { - lid = opa_get_lid_from_gid(&lap_msg->alt_local_gid); + lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR( + CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg)); sa_path_set_dlid(path, lid); - lid = opa_get_lid_from_gid(&lap_msg->alt_remote_gid); + lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR( + CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg)); sa_path_set_slid(path, lid); } } @@ -3163,8 +3198,9 @@ static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv, struct sa_path_rec *path, struct cm_lap_msg *lap_msg) { - path->dgid = lap_msg->alt_local_gid; - path->sgid = lap_msg->alt_remote_gid; + path->dgid = *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg); + path->sgid = + *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg); path->flow_label = cpu_to_be32(IBA_GET(CM_LAP_ALTERNATE_FLOW_LABEL, lap_msg)); path->hop_limit = IBA_GET(CM_LAP_ALTERNATE_HOP_LIMIT, lap_msg); @@ -3209,12 +3245,13 @@ static int cm_lap_handler(struct cm_work *work) param = &work->cm_event.param.lap_rcvd; memset(&work->path[0], 0, sizeof(work->path[1])); cm_path_set_rec_type(work->port->cm_dev->ib_device, - work->port->port_num, - &work->path[0], - &lap_msg->alt_local_gid); + work->port->port_num, &work->path[0], + IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID, + lap_msg)); param->alternate_path = &work->path[0]; cm_format_path_from_lap(cm_id_priv, param->alternate_path, lap_msg); - work->cm_event.private_data = &lap_msg->private_data; + work->cm_event.private_data = + IBA_GET_MEM_PTR(CM_LAP_PRIVATE_DATA, lap_msg); spin_lock_irq(&cm_id_priv->lock); if (cm_id_priv->id.state != IB_CM_ESTABLISHED) @@ -3301,10 +3338,12 @@ static int cm_apr_handler(struct cm_work *work) work->cm_event.param.apr_rcvd.ap_status = IBA_GET(CM_APR_AR_STATUS, apr_msg); - work->cm_event.param.apr_rcvd.apr_info = &apr_msg->info; + work->cm_event.param.apr_rcvd.apr_info = + IBA_GET_MEM_PTR(CM_APR_ADDITIONAL_INFORMATION, apr_msg); work->cm_event.param.apr_rcvd.info_len = IBA_GET(CM_APR_ADDITIONAL_INFORMATION_LENGTH, apr_msg); - work->cm_event.private_data = &apr_msg->private_data; + work->cm_event.private_data = + IBA_GET_MEM_PTR(CM_APR_PRIVATE_DATA, apr_msg); spin_lock_irq(&cm_id_priv->lock); if (cm_id_priv->id.state != IB_CM_ESTABLISHED || @@ -3384,8 +3423,8 @@ static void cm_format_sidr_req(struct cm_sidr_req_msg *sidr_req_msg, be64_to_cpu(param->service_id)); if (param->private_data && param->private_data_len) - memcpy(sidr_req_msg->private_data, param->private_data, - param->private_data_len); + IBA_SET_MEM(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg, + param->private_data, param->private_data_len); } int ib_send_cm_sidr_req(struct ib_cm_id *cm_id, @@ -3456,7 +3495,8 @@ static void cm_format_sidr_req_event(struct cm_work *work, param->bth_pkey = cm_get_bth_pkey(work); param->port = work->port->port_num; param->sgid_attr = rx_cm_id->av.ah_attr.grh.sgid_attr; - work->cm_event.private_data = &sidr_req_msg->private_data; + work->cm_event.private_data = + IBA_GET_MEM_PTR(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg); } static int cm_sidr_req_handler(struct cm_work *work) @@ -3540,11 +3580,12 @@ static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg, IBA_SET(CM_SIDR_REP_Q_KEY, sidr_rep_msg, param->qkey); if (param->info && param->info_length) - memcpy(sidr_rep_msg->info, param->info, param->info_length); + IBA_SET_MEM(CM_SIDR_REP_ADDITIONAL_INFORMATION, sidr_rep_msg, + param->info, param->info_length); if (param->private_data && param->private_data_len) - memcpy(sidr_rep_msg->private_data, param->private_data, - param->private_data_len); + IBA_SET_MEM(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg, + param->private_data, param->private_data_len); } int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id, @@ -3607,11 +3648,13 @@ static void cm_format_sidr_rep_event(struct cm_work *work, param->status = IBA_GET(CM_SIDR_REP_STATUS, sidr_rep_msg); param->qkey = IBA_GET(CM_SIDR_REP_Q_KEY, sidr_rep_msg); param->qpn = IBA_GET(CM_SIDR_REP_QPN, sidr_rep_msg); - param->info = &sidr_rep_msg->info; + param->info = IBA_GET_MEM_PTR(CM_SIDR_REP_ADDITIONAL_INFORMATION, + sidr_rep_msg); param->info_len = IBA_GET(CM_SIDR_REP_ADDITIONAL_INFORMATION_LENGTH, sidr_rep_msg); param->sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr; - work->cm_event.private_data = &sidr_rep_msg->private_data; + work->cm_event.private_data = + IBA_GET_MEM_PTR(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg); } static int cm_sidr_rep_handler(struct cm_work *work) -- cgit v1.2.3-59-g8ed1b From 13e0af1801f2e74639b4eadb69ed1fad2cf802f7 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Thu, 16 Jan 2020 13:00:37 -0400 Subject: RDMA/cm: Remove CM message structs All accesses now use the new IBA acessor scheme, so delete the structs entirely and generate the structures from the schema file. Link: https://lore.kernel.org/r/20200116170037.30109-8-jgg@ziepe.ca Tested-by: Leon Romanovsky Reviewed-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/cm.c | 77 -------------- drivers/infiniband/core/cm_msgs.h | 211 -------------------------------------- include/rdma/ibta_vol1_c12.h | 7 +- 3 files changed, 6 insertions(+), 289 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c index 5ccd59f1ebb8..68cc1b2d6824 100644 --- a/drivers/infiniband/core/cm.c +++ b/drivers/infiniband/core/cm.c @@ -4426,83 +4426,6 @@ static void cm_remove_one(struct ib_device *ib_device, void *client_data) kfree(cm_dev); } -/* - * Check at compile time that the byte offset and length of field old_name in - * the struct matches the byte offset and length in the new macro. - */ -#define _IBA_CHECK_OFF(old_name, field_struct, field_offset, mask, bits) \ - static_assert(offsetof(field_struct, old_name) == (field_offset)); \ - static_assert(bits == sizeof(((field_struct *)0)->old_name) * 8) -#define IBA_CHECK_OFF(field, old_name) _IBA_CHECK_OFF(old_name, field) - -IBA_CHECK_OFF(CM_REQ_LOCAL_COMM_ID, local_comm_id); -IBA_CHECK_OFF(CM_REQ_SERVICE_ID, service_id); -IBA_CHECK_OFF(CM_REQ_LOCAL_CA_GUID, local_ca_guid); -IBA_CHECK_OFF(CM_REQ_LOCAL_Q_KEY, local_qkey); -IBA_CHECK_OFF(CM_REQ_PARTITION_KEY, pkey); -IBA_CHECK_OFF(CM_REQ_PRIMARY_LOCAL_PORT_LID, primary_local_lid); -IBA_CHECK_OFF(CM_REQ_PRIMARY_REMOTE_PORT_LID, primary_remote_lid); -IBA_CHECK_OFF(CM_REQ_PRIMARY_LOCAL_PORT_GID, primary_local_gid); -IBA_CHECK_OFF(CM_REQ_PRIMARY_REMOTE_PORT_GID, primary_remote_gid); -IBA_CHECK_OFF(CM_REQ_PRIMARY_TRAFFIC_CLASS, primary_traffic_class); -IBA_CHECK_OFF(CM_REQ_PRIMARY_HOP_LIMIT, primary_hop_limit); -IBA_CHECK_OFF(CM_REQ_ALTERNATE_LOCAL_PORT_LID, alt_local_lid); -IBA_CHECK_OFF(CM_REQ_ALTERNATE_REMOTE_PORT_LID, alt_remote_lid); -IBA_CHECK_OFF(CM_REQ_ALTERNATE_LOCAL_PORT_GID, alt_local_gid); -IBA_CHECK_OFF(CM_REQ_ALTERNATE_REMOTE_PORT_GID, alt_remote_gid); -IBA_CHECK_OFF(CM_REQ_ALTERNATE_TRAFFIC_CLASS, alt_traffic_class); -IBA_CHECK_OFF(CM_REQ_ALTERNATE_HOP_LIMIT, alt_hop_limit); -IBA_CHECK_OFF(CM_REQ_PRIVATE_DATA, private_data); -IBA_CHECK_OFF(CM_MRA_LOCAL_COMM_ID, local_comm_id); -IBA_CHECK_OFF(CM_MRA_REMOTE_COMM_ID, remote_comm_id); -IBA_CHECK_OFF(CM_MRA_PRIVATE_DATA, private_data); -IBA_CHECK_OFF(CM_REJ_LOCAL_COMM_ID, local_comm_id); -IBA_CHECK_OFF(CM_REJ_REMOTE_COMM_ID, remote_comm_id); -IBA_CHECK_OFF(CM_REJ_REASON, reason); -IBA_CHECK_OFF(CM_REJ_ARI, ari); -IBA_CHECK_OFF(CM_REJ_PRIVATE_DATA, private_data); -IBA_CHECK_OFF(CM_REP_LOCAL_COMM_ID, local_comm_id); -IBA_CHECK_OFF(CM_REP_REMOTE_COMM_ID, remote_comm_id); -IBA_CHECK_OFF(CM_REP_LOCAL_Q_KEY, local_qkey); -IBA_CHECK_OFF(CM_REP_RESPONDER_RESOURCES, resp_resources); -IBA_CHECK_OFF(CM_REP_INITIATOR_DEPTH, initiator_depth); -IBA_CHECK_OFF(CM_REP_LOCAL_CA_GUID, local_ca_guid); -IBA_CHECK_OFF(CM_REP_PRIVATE_DATA, private_data); -IBA_CHECK_OFF(CM_RTU_LOCAL_COMM_ID, local_comm_id); -IBA_CHECK_OFF(CM_RTU_REMOTE_COMM_ID, remote_comm_id); -IBA_CHECK_OFF(CM_RTU_PRIVATE_DATA, private_data); -IBA_CHECK_OFF(CM_DREQ_LOCAL_COMM_ID, local_comm_id); -IBA_CHECK_OFF(CM_DREQ_REMOTE_COMM_ID, remote_comm_id); -IBA_CHECK_OFF(CM_DREQ_PRIVATE_DATA, private_data); -IBA_CHECK_OFF(CM_DREP_LOCAL_COMM_ID, local_comm_id); -IBA_CHECK_OFF(CM_DREP_REMOTE_COMM_ID, remote_comm_id); -IBA_CHECK_OFF(CM_DREP_PRIVATE_DATA, private_data); -IBA_CHECK_OFF(CM_LAP_LOCAL_COMM_ID, local_comm_id); -IBA_CHECK_OFF(CM_LAP_REMOTE_COMM_ID, remote_comm_id); -IBA_CHECK_OFF(CM_LAP_ALTERNATE_LOCAL_PORT_LID, alt_local_lid); -IBA_CHECK_OFF(CM_LAP_ALTERNATE_REMOTE_PORT_LID, alt_remote_lid); -IBA_CHECK_OFF(CM_LAP_ALTERNATE_LOCAL_PORT_GID, alt_local_gid); -IBA_CHECK_OFF(CM_LAP_ALTERNATE_REMOTE_PORT_GID, alt_remote_gid); -IBA_CHECK_OFF(CM_LAP_ALTERNATE_HOP_LIMIT, alt_hop_limit); -IBA_CHECK_OFF(CM_LAP_PRIVATE_DATA, private_data); -IBA_CHECK_OFF(CM_APR_LOCAL_COMM_ID, local_comm_id); -IBA_CHECK_OFF(CM_APR_REMOTE_COMM_ID, remote_comm_id); -IBA_CHECK_OFF(CM_APR_ADDITIONAL_INFORMATION_LENGTH, info_length); -IBA_CHECK_OFF(CM_APR_AR_STATUS, ap_status); -IBA_CHECK_OFF(CM_APR_ADDITIONAL_INFORMATION, info); -IBA_CHECK_OFF(CM_APR_PRIVATE_DATA, private_data); -IBA_CHECK_OFF(CM_SIDR_REQ_REQUESTID, request_id); -IBA_CHECK_OFF(CM_SIDR_REQ_PARTITION_KEY, pkey); -IBA_CHECK_OFF(CM_SIDR_REQ_SERVICEID, service_id); -IBA_CHECK_OFF(CM_SIDR_REQ_PRIVATE_DATA, private_data); -IBA_CHECK_OFF(CM_SIDR_REP_REQUESTID, request_id); -IBA_CHECK_OFF(CM_SIDR_REP_STATUS, status); -IBA_CHECK_OFF(CM_SIDR_REP_ADDITIONAL_INFORMATION_LENGTH, info_length); -IBA_CHECK_OFF(CM_SIDR_REP_SERVICEID, service_id); -IBA_CHECK_OFF(CM_SIDR_REP_Q_KEY, qkey); -IBA_CHECK_OFF(CM_SIDR_REP_ADDITIONAL_INFORMATION, info); -IBA_CHECK_OFF(CM_SIDR_REP_PRIVATE_DATA, private_data); - static int __init ib_cm_init(void) { int ret; diff --git a/drivers/infiniband/core/cm_msgs.h b/drivers/infiniband/core/cm_msgs.h index 86ab6952d5d8..0cc40656b5c5 100644 --- a/drivers/infiniband/core/cm_msgs.h +++ b/drivers/infiniband/core/cm_msgs.h @@ -19,62 +19,6 @@ #define IB_CM_CLASS_VERSION 2 /* IB specification 1.2 */ -struct cm_req_msg { - struct ib_mad_hdr hdr; - - __be32 local_comm_id; - __be32 rsvd4; - __be64 service_id; - __be64 local_ca_guid; - __be32 rsvd24; - __be32 local_qkey; - /* local QPN:24, responder resources:8 */ - __be32 offset32; - /* local EECN:24, initiator depth:8 */ - __be32 offset36; - /* - * remote EECN:24, remote CM response timeout:5, - * transport service type:2, end-to-end flow control:1 - */ - __be32 offset40; - /* starting PSN:24, local CM response timeout:5, retry count:3 */ - __be32 offset44; - __be16 pkey; - /* path MTU:4, RDC exists:1, RNR retry count:3. */ - u8 offset50; - /* max CM Retries:4, SRQ:1, extended transport type:3 */ - u8 offset51; - - __be16 primary_local_lid; - __be16 primary_remote_lid; - union ib_gid primary_local_gid; - union ib_gid primary_remote_gid; - /* flow label:20, rsvd:6, packet rate:6 */ - __be32 primary_offset88; - u8 primary_traffic_class; - u8 primary_hop_limit; - /* SL:4, subnet local:1, rsvd:3 */ - u8 primary_offset94; - /* local ACK timeout:5, rsvd:3 */ - u8 primary_offset95; - - __be16 alt_local_lid; - __be16 alt_remote_lid; - union ib_gid alt_local_gid; - union ib_gid alt_remote_gid; - /* flow label:20, rsvd:6, packet rate:6 */ - __be32 alt_offset132; - u8 alt_traffic_class; - u8 alt_hop_limit; - /* SL:4, subnet local:1, rsvd:3 */ - u8 alt_offset138; - /* local ACK timeout:5, rsvd:3 */ - u8 alt_offset139; - - u32 private_data[IB_CM_REQ_PRIVATE_DATA_SIZE / sizeof(u32)]; - -} __packed; - static inline enum ib_qp_type cm_req_get_qp_type(struct cm_req_msg *req_msg) { u8 transport_type = IBA_GET(CM_REQ_TRANSPORT_SERVICE_TYPE, req_msg); @@ -113,60 +57,6 @@ enum cm_msg_response { CM_MSG_RESPONSE_OTHER = 0x2 }; - struct cm_mra_msg { - struct ib_mad_hdr hdr; - - __be32 local_comm_id; - __be32 remote_comm_id; - /* message MRAed:2, rsvd:6 */ - u8 offset8; - /* service timeout:5, rsvd:3 */ - u8 offset9; - - u8 private_data[IB_CM_MRA_PRIVATE_DATA_SIZE]; - -} __packed; - -struct cm_rej_msg { - struct ib_mad_hdr hdr; - - __be32 local_comm_id; - __be32 remote_comm_id; - /* message REJected:2, rsvd:6 */ - u8 offset8; - /* reject info length:7, rsvd:1. */ - u8 offset9; - __be16 reason; - u8 ari[IB_CM_REJ_ARI_LENGTH]; - - u8 private_data[IB_CM_REJ_PRIVATE_DATA_SIZE]; - -} __packed; - -struct cm_rep_msg { - struct ib_mad_hdr hdr; - - __be32 local_comm_id; - __be32 remote_comm_id; - __be32 local_qkey; - /* local QPN:24, rsvd:8 */ - __be32 offset12; - /* local EECN:24, rsvd:8 */ - __be32 offset16; - /* starting PSN:24 rsvd:8 */ - __be32 offset20; - u8 resp_resources; - u8 initiator_depth; - /* target ACK delay:5, failover accepted:2, end-to-end flow control:1 */ - u8 offset26; - /* RNR retry count:3, SRQ:1, rsvd:5 */ - u8 offset27; - __be64 local_ca_guid; - - u8 private_data[IB_CM_REP_PRIVATE_DATA_SIZE]; - -} __packed; - static inline __be32 cm_rep_get_qpn(struct cm_rep_msg *rep_msg, enum ib_qp_type qp_type) { return (qp_type == IB_QPT_XRC_INI) ? @@ -175,105 +65,4 @@ static inline __be32 cm_rep_get_qpn(struct cm_rep_msg *rep_msg, enum ib_qp_type cpu_to_be32(IBA_GET(CM_REP_LOCAL_QPN, rep_msg)); } -struct cm_rtu_msg { - struct ib_mad_hdr hdr; - - __be32 local_comm_id; - __be32 remote_comm_id; - - u8 private_data[IB_CM_RTU_PRIVATE_DATA_SIZE]; - -} __packed; - -struct cm_dreq_msg { - struct ib_mad_hdr hdr; - - __be32 local_comm_id; - __be32 remote_comm_id; - /* remote QPN/EECN:24, rsvd:8 */ - __be32 offset8; - - u8 private_data[IB_CM_DREQ_PRIVATE_DATA_SIZE]; - -} __packed; - -struct cm_drep_msg { - struct ib_mad_hdr hdr; - - __be32 local_comm_id; - __be32 remote_comm_id; - - u8 private_data[IB_CM_DREP_PRIVATE_DATA_SIZE]; - -} __packed; - -struct cm_lap_msg { - struct ib_mad_hdr hdr; - - __be32 local_comm_id; - __be32 remote_comm_id; - - __be32 rsvd8; - /* remote QPN/EECN:24, remote CM response timeout:5, rsvd:3 */ - __be32 offset12; - __be32 rsvd16; - - __be16 alt_local_lid; - __be16 alt_remote_lid; - union ib_gid alt_local_gid; - union ib_gid alt_remote_gid; - /* flow label:20, rsvd:4, traffic class:8 */ - __be32 offset56; - u8 alt_hop_limit; - /* rsvd:2, packet rate:6 */ - u8 offset61; - /* SL:4, subnet local:1, rsvd:3 */ - u8 offset62; - /* local ACK timeout:5, rsvd:3 */ - u8 offset63; - - u8 private_data[IB_CM_LAP_PRIVATE_DATA_SIZE]; -} __packed; - -struct cm_apr_msg { - struct ib_mad_hdr hdr; - - __be32 local_comm_id; - __be32 remote_comm_id; - - u8 info_length; - u8 ap_status; - __be16 rsvd; - u8 info[IB_CM_APR_INFO_LENGTH]; - - u8 private_data[IB_CM_APR_PRIVATE_DATA_SIZE]; -} __packed; - -struct cm_sidr_req_msg { - struct ib_mad_hdr hdr; - - __be32 request_id; - __be16 pkey; - __be16 rsvd; - __be64 service_id; - - u32 private_data[IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE / sizeof(u32)]; -} __packed; - -struct cm_sidr_rep_msg { - struct ib_mad_hdr hdr; - - __be32 request_id; - u8 status; - u8 info_length; - __be16 rsvd; - /* QPN:24, rsvd:8 */ - __be32 offset8; - __be64 service_id; - __be32 qkey; - u8 info[IB_CM_SIDR_REP_INFO_LENGTH]; - - u8 private_data[IB_CM_SIDR_REP_PRIVATE_DATA_SIZE]; -} __packed; - #endif /* CM_MSGS_H */ diff --git a/include/rdma/ibta_vol1_c12.h b/include/rdma/ibta_vol1_c12.h index 916db5c27dc5..269904425d3f 100644 --- a/include/rdma/ibta_vol1_c12.h +++ b/include/rdma/ibta_vol1_c12.h @@ -29,7 +29,12 @@ IBA_FIELD_MLOC(field_struct, \ (byte_offset + sizeof(struct ib_mad_hdr)), width, type) #define CM_STRUCT(field_struct, total_len) \ - static_assert((total_len) % 32 == 0); + field_struct \ + { \ + struct ib_mad_hdr hdr; \ + u32 _data[(total_len) / 32 + \ + BUILD_BUG_ON_ZERO((total_len) % 32 != 0)]; \ + } /* Table 106 REQ Message Contents */ #define CM_REQ_LOCAL_COMM_ID CM_FIELD32_LOC(struct cm_req_msg, 0, 32) -- cgit v1.2.3-59-g8ed1b From 36798d5ae1af62e830c5e045b2e41ce038690c61 Mon Sep 17 00:00:00 2001 From: Artemy Kovalyov Date: Tue, 28 Jan 2020 15:56:12 +0200 Subject: RDMA/umem: Fix ib_umem_find_best_pgsz() Except for the last entry, the ending iova alignment sets the maximum possible page size as the low bits of the iova must be zero when starting the next chunk. Fixes: 4a35339958f1 ("RDMA/umem: Add API to find best driver supported page size in an MR") Link: https://lore.kernel.org/r/20200128135612.174820-1-leon@kernel.org Signed-off-by: Artemy Kovalyov Signed-off-by: Leon Romanovsky Tested-by: Gal Pressman Reviewed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/umem.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index 146f98fbf22b..ac4738d2e0dd 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -166,10 +166,13 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, * for any address. */ mask |= (sg_dma_address(sg) + pgoff) ^ va; - if (i && i != (umem->nmap - 1)) - /* restrict by length as well for interior SGEs */ - mask |= sg_dma_len(sg); va += sg_dma_len(sg) - pgoff; + /* Except for the last entry, the ending iova alignment sets + * the maximum possible page size as the low bits of the iova + * must be zero when starting the next chunk. + */ + if (i != (umem->nmap - 1)) + mask |= va; pgoff = 0; } best_pg_bit = rdma_find_pg_bit(mask, pgsz_bitmap); -- cgit v1.2.3-59-g8ed1b From b4fb4cc5ba83b20dae13cef116c33648e81d2f44 Mon Sep 17 00:00:00 2001 From: Parav Pandit Date: Sun, 26 Jan 2020 16:26:46 +0200 Subject: RDMA/cma: Fix unbalanced cm_id reference count during address resolve Below commit missed the AF_IB and loopback code flow in rdma_resolve_addr(). This leads to an unbalanced cm_id refcount in cma_work_handler() which puts the refcount which was not incremented prior to queuing the work. A call trace is observed with such code flow: BUG: unable to handle kernel NULL pointer dereference at (null) [] __mutex_lock_slowpath+0x166/0x1d0 [] mutex_lock+0x1f/0x2f [] cma_work_handler+0x25/0xa0 [] process_one_work+0x17f/0x440 [] worker_thread+0x126/0x3c0 Hence, hold the cm_id reference when scheduling the resolve work item. Fixes: 722c7b2bfead ("RDMA/{cma, core}: Avoid callback on rdma_addr_cancel()") Link: https://lore.kernel.org/r/20200126142652.104803-2-leon@kernel.org Signed-off-by: Parav Pandit Signed-off-by: Leon Romanovsky Reviewed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/cma.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index 55a9afacfedd..72f032160c4b 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -3148,6 +3148,7 @@ static int cma_resolve_loopback(struct rdma_id_private *id_priv) rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid); rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid); + atomic_inc(&id_priv->refcount); cma_init_resolve_addr_work(work, id_priv); queue_work(cma_wq, &work->work); return 0; @@ -3174,6 +3175,7 @@ static int cma_resolve_ib_addr(struct rdma_id_private *id_priv) rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, (union ib_gid *) &(((struct sockaddr_ib *) &id_priv->id.route.addr.dst_addr)->sib_addr)); + atomic_inc(&id_priv->refcount); cma_init_resolve_addr_work(work, id_priv); queue_work(cma_wq, &work->work); return 0; -- cgit v1.2.3-59-g8ed1b From 8889f6fa35884d09f24734e10fea0c9ddcbc6429 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Thu, 30 Jan 2020 11:21:21 -0400 Subject: RDMA/core: Make the entire API tree static Compilation of mlx5 driver without CONFIG_INFINIBAND_USER_ACCESS generates the following error. on x86_64: ld: drivers/infiniband/hw/mlx5/main.o: in function `mlx5_ib_handler_MLX5_IB_METHOD_VAR_OBJ_ALLOC': main.c:(.text+0x186d): undefined reference to `ib_uverbs_get_ucontext_file' ld: drivers/infiniband/hw/mlx5/main.o:(.rodata+0x2480): undefined reference to `uverbs_idr_class' ld: drivers/infiniband/hw/mlx5/main.o:(.rodata+0x24d8): undefined reference to `uverbs_destroy_def_handler' This is happening because some parts of the UAPI description are not static. This is a hold over from earlier code that relied on struct pointers to refer to object types, now object types are referenced by number. Remove the unused globals and add statics to the remaining UAPI description elements. Remove the redundent #ifdefs around mlx5_ib_*defs and obsolete mlx5_ib_get_devx_tree(). The compiler now trims alot more unused code, including the above problematic definitions when !CONFIG_INFINIBAND_USER_ACCESS. Fixes: 7be76bef320b ("IB/mlx5: Introduce VAR object and its alloc/destroy methods") Reported-by: Randy Dunlap Acked-by: Randy Dunlap Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/uverbs.h | 17 ----------------- drivers/infiniband/hw/mlx5/main.c | 2 -- drivers/infiniband/hw/mlx5/mlx5_ib.h | 7 +++---- include/rdma/uverbs_named_ioctl.h | 6 +++--- 4 files changed, 6 insertions(+), 26 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h index 4d4cec46d251..7df71983212d 100644 --- a/drivers/infiniband/core/uverbs.h +++ b/drivers/infiniband/core/uverbs.h @@ -271,23 +271,6 @@ int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type, size_t kern_filter_sz, union ib_flow_spec *ib_spec); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_DEVICE); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_PD); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_MR); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_COMP_CHANNEL); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_CQ); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_QP); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_AH); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_MW); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_SRQ); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_FLOW); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_WQ); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_RWQ_IND_TBL); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_XRCD); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_FLOW_ACTION); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_DM); -extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_COUNTERS); - /* * ib_uverbs_query_port_resp.port_cap_flags started out as just a copy of the * PortInfo CapabilityMask, but was extended with unique bits. diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 01fc09f3ddd3..0ca958143280 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -6247,10 +6247,8 @@ ADD_UVERBS_ATTRIBUTES_SIMPLE( enum mlx5_ib_uapi_flow_action_flags)); static const struct uapi_definition mlx5_ib_defs[] = { -#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) UAPI_DEF_CHAIN(mlx5_ib_devx_defs), UAPI_DEF_CHAIN(mlx5_ib_flow_defs), -#endif UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION, &mlx5_ib_flow_action), diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 7b019bd4de4b..d9bffcc93587 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -1379,14 +1379,14 @@ int mlx5_ib_fill_res_entry(struct sk_buff *msg, int mlx5_ib_fill_stat_entry(struct sk_buff *msg, struct rdma_restrack_entry *res); +extern const struct uapi_definition mlx5_ib_devx_defs[]; +extern const struct uapi_definition mlx5_ib_flow_defs[]; + #if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user); void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid); void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev); void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev); -const struct uverbs_object_tree_def *mlx5_ib_get_devx_tree(void); -extern const struct uapi_definition mlx5_ib_devx_defs[]; -extern const struct uapi_definition mlx5_ib_flow_defs[]; struct mlx5_ib_flow_handler *mlx5_ib_raw_fs_rule_add( struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher, struct mlx5_flow_context *flow_context, @@ -1394,7 +1394,6 @@ struct mlx5_ib_flow_handler *mlx5_ib_raw_fs_rule_add( void *cmd_in, int inlen, int dest_id, int dest_type); bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type); bool mlx5_ib_devx_is_flow_counter(void *obj, u32 offset, u32 *counter_id); -int mlx5_ib_get_flow_trees(const struct uverbs_object_tree_def **root); void mlx5_ib_destroy_flow_action_raw(struct mlx5_ib_flow_action *maction); #else static inline int diff --git a/include/rdma/uverbs_named_ioctl.h b/include/rdma/uverbs_named_ioctl.h index 3447bfe356d6..6ae6cf8e4c2e 100644 --- a/include/rdma/uverbs_named_ioctl.h +++ b/include/rdma/uverbs_named_ioctl.h @@ -76,7 +76,7 @@ #define DECLARE_UVERBS_NAMED_OBJECT(_object_id, _type_attrs, ...) \ static const struct uverbs_method_def *const UVERBS_OBJECT_METHODS( \ _object_id)[] = { __VA_ARGS__ }; \ - const struct uverbs_object_def UVERBS_OBJECT(_object_id) = { \ + static const struct uverbs_object_def UVERBS_OBJECT(_object_id) = { \ .id = _object_id, \ .type_attrs = &_type_attrs, \ .num_methods = ARRAY_SIZE(UVERBS_OBJECT_METHODS(_object_id)), \ @@ -88,10 +88,10 @@ * identify all uapi methods with a (object,method) tuple. However, they have * no type pointer. */ -#define DECLARE_UVERBS_GLOBAL_METHODS(_object_id, ...) \ +#define DECLARE_UVERBS_GLOBAL_METHODS(_object_id, ...) \ static const struct uverbs_method_def *const UVERBS_OBJECT_METHODS( \ _object_id)[] = { __VA_ARGS__ }; \ - const struct uverbs_object_def UVERBS_OBJECT(_object_id) = { \ + static const struct uverbs_object_def UVERBS_OBJECT(_object_id) = { \ .id = _object_id, \ .num_methods = ARRAY_SIZE(UVERBS_OBJECT_METHODS(_object_id)), \ .methods = &UVERBS_OBJECT_METHODS(_object_id) \ -- cgit v1.2.3-59-g8ed1b