diff options
Diffstat (limited to 'drivers/infiniband/sw/rxe')
| -rw-r--r-- | drivers/infiniband/sw/rxe/rxe.c | 67 | ||||
| -rw-r--r-- | drivers/infiniband/sw/rxe/rxe.h | 16 | ||||
| -rw-r--r-- | drivers/infiniband/sw/rxe/rxe_av.c | 7 | ||||
| -rw-r--r-- | drivers/infiniband/sw/rxe/rxe_comp.c | 6 | ||||
| -rw-r--r-- | drivers/infiniband/sw/rxe/rxe_loc.h | 9 | ||||
| -rw-r--r-- | drivers/infiniband/sw/rxe/rxe_mr.c | 15 | ||||
| -rw-r--r-- | drivers/infiniband/sw/rxe/rxe_net.c | 97 | ||||
| -rw-r--r-- | drivers/infiniband/sw/rxe/rxe_net.h | 2 | ||||
| -rw-r--r-- | drivers/infiniband/sw/rxe/rxe_param.h | 3 | ||||
| -rw-r--r-- | drivers/infiniband/sw/rxe/rxe_pool.c | 77 | ||||
| -rw-r--r-- | drivers/infiniband/sw/rxe/rxe_pool.h | 4 | ||||
| -rw-r--r-- | drivers/infiniband/sw/rxe/rxe_qp.c | 15 | ||||
| -rw-r--r-- | drivers/infiniband/sw/rxe/rxe_recv.c | 12 | ||||
| -rw-r--r-- | drivers/infiniband/sw/rxe/rxe_resp.c | 3 | ||||
| -rw-r--r-- | drivers/infiniband/sw/rxe/rxe_sysfs.c | 40 | ||||
| -rw-r--r-- | drivers/infiniband/sw/rxe/rxe_verbs.c | 103 | ||||
| -rw-r--r-- | drivers/infiniband/sw/rxe/rxe_verbs.h | 9 |
17 files changed, 232 insertions, 253 deletions
diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c index 383e65c7bbc0..a8c11b5e1e94 100644 --- a/drivers/infiniband/sw/rxe/rxe.c +++ b/drivers/infiniband/sw/rxe/rxe.c @@ -31,6 +31,7 @@ * SOFTWARE. */ +#include <rdma/rdma_netlink.h> #include <net/addrconf.h> #include "rxe.h" #include "rxe_loc.h" @@ -50,8 +51,10 @@ static void rxe_cleanup_ports(struct rxe_dev *rxe) /* free resources for a rxe device all objects created for this device must * have been destroyed */ -static void rxe_cleanup(struct rxe_dev *rxe) +void rxe_dealloc(struct ib_device *ib_dev) { + struct rxe_dev *rxe = container_of(ib_dev, struct rxe_dev, ib_dev); + rxe_pool_cleanup(&rxe->uc_pool); rxe_pool_cleanup(&rxe->pd_pool); rxe_pool_cleanup(&rxe->ah_pool); @@ -65,16 +68,8 @@ static void rxe_cleanup(struct rxe_dev *rxe) rxe_cleanup_ports(rxe); - crypto_free_shash(rxe->tfm); -} - -/* called when all references have been dropped */ -void rxe_release(struct kref *kref) -{ - struct rxe_dev *rxe = container_of(kref, struct rxe_dev, ref_cnt); - - rxe_cleanup(rxe); - ib_dealloc_device(&rxe->ib_dev); + if (rxe->tfm) + crypto_free_shash(rxe->tfm); } /* initialize rxe device parameters */ @@ -279,7 +274,6 @@ static int rxe_init(struct rxe_dev *rxe) spin_lock_init(&rxe->mmap_offset_lock); spin_lock_init(&rxe->pending_lock); INIT_LIST_HEAD(&rxe->pending_mmaps); - INIT_LIST_HEAD(&rxe->list); mutex_init(&rxe->usdev_lock); @@ -308,37 +302,46 @@ void rxe_set_mtu(struct rxe_dev *rxe, unsigned int ndev_mtu) /* called by ifc layer to create new rxe device. * The caller should allocate memory for rxe by calling ib_alloc_device. */ -int rxe_add(struct rxe_dev *rxe, unsigned int mtu) +int rxe_add(struct rxe_dev *rxe, unsigned int mtu, const char *ibdev_name) { int err; - kref_init(&rxe->ref_cnt); - err = rxe_init(rxe); if (err) - goto err1; + return err; rxe_set_mtu(rxe, mtu); - err = rxe_register_device(rxe); - if (err) - goto err1; - - return 0; - -err1: - rxe_dev_put(rxe); - return err; + return rxe_register_device(rxe, ibdev_name); } -/* called by the ifc layer to remove a device */ -void rxe_remove(struct rxe_dev *rxe) +static int rxe_newlink(const char *ibdev_name, struct net_device *ndev) { - rxe_unregister_device(rxe); + struct rxe_dev *exists; + int err = 0; + + exists = rxe_get_dev_from_net(ndev); + if (exists) { + ib_device_put(&exists->ib_dev); + pr_err("already configured on %s\n", ndev->name); + err = -EEXIST; + goto err; + } - rxe_dev_put(rxe); + err = rxe_net_add(ibdev_name, ndev); + if (err) { + pr_err("failed to add %s\n", ndev->name); + goto err; + } +err: + return err; } +static struct rdma_link_ops rxe_link_ops = { + .type = "rxe", + .newlink = rxe_newlink, +}; + static int __init rxe_module_init(void) { int err; @@ -354,13 +357,15 @@ static int __init rxe_module_init(void) if (err) return err; + rdma_link_register(&rxe_link_ops); pr_info("loaded\n"); return 0; } static void __exit rxe_module_exit(void) { - rxe_remove_all(); + rdma_link_unregister(&rxe_link_ops); + ib_unregister_driver(RDMA_DRIVER_RXE); rxe_net_exit(); rxe_cache_exit(); @@ -369,3 +374,5 @@ static void __exit rxe_module_exit(void) late_initcall(rxe_module_init); module_exit(rxe_module_exit); + +MODULE_ALIAS_RDMA_LINK("rxe"); diff --git a/drivers/infiniband/sw/rxe/rxe.h b/drivers/infiniband/sw/rxe/rxe.h index 5bde2ad964d2..2e2dff478833 100644 --- a/drivers/infiniband/sw/rxe/rxe.h +++ b/drivers/infiniband/sw/rxe/rxe.h @@ -95,18 +95,20 @@ static inline u32 rxe_crc32(struct rxe_dev *rxe, void rxe_set_mtu(struct rxe_dev *rxe, unsigned int dev_mtu); -int rxe_add(struct rxe_dev *rxe, unsigned int mtu); -void rxe_remove(struct rxe_dev *rxe); -void rxe_remove_all(void); +int rxe_add(struct rxe_dev *rxe, unsigned int mtu, const char *ibdev_name); void rxe_rcv(struct sk_buff *skb); -static inline void rxe_dev_put(struct rxe_dev *rxe) +/* The caller must do a matching ib_device_put(&dev->ib_dev) */ +static inline struct rxe_dev *rxe_get_dev_from_net(struct net_device *ndev) { - kref_put(&rxe->ref_cnt, rxe_release); + struct ib_device *ibdev = + ib_device_get_by_netdev(ndev, RDMA_DRIVER_RXE); + + if (!ibdev) + return NULL; + return container_of(ibdev, struct rxe_dev, ib_dev); } -struct rxe_dev *net_to_rxe(struct net_device *ndev); -struct rxe_dev *get_rxe_by_name(const char *name); void rxe_port_up(struct rxe_dev *rxe); void rxe_port_down(struct rxe_dev *rxe); diff --git a/drivers/infiniband/sw/rxe/rxe_av.c b/drivers/infiniband/sw/rxe/rxe_av.c index 26fe8d7dbc55..81ee756c19b8 100644 --- a/drivers/infiniband/sw/rxe/rxe_av.c +++ b/drivers/infiniband/sw/rxe/rxe_av.c @@ -34,6 +34,13 @@ #include "rxe.h" #include "rxe_loc.h" +void rxe_init_av(struct rdma_ah_attr *attr, struct rxe_av *av) +{ + rxe_av_from_attr(rdma_ah_get_port_num(attr), av, attr); + rxe_av_fill_ip_info(av, attr); + memcpy(av->dmac, attr->roce.dmac, ETH_ALEN); +} + int rxe_av_chk_attr(struct rxe_dev *rxe, struct rdma_ah_attr *attr) { struct rxe_port *port; diff --git a/drivers/infiniband/sw/rxe/rxe_comp.c b/drivers/infiniband/sw/rxe/rxe_comp.c index e996da67a851..00eb99d3df86 100644 --- a/drivers/infiniband/sw/rxe/rxe_comp.c +++ b/drivers/infiniband/sw/rxe/rxe_comp.c @@ -146,8 +146,7 @@ void retransmit_timer(struct timer_list *t) } } -void rxe_comp_queue_pkt(struct rxe_dev *rxe, struct rxe_qp *qp, - struct sk_buff *skb) +void rxe_comp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb) { int must_sched; @@ -155,7 +154,8 @@ void rxe_comp_queue_pkt(struct rxe_dev *rxe, struct rxe_qp *qp, must_sched = skb_queue_len(&qp->resp_pkts) > 1; if (must_sched != 0) - rxe_counter_inc(rxe, RXE_CNT_COMPLETER_SCHED); + rxe_counter_inc(SKB_TO_PKT(skb)->rxe, RXE_CNT_COMPLETER_SCHED); + rxe_run_task(&qp->comp.task, must_sched); } diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h index 01b74597b36a..3d8cef836f0d 100644 --- a/drivers/infiniband/sw/rxe/rxe_loc.h +++ b/drivers/infiniband/sw/rxe/rxe_loc.h @@ -35,6 +35,7 @@ #define RXE_LOC_H /* rxe_av.c */ +void rxe_init_av(struct rdma_ah_attr *attr, struct rxe_av *av); int rxe_av_chk_attr(struct rxe_dev *rxe, struct rdma_ah_attr *attr); @@ -231,7 +232,7 @@ int rxe_srq_from_attr(struct rxe_dev *rxe, struct rxe_srq *srq, struct ib_srq_attr *attr, enum ib_srq_attr_mask mask, struct rxe_modify_srq_cmd *ucmd); -void rxe_release(struct kref *kref); +void rxe_dealloc(struct ib_device *ib_dev); int rxe_completer(void *arg); int rxe_requester(void *arg); @@ -239,11 +240,9 @@ int rxe_responder(void *arg); u32 rxe_icrc_hdr(struct rxe_pkt_info *pkt, struct sk_buff *skb); -void rxe_resp_queue_pkt(struct rxe_dev *rxe, - struct rxe_qp *qp, struct sk_buff *skb); +void rxe_resp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb); -void rxe_comp_queue_pkt(struct rxe_dev *rxe, - struct rxe_qp *qp, struct sk_buff *skb); +void rxe_comp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb); static inline unsigned int wr_opcode_mask(int opcode, struct rxe_qp *qp) { diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c index 9d3916b93f23..42f0f25e396c 100644 --- a/drivers/infiniband/sw/rxe/rxe_mr.c +++ b/drivers/infiniband/sw/rxe/rxe_mr.c @@ -162,16 +162,15 @@ int rxe_mem_init_user(struct rxe_pd *pd, u64 start, u64 length, u64 iova, int access, struct ib_udata *udata, struct rxe_mem *mem) { - int entry; struct rxe_map **map; struct rxe_phys_buf *buf = NULL; struct ib_umem *umem; - struct scatterlist *sg; + struct sg_page_iter sg_iter; int num_buf; void *vaddr; int err; - umem = ib_umem_get(pd->ibpd.uobject->context, start, length, access, 0); + umem = ib_umem_get(udata, start, length, access, 0); if (IS_ERR(umem)) { pr_warn("err %d from rxe_umem_get\n", (int)PTR_ERR(umem)); @@ -191,16 +190,16 @@ int rxe_mem_init_user(struct rxe_pd *pd, u64 start, goto err1; } - mem->page_shift = umem->page_shift; - mem->page_mask = BIT(umem->page_shift) - 1; + mem->page_shift = PAGE_SHIFT; + mem->page_mask = PAGE_SIZE - 1; num_buf = 0; map = mem->map; if (length > 0) { buf = map[0]->buf; - for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { - vaddr = page_address(sg_page(sg)); + for_each_sg_page(umem->sg_head.sgl, &sg_iter, umem->nmap, 0) { + vaddr = page_address(sg_page_iter_page(&sg_iter)); if (!vaddr) { pr_warn("null vaddr\n"); err = -ENOMEM; @@ -208,7 +207,7 @@ int rxe_mem_init_user(struct rxe_pd *pd, u64 start, } buf->addr = (uintptr_t)vaddr; - buf->size = BIT(umem->page_shift); + buf->size = PAGE_SIZE; num_buf++; buf++; diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c index 8fd03ae20efc..753cabcd441c 100644 --- a/drivers/infiniband/sw/rxe/rxe_net.c +++ b/drivers/infiniband/sw/rxe/rxe_net.c @@ -45,43 +45,6 @@ #include "rxe_net.h" #include "rxe_loc.h" -static LIST_HEAD(rxe_dev_list); -static DEFINE_SPINLOCK(dev_list_lock); /* spinlock for device list */ - -struct rxe_dev *net_to_rxe(struct net_device *ndev) -{ - struct rxe_dev *rxe; - struct rxe_dev *found = NULL; - - spin_lock_bh(&dev_list_lock); - list_for_each_entry(rxe, &rxe_dev_list, list) { - if (rxe->ndev == ndev) { - found = rxe; - break; - } - } - spin_unlock_bh(&dev_list_lock); - - return found; -} - -struct rxe_dev *get_rxe_by_name(const char *name) -{ - struct rxe_dev *rxe; - struct rxe_dev *found = NULL; - - spin_lock_bh(&dev_list_lock); - list_for_each_entry(rxe, &rxe_dev_list, list) { - if (!strcmp(name, dev_name(&rxe->ib_dev.dev))) { - found = rxe; - break; - } - } - spin_unlock_bh(&dev_list_lock); - return found; -} - - static struct rxe_recv_sockets recv_sockets; struct device *rxe_dma_device(struct rxe_dev *rxe) @@ -229,18 +192,19 @@ static int rxe_udp_encap_recv(struct sock *sk, struct sk_buff *skb) struct udphdr *udph; struct net_device *ndev = skb->dev; struct net_device *rdev = ndev; - struct rxe_dev *rxe = net_to_rxe(ndev); + struct rxe_dev *rxe = rxe_get_dev_from_net(ndev); struct rxe_pkt_info *pkt = SKB_TO_PKT(skb); if (!rxe && is_vlan_dev(rdev)) { rdev = vlan_dev_real_dev(ndev); - rxe = net_to_rxe(rdev); + rxe = rxe_get_dev_from_net(rdev); } if (!rxe) goto drop; if (skb_linearize(skb)) { pr_err("skb_linearize failed\n"); + ib_device_put(&rxe->ib_dev); goto drop; } @@ -253,6 +217,12 @@ static int rxe_udp_encap_recv(struct sock *sk, struct sk_buff *skb) rxe_rcv(skb); + /* + * FIXME: this is in the wrong place, it needs to be done when pkt is + * destroyed + */ + ib_device_put(&rxe->ib_dev); + return 0; drop: kfree_skb(skb); @@ -384,9 +354,6 @@ static int prepare4(struct rxe_pkt_info *pkt, struct sk_buff *skb, return -EHOSTUNREACH; } - if (!memcmp(saddr, daddr, sizeof(*daddr))) - pkt->mask |= RXE_LOOPBACK_MASK; - prepare_udp_hdr(skb, cpu_to_be16(qp->src_port), cpu_to_be16(ROCE_V2_UDP_DPORT)); @@ -411,9 +378,6 @@ static int prepare6(struct rxe_pkt_info *pkt, struct sk_buff *skb, return -EHOSTUNREACH; } - if (!memcmp(saddr, daddr, sizeof(*daddr))) - pkt->mask |= RXE_LOOPBACK_MASK; - prepare_udp_hdr(skb, cpu_to_be16(qp->src_port), cpu_to_be16(ROCE_V2_UDP_DPORT)); @@ -437,6 +401,9 @@ int rxe_prepare(struct rxe_pkt_info *pkt, struct sk_buff *skb, u32 *crc) *crc = rxe_icrc_hdr(pkt, skb); + if (ether_addr_equal(skb->dev->dev_addr, av->dmac)) + pkt->mask |= RXE_LOOPBACK_MASK; + return err; } @@ -550,42 +517,24 @@ enum rdma_link_layer rxe_link_layer(struct rxe_dev *rxe, unsigned int port_num) return IB_LINK_LAYER_ETHERNET; } -struct rxe_dev *rxe_net_add(struct net_device *ndev) +int rxe_net_add(const char *ibdev_name, struct net_device *ndev) { int err; struct rxe_dev *rxe = NULL; - rxe = (struct rxe_dev *)ib_alloc_device(sizeof(*rxe)); + rxe = ib_alloc_device(rxe_dev, ib_dev); if (!rxe) - return NULL; + return -ENOMEM; rxe->ndev = ndev; - err = rxe_add(rxe, ndev->mtu); + err = rxe_add(rxe, ndev->mtu, ibdev_name); if (err) { ib_dealloc_device(&rxe->ib_dev); - return NULL; + return err; } - spin_lock_bh(&dev_list_lock); - list_add_tail(&rxe->list, &rxe_dev_list); - spin_unlock_bh(&dev_list_lock); - return rxe; -} - -void rxe_remove_all(void) -{ - spin_lock_bh(&dev_list_lock); - while (!list_empty(&rxe_dev_list)) { - struct rxe_dev *rxe = - list_first_entry(&rxe_dev_list, struct rxe_dev, list); - - list_del(&rxe->list); - spin_unlock_bh(&dev_list_lock); - rxe_remove(rxe); - spin_lock_bh(&dev_list_lock); - } - spin_unlock_bh(&dev_list_lock); + return 0; } static void rxe_port_event(struct rxe_dev *rxe, @@ -638,15 +587,14 @@ static int rxe_notify(struct notifier_block *not_blk, void *arg) { struct net_device *ndev = netdev_notifier_info_to_dev(arg); - struct rxe_dev *rxe = net_to_rxe(ndev); + struct rxe_dev *rxe = rxe_get_dev_from_net(ndev); if (!rxe) - goto out; + return NOTIFY_OK; switch (event) { case NETDEV_UNREGISTER: - list_del(&rxe->list); - rxe_remove(rxe); + ib_unregister_device_queued(&rxe->ib_dev); break; case NETDEV_UP: rxe_port_up(rxe); @@ -671,7 +619,8 @@ static int rxe_notify(struct notifier_block *not_blk, event, ndev->name); break; } -out: + + ib_device_put(&rxe->ib_dev); return NOTIFY_OK; } diff --git a/drivers/infiniband/sw/rxe/rxe_net.h b/drivers/infiniband/sw/rxe/rxe_net.h index 106c586dbb26..2ca71d3d245c 100644 --- a/drivers/infiniband/sw/rxe/rxe_net.h +++ b/drivers/infiniband/sw/rxe/rxe_net.h @@ -43,7 +43,7 @@ struct rxe_recv_sockets { struct socket *sk6; }; -struct rxe_dev *rxe_net_add(struct net_device *ndev); +int rxe_net_add(const char *ibdev_name, struct net_device *ndev); int rxe_net_init(void); void rxe_net_exit(void); diff --git a/drivers/infiniband/sw/rxe/rxe_param.h b/drivers/infiniband/sw/rxe/rxe_param.h index bdea899a58ac..1abed47ca221 100644 --- a/drivers/infiniband/sw/rxe/rxe_param.h +++ b/drivers/infiniband/sw/rxe/rxe_param.h @@ -78,7 +78,8 @@ enum rxe_device_param { | IB_DEVICE_SYS_IMAGE_GUID | IB_DEVICE_RC_RNR_NAK_GEN | IB_DEVICE_SRQ_RESIZE - | IB_DEVICE_MEM_MGT_EXTENSIONS, + | IB_DEVICE_MEM_MGT_EXTENSIONS + | IB_DEVICE_ALLOW_USER_UNREG, RXE_MAX_SGE = 32, RXE_MAX_SGE_RD = 32, RXE_MAX_CQ = 16384, diff --git a/drivers/infiniband/sw/rxe/rxe_pool.c b/drivers/infiniband/sw/rxe/rxe_pool.c index b5c91df22047..120fa9005954 100644 --- a/drivers/infiniband/sw/rxe/rxe_pool.c +++ b/drivers/infiniband/sw/rxe/rxe_pool.c @@ -42,10 +42,12 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = { [RXE_TYPE_UC] = { .name = "rxe-uc", .size = sizeof(struct rxe_ucontext), + .flags = RXE_POOL_NO_ALLOC, }, [RXE_TYPE_PD] = { .name = "rxe-pd", .size = sizeof(struct rxe_pd), + .flags = RXE_POOL_NO_ALLOC, }, [RXE_TYPE_AH] = { .name = "rxe-ah", @@ -119,8 +121,10 @@ static void rxe_cache_clean(size_t cnt) for (i = 0; i < cnt; i++) { type = &rxe_type_info[i]; - kmem_cache_destroy(type->cache); - type->cache = NULL; + if (!(type->flags & RXE_POOL_NO_ALLOC)) { + kmem_cache_destroy(type->cache); + type->cache = NULL; + } } } @@ -134,14 +138,17 @@ int rxe_cache_init(void) for (i = 0; i < RXE_NUM_TYPES; i++) { type = &rxe_type_info[i]; size = ALIGN(type->size, RXE_POOL_ALIGN); - type->cache = kmem_cache_create(type->name, size, - RXE_POOL_ALIGN, - RXE_POOL_CACHE_FLAGS, NULL); - if (!type->cache) { - pr_err("Unable to init kmem cache for %s\n", - type->name); - err = -ENOMEM; - goto err1; + if (!(type->flags & RXE_POOL_NO_ALLOC)) { + type->cache = + kmem_cache_create(type->name, size, + RXE_POOL_ALIGN, + RXE_POOL_CACHE_FLAGS, NULL); + if (!type->cache) { + pr_err("Unable to init kmem cache for %s\n", + type->name); + err = -ENOMEM; + goto err1; + } } } @@ -392,29 +399,64 @@ void *rxe_alloc(struct rxe_pool *pool) kref_get(&pool->ref_cnt); read_unlock_irqrestore(&pool->pool_lock, flags); - kref_get(&pool->rxe->ref_cnt); + if (!ib_device_try_get(&pool->rxe->ib_dev)) + goto out_put_pool; if (atomic_inc_return(&pool->num_elem) > pool->max_elem) - goto out_put_pool; + goto out_cnt; elem = kmem_cache_zalloc(pool_cache(pool), (pool->flags & RXE_POOL_ATOMIC) ? GFP_ATOMIC : GFP_KERNEL); if (!elem) - goto out_put_pool; + goto out_cnt; elem->pool = pool; kref_init(&elem->ref_cnt); return elem; -out_put_pool: +out_cnt: atomic_dec(&pool->num_elem); - rxe_dev_put(pool->rxe); + ib_device_put(&pool->rxe->ib_dev); +out_put_pool: rxe_pool_put(pool); return NULL; } +int rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_entry *elem) +{ + unsigned long flags; + + might_sleep_if(!(pool->flags & RXE_POOL_ATOMIC)); + + read_lock_irqsave(&pool->pool_lock, flags); + if (pool->state != RXE_POOL_STATE_VALID) { + read_unlock_irqrestore(&pool->pool_lock, flags); + return -EINVAL; + } + kref_get(&pool->ref_cnt); + read_unlock_irqrestore(&pool->pool_lock, flags); + + if (!ib_device_try_get(&pool->rxe->ib_dev)) + goto out_put_pool; + + if (atomic_inc_return(&pool->num_elem) > pool->max_elem) + goto out_cnt; + + elem->pool = pool; + kref_init(&elem->ref_cnt); + + return 0; + +out_cnt: + atomic_dec(&pool->num_elem); + ib_device_put(&pool->rxe->ib_dev); +out_put_pool: + rxe_pool_put(pool); + return -EINVAL; +} + void rxe_elem_release(struct kref *kref) { struct rxe_pool_entry *elem = @@ -424,9 +466,10 @@ void rxe_elem_release(struct kref *kref) if (pool->cleanup) pool->cleanup(elem); - kmem_cache_free(pool_cache(pool), elem); + if (!(pool->flags & RXE_POOL_NO_ALLOC)) + kmem_cache_free(pool_cache(pool), elem); atomic_dec(&pool->num_elem); - rxe_dev_put(pool->rxe); + ib_device_put(&pool->rxe->ib_dev); rxe_pool_put(pool); } diff --git a/drivers/infiniband/sw/rxe/rxe_pool.h b/drivers/infiniband/sw/rxe/rxe_pool.h index 72968c29e01f..2f2cff1cbe43 100644 --- a/drivers/infiniband/sw/rxe/rxe_pool.h +++ b/drivers/infiniband/sw/rxe/rxe_pool.h @@ -41,6 +41,7 @@ enum rxe_pool_flags { RXE_POOL_ATOMIC = BIT(0), RXE_POOL_INDEX = BIT(1), RXE_POOL_KEY = BIT(2), + RXE_POOL_NO_ALLOC = BIT(4), }; enum rxe_elem_type { @@ -131,6 +132,9 @@ void rxe_pool_cleanup(struct rxe_pool *pool); /* allocate an object from pool */ void *rxe_alloc(struct rxe_pool *pool); +/* connect already allocated object to pool */ +int rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_entry *elem); + /* assign an index to an indexed object and insert object into * pool's rb tree */ diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c index fd86fd2fbb26..09ede70dc1e8 100644 --- a/drivers/infiniband/sw/rxe/rxe_qp.c +++ b/drivers/infiniband/sw/rxe/rxe_qp.c @@ -35,6 +35,7 @@ #include <linux/delay.h> #include <linux/sched.h> #include <linux/vmalloc.h> +#include <rdma/uverbs_ioctl.h> #include "rxe.h" #include "rxe_loc.h" @@ -343,7 +344,8 @@ int rxe_qp_from_init(struct rxe_dev *rxe, struct rxe_qp *qp, struct rxe_pd *pd, struct rxe_cq *rcq = to_rcq(init->recv_cq); struct rxe_cq *scq = to_rcq(init->send_cq); struct rxe_srq *srq = init->srq ? to_rsrq(init->srq) : NULL; - struct ib_ucontext *context = udata ? ibpd->uobject->context : NULL; + struct rxe_ucontext *ucontext = + rdma_udata_to_drv_context(udata, struct rxe_ucontext, ibuc); rxe_add_ref(pd); rxe_add_ref(rcq); @@ -358,11 +360,11 @@ int rxe_qp_from_init(struct rxe_dev *rxe, struct rxe_qp *qp, struct rxe_pd *pd, rxe_qp_init_misc(rxe, qp, init); - err = rxe_qp_init_req(rxe, qp, init, context, uresp); + err = rxe_qp_init_req(rxe, qp, init, &ucontext->ibuc, uresp); if (err) goto err1; - err = rxe_qp_init_resp(rxe, qp, init, context, uresp); + err = rxe_qp_init_resp(rxe, qp, init, &ucontext->ibuc, uresp); if (err) goto err2; @@ -631,14 +633,11 @@ int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask, qp->attr.qkey = attr->qkey; if (mask & IB_QP_AV) { - rxe_av_from_attr(attr->port_num, &qp->pri_av, &attr->ah_attr); - rxe_av_fill_ip_info(&qp->pri_av, &attr->ah_attr); + rxe_init_av(&attr->ah_attr, &qp->pri_av); } if (mask & IB_QP_ALT_PATH) { - rxe_av_from_attr(attr->alt_port_num, &qp->alt_av, - &attr->alt_ah_attr); - rxe_av_fill_ip_info(&qp->alt_av, &attr->alt_ah_attr); + rxe_init_av(&attr->alt_ah_attr, &qp->alt_av); qp->attr.alt_port_num = attr->alt_port_num; qp->attr.alt_pkey_index = attr->alt_pkey_index; qp->attr.alt_timeout = attr->alt_timeout; diff --git a/drivers/infiniband/sw/rxe/rxe_recv.c b/drivers/infiniband/sw/rxe/rxe_recv.c index 5c29a1bb575a..f9a492ed900b 100644 --- a/drivers/infiniband/sw/rxe/rxe_recv.c +++ b/drivers/infiniband/sw/rxe/rxe_recv.c @@ -266,14 +266,12 @@ err1: return -EINVAL; } -static inline void rxe_rcv_pkt(struct rxe_dev *rxe, - struct rxe_pkt_info *pkt, - struct sk_buff *skb) +static inline void rxe_rcv_pkt(struct rxe_pkt_info *pkt, struct sk_buff *skb) { if (pkt->mask & RXE_REQ_MASK) - rxe_resp_queue_pkt(rxe, pkt->qp, skb); + rxe_resp_queue_pkt(pkt->qp, skb); else - rxe_comp_queue_pkt(rxe, pkt->qp, skb); + rxe_comp_queue_pkt(pkt->qp, skb); } static void rxe_rcv_mcast_pkt(struct rxe_dev *rxe, struct sk_buff *skb) @@ -319,7 +317,7 @@ static void rxe_rcv_mcast_pkt(struct rxe_dev *rxe, struct sk_buff *skb) pkt->qp = qp; rxe_add_ref(qp); - rxe_rcv_pkt(rxe, pkt, skb); + rxe_rcv_pkt(pkt, skb); } spin_unlock_bh(&mcg->mcg_lock); @@ -411,7 +409,7 @@ void rxe_rcv(struct sk_buff *skb) if (unlikely(bth_qpn(pkt) == IB_MULTICAST_QPN)) rxe_rcv_mcast_pkt(rxe, skb); else - rxe_rcv_pkt(rxe, pkt, skb); + rxe_rcv_pkt(pkt, skb); return; diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c index 231528188250..aca9f60f9b21 100644 --- a/drivers/infiniband/sw/rxe/rxe_resp.c +++ b/drivers/infiniband/sw/rxe/rxe_resp.c @@ -104,8 +104,7 @@ static char *resp_state_name[] = { }; /* rxe_recv calls here to add a request packet to the input queue */ -void rxe_resp_queue_pkt(struct rxe_dev *rxe, struct rxe_qp *qp, - struct sk_buff *skb) +void rxe_resp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb) { int must_sched; struct rxe_pkt_info *pkt = SKB_TO_PKT(skb); diff --git a/drivers/infiniband/sw/rxe/rxe_sysfs.c b/drivers/infiniband/sw/rxe/rxe_sysfs.c index 95a15892f7e6..ccda5f5a3bc0 100644 --- a/drivers/infiniband/sw/rxe/rxe_sysfs.c +++ b/drivers/infiniband/sw/rxe/rxe_sysfs.c @@ -58,41 +58,37 @@ static int rxe_param_set_add(const char *val, const struct kernel_param *kp) int len; int err = 0; char intf[32]; - struct net_device *ndev = NULL; - struct rxe_dev *rxe; + struct net_device *ndev; + struct rxe_dev *exists; len = sanitize_arg(val, intf, sizeof(intf)); if (!len) { pr_err("add: invalid interface name\n"); - err = -EINVAL; - goto err; + return -EINVAL; } ndev = dev_get_by_name(&init_net, intf); if (!ndev) { pr_err("interface %s not found\n", intf); - err = -EINVAL; - goto err; + return -EINVAL; } - if (net_to_rxe(ndev)) { + exists = rxe_get_dev_from_net(ndev); + if (exists) { + ib_device_put(&exists->ib_dev); pr_err("already configured on %s\n", intf); err = -EINVAL; goto err; } - rxe = rxe_net_add(ndev); - if (!rxe) { + err = rxe_net_add("rxe%d", ndev); + if (err) { pr_err("failed to add %s\n", intf); - err = -EINVAL; goto err; } - rxe_set_port_state(rxe); - dev_info(&rxe->ib_dev.dev, "added %s\n", intf); err: - if (ndev) - dev_put(ndev); + dev_put(ndev); return err; } @@ -100,7 +96,7 @@ static int rxe_param_set_remove(const char *val, const struct kernel_param *kp) { int len; char intf[32]; - struct rxe_dev *rxe; + struct ib_device *ib_dev; len = sanitize_arg(val, intf, sizeof(intf)); if (!len) { @@ -110,19 +106,17 @@ static int rxe_param_set_remove(const char *val, const struct kernel_param *kp) if (strncmp("all", intf, len) == 0) { pr_info("rxe_sys: remove all"); - rxe_remove_all(); + ib_unregister_driver(RDMA_DRIVER_RXE); return 0; } - rxe = get_rxe_by_name(intf); - - if (!rxe) { + ib_dev = ib_device_get_by_name(intf, RDMA_DRIVER_RXE); + if (!ib_dev) { pr_err("not configured on %s\n", intf); return -EINVAL; } - list_del(&rxe->list); - rxe_remove(rxe); + ib_unregister_device_and_put(ib_dev); return 0; } @@ -136,6 +130,6 @@ static const struct kernel_param_ops rxe_remove_ops = { }; module_param_cb(add, &rxe_add_ops, NULL, 0200); -MODULE_PARM_DESC(add, "Create RXE device over network interface"); +MODULE_PARM_DESC(add, "DEPRECATED. Create RXE device over network interface"); module_param_cb(remove, &rxe_remove_ops, NULL, 0200); -MODULE_PARM_DESC(remove, "Remove RXE device over network interface"); +MODULE_PARM_DESC(remove, "DEPRECATED. Remove RXE device over network interface"); diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c index b20e6e0415f5..6ecf28570ff0 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.c +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c @@ -33,6 +33,7 @@ #include <linux/dma-mapping.h> #include <net/addrconf.h> +#include <rdma/uverbs_ioctl.h> #include "rxe.h" #include "rxe_loc.h" #include "rxe_queue.h" @@ -79,19 +80,6 @@ static int rxe_query_port(struct ib_device *dev, return rc; } -static struct net_device *rxe_get_netdev(struct ib_device *device, - u8 port_num) -{ - struct rxe_dev *rxe = to_rdev(device); - - if (rxe->ndev) { - dev_hold(rxe->ndev); - return rxe->ndev; - } - - return NULL; -} - static int rxe_query_pkey(struct ib_device *device, u8 port_num, u16 index, u16 *pkey) { @@ -154,22 +142,19 @@ static enum rdma_link_layer rxe_get_link_layer(struct ib_device *dev, return rxe_link_layer(rxe, port_num); } -static struct ib_ucontext *rxe_alloc_ucontext(struct ib_device *dev, - struct ib_udata *udata) +static int rxe_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) { - struct rxe_dev *rxe = to_rdev(dev); - struct rxe_ucontext *uc; + struct rxe_dev *rxe = to_rdev(uctx->device); + struct rxe_ucontext *uc = to_ruc(uctx); - uc = rxe_alloc(&rxe->uc_pool); - return uc ? &uc->ibuc : ERR_PTR(-ENOMEM); + return rxe_add_to_pool(&rxe->uc_pool, &uc->pelem); } -static int rxe_dealloc_ucontext(struct ib_ucontext *ibuc) +static void rxe_dealloc_ucontext(struct ib_ucontext *ibuc) { struct rxe_ucontext *uc = to_ruc(ibuc); rxe_drop_ref(uc); - return 0; } static int rxe_port_immutable(struct ib_device *dev, u8 port_num, @@ -191,30 +176,20 @@ static int rxe_port_immutable(struct ib_device *dev, u8 port_num, return 0; } -static struct ib_pd *rxe_alloc_pd(struct ib_device *dev, - struct ib_ucontext *context, - struct ib_udata *udata) +static int rxe_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context, + struct ib_udata *udata) { - struct rxe_dev *rxe = to_rdev(dev); - struct rxe_pd *pd; + struct rxe_dev *rxe = to_rdev(ibpd->device); + struct rxe_pd *pd = to_rpd(ibpd); - pd = rxe_alloc(&rxe->pd_pool); - return pd ? &pd->ibpd : ERR_PTR(-ENOMEM); + return rxe_add_to_pool(&rxe->pd_pool, &pd->pelem); } -static int rxe_dealloc_pd(struct ib_pd *ibpd) +static void rxe_dealloc_pd(struct ib_pd *ibpd) { struct rxe_pd *pd = to_rpd(ibpd); rxe_drop_ref(pd); - return 0; -} - -static void rxe_init_av(struct rxe_dev *rxe, struct rdma_ah_attr *attr, - struct rxe_av *av) -{ - rxe_av_from_attr(rdma_ah_get_port_num(attr), av, attr); - rxe_av_fill_ip_info(av, attr); } static struct ib_ah *rxe_create_ah(struct ib_pd *ibpd, @@ -239,7 +214,7 @@ static struct ib_ah *rxe_create_ah(struct ib_pd *ibpd, rxe_add_ref(pd); ah->pd = pd; - rxe_init_av(rxe, attr, &ah->av); + rxe_init_av(attr, &ah->av); return &ah->ibah; } @@ -253,7 +228,7 @@ static int rxe_modify_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr) if (err) return err; - rxe_init_av(rxe, attr, &ah->av); + rxe_init_av(attr, &ah->av); return 0; } @@ -330,8 +305,9 @@ static struct ib_srq *rxe_create_srq(struct ib_pd *ibpd, int err; struct rxe_dev *rxe = to_rdev(ibpd->device); struct rxe_pd *pd = to_rpd(ibpd); + struct rxe_ucontext *ucontext = + rdma_udata_to_drv_context(udata, struct rxe_ucontext, ibuc); struct rxe_srq *srq; - struct ib_ucontext *context = udata ? ibpd->uobject->context : NULL; struct rxe_create_srq_resp __user *uresp = NULL; if (udata) { @@ -354,7 +330,7 @@ static struct ib_srq *rxe_create_srq(struct ib_pd *ibpd, rxe_add_ref(pd); srq->pd = pd; - err = rxe_srq_from_init(rxe, srq, init, context, uresp); + err = rxe_srq_from_init(rxe, srq, init, &ucontext->ibuc, uresp); if (err) goto err2; @@ -1129,8 +1105,8 @@ static int rxe_detach_mcast(struct ib_qp *ibqp, union ib_gid *mgid, u16 mlid) static ssize_t parent_show(struct device *device, struct device_attribute *attr, char *buf) { - struct rxe_dev *rxe = container_of(device, struct rxe_dev, - ib_dev.dev); + struct rxe_dev *rxe = + rdma_device_to_drv_device(device, struct rxe_dev, ib_dev); return snprintf(buf, 16, "%s\n", rxe_parent_name(rxe, 1)); } @@ -1146,6 +1122,15 @@ static const struct attribute_group rxe_attr_group = { .attrs = rxe_dev_attributes, }; +static int rxe_enable_driver(struct ib_device *ib_dev) +{ + struct rxe_dev *rxe = container_of(ib_dev, struct rxe_dev, ib_dev); + + rxe_set_port_state(rxe); + dev_info(&rxe->ib_dev.dev, "added %s\n", netdev_name(rxe->ndev)); + return 0; +} + static const struct ib_device_ops rxe_dev_ops = { .alloc_hw_stats = rxe_ib_alloc_hw_stats, .alloc_mr = rxe_alloc_mr, @@ -1156,6 +1141,7 @@ static const struct ib_device_ops rxe_dev_ops = { .create_cq = rxe_create_cq, .create_qp = rxe_create_qp, .create_srq = rxe_create_srq, + .dealloc_driver = rxe_dealloc, .dealloc_pd = rxe_dealloc_pd, .dealloc_ucontext = rxe_dealloc_ucontext, .dereg_mr = rxe_dereg_mr, @@ -1164,10 +1150,10 @@ static const struct ib_device_ops rxe_dev_ops = { .destroy_qp = rxe_destroy_qp, .destroy_srq = rxe_destroy_srq, .detach_mcast = rxe_detach_mcast, + .enable_driver = rxe_enable_driver, .get_dma_mr = rxe_get_dma_mr, .get_hw_stats = rxe_ib_get_hw_stats, .get_link_layer = rxe_get_link_layer, - .get_netdev = rxe_get_netdev, .get_port_immutable = rxe_port_immutable, .map_mr_sg = rxe_map_mr_sg, .mmap = rxe_mmap, @@ -1190,9 +1176,11 @@ static const struct ib_device_ops rxe_dev_ops = { .reg_user_mr = rxe_reg_user_mr, .req_notify_cq = rxe_req_notify_cq, .resize_cq = rxe_resize_cq, + INIT_RDMA_OBJ_SIZE(ib_pd, rxe_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_ucontext, rxe_ucontext, ibuc), }; -int rxe_register_device(struct rxe_dev *rxe) +int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name) { int err; struct ib_device *dev = &rxe->ib_dev; @@ -1247,6 +1235,9 @@ int rxe_register_device(struct rxe_dev *rxe) ; ib_set_device_ops(dev, &rxe_dev_ops); + err = ib_device_set_netdev(&rxe->ib_dev, rxe->ndev, 1); + if (err) + return err; tfm = crypto_alloc_shash("crc32", 0, 0); if (IS_ERR(tfm)) { @@ -1258,23 +1249,13 @@ int rxe_register_device(struct rxe_dev *rxe) rdma_set_device_sysfs_group(dev, &rxe_attr_group); dev->driver_id = RDMA_DRIVER_RXE; - err = ib_register_device(dev, "rxe%d", NULL); - if (err) { + err = ib_register_device(dev, ibdev_name); + if (err) pr_warn("%s failed with error %d\n", __func__, err); - goto err1; - } - - return 0; - -err1: - crypto_free_shash(rxe->tfm); + /* + * Note that rxe may be invalid at this point if another thread + * unregistered it. + */ return err; } - -void rxe_unregister_device(struct rxe_dev *rxe) -{ - struct ib_device *dev = &rxe->ib_dev; - - ib_unregister_device(dev); -} diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h index 74e04801d34d..157e51aeb1e1 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.h +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h @@ -61,13 +61,13 @@ static inline int psn_compare(u32 psn_a, u32 psn_b) } struct rxe_ucontext { + struct ib_ucontext ibuc; struct rxe_pool_entry pelem; - struct ib_ucontext ibuc; }; struct rxe_pd { + struct ib_pd ibpd; struct rxe_pool_entry pelem; - struct ib_pd ibpd; }; struct rxe_ah { @@ -385,7 +385,6 @@ struct rxe_dev { struct ib_device_attr attr; int max_ucontext; int max_inline_data; - struct kref ref_cnt; struct mutex usdev_lock; struct net_device *ndev; @@ -412,7 +411,6 @@ struct rxe_dev { atomic64_t stats_counters[RXE_NUM_OF_COUNTERS]; struct rxe_port port; - struct list_head list; struct crypto_shash *tfm; }; @@ -466,8 +464,7 @@ static inline struct rxe_mem *to_rmw(struct ib_mw *mw) return mw ? container_of(mw, struct rxe_mem, ibmw) : NULL; } -int rxe_register_device(struct rxe_dev *rxe); -void rxe_unregister_device(struct rxe_dev *rxe); +int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name); void rxe_mc_cleanup(struct rxe_pool_entry *arg); |