From 0bc0602ae8275aadc7288692331589350987cef7 Mon Sep 17 00:00:00 2001 From: Anand Khoje Date: Mon, 12 Jul 2021 17:56:23 +0530 Subject: IB/core: Updating cache for subnet_prefix in config_non_roce_gid_cache() Currently, cache for subnet_prefix was getting updated by reading port attributes via ib_query_port. ib_query_port() calls ops.query_gid() to get subnet_prefix and returns it via port_attr. In ib_cache_update(), config_non_roce_gid_cache() obtains GIDs by calling ops.query_gid(). We utilize this to store subnet_prefix in cache. Link: https://lore.kernel.org/r/20210712122625.1147-2-anand.a.khoje@oracle.com Suggested-by: Jason Gunthorpe Suggested-by: Aru Kolappan Signed-off-by: Anand Khoje Signed-off-by: Haakon Bugge Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/cache.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c index c9e9fc81447e..929399e103d1 100644 --- a/drivers/infiniband/core/cache.c +++ b/drivers/infiniband/core/cache.c @@ -1429,7 +1429,7 @@ int rdma_read_gid_l2_fields(const struct ib_gid_attr *attr, EXPORT_SYMBOL(rdma_read_gid_l2_fields); static int config_non_roce_gid_cache(struct ib_device *device, - u32 port, int gid_tbl_len) + u32 port, struct ib_port_attr *tprops) { struct ib_gid_attr gid_attr = {}; struct ib_gid_table *table; @@ -1441,7 +1441,7 @@ static int config_non_roce_gid_cache(struct ib_device *device, table = rdma_gid_table(device, port); mutex_lock(&table->lock); - for (i = 0; i < gid_tbl_len; ++i) { + for (i = 0; i < tprops->gid_tbl_len; ++i) { if (!device->ops.query_gid) continue; ret = device->ops.query_gid(device, port, i, &gid_attr.gid); @@ -1452,6 +1452,8 @@ static int config_non_roce_gid_cache(struct ib_device *device, goto err; } gid_attr.index = i; + tprops->subnet_prefix = + be64_to_cpu(gid_attr.gid.global.subnet_prefix); add_modify_gid(table, &gid_attr); } err: @@ -1484,7 +1486,7 @@ ib_cache_update(struct ib_device *device, u32 port, bool update_gids, if (!rdma_protocol_roce(device, port) && update_gids) { ret = config_non_roce_gid_cache(device, port, - tprops->gid_tbl_len); + tprops); if (ret) goto err; } -- cgit v1.2.3-59-g8ed1b From 36721a6d4cf2a39f411e3ac04d5771d1ffef9d23 Mon Sep 17 00:00:00 2001 From: Anand Khoje Date: Mon, 12 Jul 2021 17:56:24 +0530 Subject: IB/core: Shifting initialization of device->cache_lock The lock cache_lock of struct ib_device is initialized in function ib_cache_setup_one(). This is much later than the device initialization in _ib_alloc_device(). This change shifts initialization of cache_lock in _ib_alloc_device(). Link: https://lore.kernel.org/r/20210712122625.1147-3-anand.a.khoje@oracle.com Suggested-by: Haakon Bugge Signed-off-by: Anand Khoje Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/cache.c | 2 -- drivers/infiniband/core/device.c | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c index 929399e103d1..0c98dd3dee67 100644 --- a/drivers/infiniband/core/cache.c +++ b/drivers/infiniband/core/cache.c @@ -1621,8 +1621,6 @@ int ib_cache_setup_one(struct ib_device *device) u32 p; int err; - rwlock_init(&device->cache_lock); - err = gid_table_setup_one(device); if (err) return err; diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index fa20b1824fb8..ba0ad7241772 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -607,6 +607,8 @@ struct ib_device *_ib_alloc_device(size_t size) for (i = 0; i < ARRAY_SIZE(device->cq_pools); i++) INIT_LIST_HEAD(&device->cq_pools[i]); + rwlock_init(&device->cache_lock); + device->uverbs_cmd_mask = BIT_ULL(IB_USER_VERBS_CMD_ALLOC_MW) | BIT_ULL(IB_USER_VERBS_CMD_ALLOC_PD) | -- cgit v1.2.3-59-g8ed1b From 21bfee9c0c7754408b1f311bffe304caf3e62250 Mon Sep 17 00:00:00 2001 From: Anand Khoje Date: Mon, 12 Jul 2021 17:56:25 +0530 Subject: IB/core: Read subnet_prefix in ib_query_port via cache. ib_query_port() calls device->ops.query_port() to get the port attributes. The method of querying is device driver specific. The same function calls device->ops.query_gid() to get the GID and extract the subnet_prefix (gid_prefix). The GID and subnet_prefix are stored in a cache. But they do not get read from the cache if the device is an Infiniband device. The following change takes advantage of the cached subnet_prefix. Testing with RDBMS has shown a significant improvement in performance with this change. Link: https://lore.kernel.org/r/20210712122625.1147-4-anand.a.khoje@oracle.com Signed-off-by: Anand Khoje Signed-off-by: Haakon Bugge Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/device.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index ba0ad7241772..9056f48bdca6 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -2052,7 +2052,6 @@ static int __ib_query_port(struct ib_device *device, u32 port_num, struct ib_port_attr *port_attr) { - union ib_gid gid = {}; int err; memset(port_attr, 0, sizeof(*port_attr)); @@ -2065,11 +2064,8 @@ static int __ib_query_port(struct ib_device *device, IB_LINK_LAYER_INFINIBAND) return 0; - err = device->ops.query_gid(device, port_num, 0, &gid); - if (err) - return err; - - port_attr->subnet_prefix = be64_to_cpu(gid.global.subnet_prefix); + ib_get_cached_subnet_prefix(device, port_num, + &port_attr->subnet_prefix); return 0; } -- cgit v1.2.3-59-g8ed1b From e677b72a0647249370f2635862bf0241c86f66ad Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 23 Jul 2021 17:08:55 +0300 Subject: RDMA/iwcm: Release resources if iw_cm module initialization fails The failure during iw_cm module initialization partially left the system with unreleased memory and other resources. Rewrite the module init/exit routines in such way that netlink commands will be opened only after successful initialization. Fixes: b493d91d333e ("iwcm: common code for port mapper") Link: https://lore.kernel.org/r/b01239f99cb1a3e6d2b0694c242d89e6410bcd93.1627048781.git.leonro@nvidia.com Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/iwcm.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/iwcm.c b/drivers/infiniband/core/iwcm.c index 42261152b489..2b47073c61a6 100644 --- a/drivers/infiniband/core/iwcm.c +++ b/drivers/infiniband/core/iwcm.c @@ -1186,29 +1186,34 @@ static int __init iw_cm_init(void) ret = iwpm_init(RDMA_NL_IWCM); if (ret) - pr_err("iw_cm: couldn't init iwpm\n"); - else - rdma_nl_register(RDMA_NL_IWCM, iwcm_nl_cb_table); + return ret; + iwcm_wq = alloc_ordered_workqueue("iw_cm_wq", 0); if (!iwcm_wq) - return -ENOMEM; + goto err_alloc; iwcm_ctl_table_hdr = register_net_sysctl(&init_net, "net/iw_cm", iwcm_ctl_table); if (!iwcm_ctl_table_hdr) { pr_err("iw_cm: couldn't register sysctl paths\n"); - destroy_workqueue(iwcm_wq); - return -ENOMEM; + goto err_sysctl; } + rdma_nl_register(RDMA_NL_IWCM, iwcm_nl_cb_table); return 0; + +err_sysctl: + destroy_workqueue(iwcm_wq); +err_alloc: + iwpm_exit(RDMA_NL_IWCM); + return -ENOMEM; } static void __exit iw_cm_cleanup(void) { + rdma_nl_unregister(RDMA_NL_IWCM); unregister_net_sysctl_table(iwcm_ctl_table_hdr); destroy_workqueue(iwcm_wq); - rdma_nl_unregister(RDMA_NL_IWCM); iwpm_exit(RDMA_NL_IWCM); } -- cgit v1.2.3-59-g8ed1b From bdb0e4e3ff19b6699ab82828412f2fd9ad811b1d Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 23 Jul 2021 17:08:56 +0300 Subject: RDMA/iwpm: Remove not-needed reference counting iwpm_init() and iwpm_exit() are called only once during iw_cm module load. This makes whole reference count implementation not needed at all. Link: https://lore.kernel.org/r/1778ded873ba58c9fadc5bb25038de1cec843bec.1627048781.git.leonro@nvidia.com Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/iwpm_util.c | 62 ++++++++++--------------------------- drivers/infiniband/core/iwpm_util.h | 1 - 2 files changed, 16 insertions(+), 47 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/iwpm_util.c b/drivers/infiniband/core/iwpm_util.c index 3f8c019c7260..45e9aa503a44 100644 --- a/drivers/infiniband/core/iwpm_util.c +++ b/drivers/infiniband/core/iwpm_util.c @@ -48,7 +48,6 @@ static DEFINE_SPINLOCK(iwpm_mapinfo_lock); static struct hlist_head *iwpm_reminfo_bucket; static DEFINE_SPINLOCK(iwpm_reminfo_lock); -static DEFINE_MUTEX(iwpm_admin_lock); static struct iwpm_admin_data iwpm_admin; /** @@ -59,39 +58,22 @@ static struct iwpm_admin_data iwpm_admin; */ int iwpm_init(u8 nl_client) { - int ret = 0; - mutex_lock(&iwpm_admin_lock); - if (!refcount_read(&iwpm_admin.refcount)) { - iwpm_hash_bucket = kcalloc(IWPM_MAPINFO_HASH_SIZE, - sizeof(struct hlist_head), - GFP_KERNEL); - if (!iwpm_hash_bucket) { - ret = -ENOMEM; - goto init_exit; - } - iwpm_reminfo_bucket = kcalloc(IWPM_REMINFO_HASH_SIZE, - sizeof(struct hlist_head), - GFP_KERNEL); - if (!iwpm_reminfo_bucket) { - kfree(iwpm_hash_bucket); - ret = -ENOMEM; - goto init_exit; - } + iwpm_hash_bucket = kcalloc(IWPM_MAPINFO_HASH_SIZE, + sizeof(struct hlist_head), GFP_KERNEL); + if (!iwpm_hash_bucket) + return -ENOMEM; - refcount_set(&iwpm_admin.refcount, 1); - } else { - refcount_inc(&iwpm_admin.refcount); + iwpm_reminfo_bucket = kcalloc(IWPM_REMINFO_HASH_SIZE, + sizeof(struct hlist_head), GFP_KERNEL); + if (!iwpm_reminfo_bucket) { + kfree(iwpm_hash_bucket); + return -ENOMEM; } -init_exit: - mutex_unlock(&iwpm_admin_lock); - if (!ret) { - iwpm_set_valid(nl_client, 1); - iwpm_set_registration(nl_client, IWPM_REG_UNDEF); - pr_debug("%s: Mapinfo and reminfo tables are created\n", - __func__); - } - return ret; + iwpm_set_valid(nl_client, 1); + iwpm_set_registration(nl_client, IWPM_REG_UNDEF); + pr_debug("%s: Mapinfo and reminfo tables are created\n", __func__); + return 0; } static void free_hash_bucket(void); @@ -105,21 +87,9 @@ static void free_reminfo_bucket(void); */ int iwpm_exit(u8 nl_client) { - - if (!iwpm_valid_client(nl_client)) - return -EINVAL; - mutex_lock(&iwpm_admin_lock); - if (!refcount_read(&iwpm_admin.refcount)) { - mutex_unlock(&iwpm_admin_lock); - pr_err("%s Incorrect usage - negative refcount\n", __func__); - return -EINVAL; - } - if (refcount_dec_and_test(&iwpm_admin.refcount)) { - free_hash_bucket(); - free_reminfo_bucket(); - pr_debug("%s: Resources are destroyed\n", __func__); - } - mutex_unlock(&iwpm_admin_lock); + free_hash_bucket(); + free_reminfo_bucket(); + pr_debug("%s: Resources are destroyed\n", __func__); iwpm_set_valid(nl_client, 0); iwpm_set_registration(nl_client, IWPM_REG_UNDEF); return 0; diff --git a/drivers/infiniband/core/iwpm_util.h b/drivers/infiniband/core/iwpm_util.h index e201835de733..e2eacc017078 100644 --- a/drivers/infiniband/core/iwpm_util.h +++ b/drivers/infiniband/core/iwpm_util.h @@ -90,7 +90,6 @@ struct iwpm_remote_info { }; struct iwpm_admin_data { - refcount_t refcount; atomic_t nlmsg_seq; int client_list[RDMA_NL_NUM_CLIENTS]; u32 reg_list[RDMA_NL_NUM_CLIENTS]; -- cgit v1.2.3-59-g8ed1b From bbafcbc2b1c9a9b01ecd97020cf5057a22cb20b2 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 23 Jul 2021 17:08:57 +0300 Subject: RDMA/iwpm: Rely on the rdma_nl_[un]register() to ensure that requests are valid The core netlink code alread guarentees that no netlink callback can be running outside the rdma_nl_register/unregister() region and this registration happens during module init/exit. Thus it is already prevented that iwpm_valid_client() can ever fail. Remove it. Link: https://lore.kernel.org/r/a9f05a78f9996bf6ea47099b5e02671bf742f5ab.1627048781.git.leonro@nvidia.com Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/iwpm_msg.c | 34 +--------------------------------- drivers/infiniband/core/iwpm_util.c | 18 ------------------ drivers/infiniband/core/iwpm_util.h | 17 ----------------- 3 files changed, 1 insertion(+), 68 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/iwpm_msg.c b/drivers/infiniband/core/iwpm_msg.c index 12a9816fc0e2..3c9a9869212b 100644 --- a/drivers/infiniband/core/iwpm_msg.c +++ b/drivers/infiniband/core/iwpm_msg.c @@ -69,10 +69,6 @@ int iwpm_register_pid(struct iwpm_dev_data *pm_msg, u8 nl_client) const char *err_str = ""; int ret = -EINVAL; - if (!iwpm_valid_client(nl_client)) { - err_str = "Invalid port mapper client"; - goto pid_query_error; - } if (iwpm_check_registration(nl_client, IWPM_REG_VALID) || iwpm_user_pid == IWPM_PID_UNAVAILABLE) return 0; @@ -153,10 +149,6 @@ int iwpm_add_mapping(struct iwpm_sa_data *pm_msg, u8 nl_client) const char *err_str = ""; int ret = -EINVAL; - if (!iwpm_valid_client(nl_client)) { - err_str = "Invalid port mapper client"; - goto add_mapping_error; - } if (!iwpm_valid_pid()) return 0; if (!iwpm_check_registration(nl_client, IWPM_REG_VALID)) { @@ -240,10 +232,6 @@ int iwpm_add_and_query_mapping(struct iwpm_sa_data *pm_msg, u8 nl_client) const char *err_str = ""; int ret = -EINVAL; - if (!iwpm_valid_client(nl_client)) { - err_str = "Invalid port mapper client"; - goto query_mapping_error; - } if (!iwpm_valid_pid()) return 0; if (!iwpm_check_registration(nl_client, IWPM_REG_VALID)) { @@ -331,10 +319,6 @@ int iwpm_remove_mapping(struct sockaddr_storage *local_addr, u8 nl_client) const char *err_str = ""; int ret = -EINVAL; - if (!iwpm_valid_client(nl_client)) { - err_str = "Invalid port mapper client"; - goto remove_mapping_error; - } if (!iwpm_valid_pid()) return 0; if (iwpm_check_registration(nl_client, IWPM_REG_UNDEF)) { @@ -444,8 +428,7 @@ int iwpm_register_pid_cb(struct sk_buff *skb, struct netlink_callback *cb) atomic_set(&echo_nlmsg_seq, cb->nlh->nlmsg_seq); pr_debug("%s: iWarp Port Mapper (pid = %d) is available!\n", __func__, iwpm_user_pid); - if (iwpm_valid_client(nl_client)) - iwpm_set_registration(nl_client, IWPM_REG_VALID); + iwpm_set_registration(nl_client, IWPM_REG_VALID); register_pid_response_exit: nlmsg_request->request_done = 1; /* always for found nlmsg_request */ @@ -649,11 +632,6 @@ int iwpm_remote_info_cb(struct sk_buff *skb, struct netlink_callback *cb) return ret; nl_client = RDMA_NL_GET_CLIENT(cb->nlh->nlmsg_type); - if (!iwpm_valid_client(nl_client)) { - pr_info("%s: Invalid port mapper client = %u\n", - __func__, nl_client); - return ret; - } atomic_set(&echo_nlmsg_seq, cb->nlh->nlmsg_seq); local_sockaddr = (struct sockaddr_storage *) @@ -736,11 +714,6 @@ int iwpm_mapping_info_cb(struct sk_buff *skb, struct netlink_callback *cb) return ret; } nl_client = RDMA_NL_GET_CLIENT(cb->nlh->nlmsg_type); - if (!iwpm_valid_client(nl_client)) { - pr_info("%s: Invalid port mapper client = %u\n", - __func__, nl_client); - return ret; - } iwpm_set_registration(nl_client, IWPM_REG_INCOMPL); atomic_set(&echo_nlmsg_seq, cb->nlh->nlmsg_seq); iwpm_user_pid = cb->nlh->nlmsg_pid; @@ -863,11 +836,6 @@ int iwpm_hello_cb(struct sk_buff *skb, struct netlink_callback *cb) } abi_version = nla_get_u16(nltb[IWPM_NLA_HELLO_ABI_VERSION]); nl_client = RDMA_NL_GET_CLIENT(cb->nlh->nlmsg_type); - if (!iwpm_valid_client(nl_client)) { - pr_info("%s: Invalid port mapper client = %u\n", - __func__, nl_client); - return ret; - } iwpm_set_registration(nl_client, IWPM_REG_INCOMPL); atomic_set(&echo_nlmsg_seq, cb->nlh->nlmsg_seq); iwpm_ulib_version = min_t(u16, IWPM_UABI_VERSION, abi_version); diff --git a/drivers/infiniband/core/iwpm_util.c b/drivers/infiniband/core/iwpm_util.c index 45e9aa503a44..54f4feb604d8 100644 --- a/drivers/infiniband/core/iwpm_util.c +++ b/drivers/infiniband/core/iwpm_util.c @@ -70,7 +70,6 @@ int iwpm_init(u8 nl_client) return -ENOMEM; } - iwpm_set_valid(nl_client, 1); iwpm_set_registration(nl_client, IWPM_REG_UNDEF); pr_debug("%s: Mapinfo and reminfo tables are created\n", __func__); return 0; @@ -90,7 +89,6 @@ int iwpm_exit(u8 nl_client) free_hash_bucket(); free_reminfo_bucket(); pr_debug("%s: Resources are destroyed\n", __func__); - iwpm_set_valid(nl_client, 0); iwpm_set_registration(nl_client, IWPM_REG_UNDEF); return 0; } @@ -115,8 +113,6 @@ int iwpm_create_mapinfo(struct sockaddr_storage *local_sockaddr, unsigned long flags; int ret = -EINVAL; - if (!iwpm_valid_client(nl_client)) - return ret; map_info = kzalloc(sizeof(struct iwpm_mapping_info), GFP_KERNEL); if (!map_info) return -ENOMEM; @@ -276,10 +272,6 @@ int iwpm_get_remote_info(struct sockaddr_storage *mapped_loc_addr, unsigned long flags; int ret = -EINVAL; - if (!iwpm_valid_client(nl_client)) { - pr_info("%s: Invalid client = %u\n", __func__, nl_client); - return ret; - } spin_lock_irqsave(&iwpm_reminfo_lock, flags); if (iwpm_reminfo_bucket) { hash_bucket_head = get_reminfo_hash_bucket( @@ -394,16 +386,6 @@ int iwpm_get_nlmsg_seq(void) return atomic_inc_return(&iwpm_admin.nlmsg_seq); } -int iwpm_valid_client(u8 nl_client) -{ - return iwpm_admin.client_list[nl_client]; -} - -void iwpm_set_valid(u8 nl_client, int valid) -{ - iwpm_admin.client_list[nl_client] = valid; -} - /* valid client */ u32 iwpm_get_registration(u8 nl_client) { diff --git a/drivers/infiniband/core/iwpm_util.h b/drivers/infiniband/core/iwpm_util.h index e2eacc017078..3a42ad43056e 100644 --- a/drivers/infiniband/core/iwpm_util.h +++ b/drivers/infiniband/core/iwpm_util.h @@ -91,7 +91,6 @@ struct iwpm_remote_info { struct iwpm_admin_data { atomic_t nlmsg_seq; - int client_list[RDMA_NL_NUM_CLIENTS]; u32 reg_list[RDMA_NL_NUM_CLIENTS]; }; @@ -146,22 +145,6 @@ int iwpm_get_nlmsg_seq(void); */ void iwpm_add_remote_info(struct iwpm_remote_info *reminfo); -/** - * iwpm_valid_client - Check if the port mapper client is valid - * @nl_client: The index of the netlink client - * - * Valid clients need to call iwpm_init() before using - * the port mapper - */ -int iwpm_valid_client(u8 nl_client); - -/** - * iwpm_set_valid - Set the port mapper client to valid or not - * @nl_client: The index of the netlink client - * @valid: 1 if valid or 0 if invalid - */ -void iwpm_set_valid(u8 nl_client, int valid); - /** * iwpm_check_registration - Check if the client registration * matches the given one -- cgit v1.2.3-59-g8ed1b From 514aee660df493cd673154a6ba6bab745ec47b8c Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 23 Jul 2021 14:39:50 +0300 Subject: RDMA: Globally allocate and release QP memory Convert QP object to follow IB/core general allocation scheme. That change allows us to make sure that restrack properly kref the memory. Link: https://lore.kernel.org/r/48e767124758aeecc433360ddd85eaa6325b34d9.1627040189.git.leonro@nvidia.com Reviewed-by: Gal Pressman #efa Tested-by: Gal Pressman Reviewed-by: Dennis Dalessandro #rdma and core Tested-by: Dennis Dalessandro Signed-off-by: Leon Romanovsky Tested-by: Tatyana Nikolova Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/core_priv.h | 28 ++++++-- drivers/infiniband/core/device.c | 2 + drivers/infiniband/core/restrack.c | 2 +- drivers/infiniband/core/verbs.c | 40 +++++------ drivers/infiniband/hw/bnxt_re/ib_verbs.c | 26 +++---- drivers/infiniband/hw/bnxt_re/ib_verbs.h | 7 +- drivers/infiniband/hw/bnxt_re/main.c | 1 + drivers/infiniband/hw/cxgb4/iw_cxgb4.h | 5 +- drivers/infiniband/hw/cxgb4/provider.c | 1 + drivers/infiniband/hw/cxgb4/qp.c | 37 ++++------ drivers/infiniband/hw/efa/efa.h | 5 +- drivers/infiniband/hw/efa/efa_main.c | 1 + drivers/infiniband/hw/efa/efa_verbs.c | 28 +++----- drivers/infiniband/hw/hns/hns_roce_device.h | 5 +- drivers/infiniband/hw/hns/hns_roce_main.c | 1 + drivers/infiniband/hw/hns/hns_roce_qp.c | 28 +++----- drivers/infiniband/hw/irdma/utils.c | 3 - drivers/infiniband/hw/irdma/verbs.c | 31 ++++----- drivers/infiniband/hw/mlx4/main.c | 1 + drivers/infiniband/hw/mlx4/mlx4_ib.h | 5 +- drivers/infiniband/hw/mlx4/qp.c | 25 +++---- drivers/infiniband/hw/mlx5/gsi.c | 2 - drivers/infiniband/hw/mlx5/main.c | 1 + drivers/infiniband/hw/mlx5/mlx5_ib.h | 5 +- drivers/infiniband/hw/mlx5/qp.c | 56 +++++---------- drivers/infiniband/hw/mthca/mthca_provider.c | 77 ++++++++------------- drivers/infiniband/hw/ocrdma/ocrdma_main.c | 1 + drivers/infiniband/hw/ocrdma/ocrdma_verbs.c | 25 +++---- drivers/infiniband/hw/ocrdma/ocrdma_verbs.h | 5 +- drivers/infiniband/hw/qedr/main.c | 1 + drivers/infiniband/hw/qedr/qedr_roce_cm.c | 13 ++-- drivers/infiniband/hw/qedr/qedr_roce_cm.h | 5 +- drivers/infiniband/hw/qedr/verbs.c | 49 ++++--------- drivers/infiniband/hw/qedr/verbs.h | 4 +- drivers/infiniband/hw/usnic/usnic_ib_main.c | 1 + drivers/infiniband/hw/usnic/usnic_ib_qp_grp.c | 34 ++++----- drivers/infiniband/hw/usnic/usnic_ib_qp_grp.h | 10 +-- drivers/infiniband/hw/usnic/usnic_ib_verbs.c | 69 +++++++++---------- drivers/infiniband/hw/usnic/usnic_ib_verbs.h | 5 +- drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c | 1 + drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c | 53 ++++++-------- drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h | 5 +- drivers/infiniband/sw/rdmavt/qp.c | 91 +++++++++---------------- drivers/infiniband/sw/rdmavt/qp.h | 5 +- drivers/infiniband/sw/rdmavt/vt.c | 9 +++ drivers/infiniband/sw/rxe/rxe_pool.c | 2 +- drivers/infiniband/sw/rxe/rxe_verbs.c | 48 ++++++------- drivers/infiniband/sw/rxe/rxe_verbs.h | 2 +- drivers/infiniband/sw/siw/siw_main.c | 1 + drivers/infiniband/sw/siw/siw_qp.c | 2 - drivers/infiniband/sw/siw/siw_verbs.c | 54 +++++++-------- drivers/infiniband/sw/siw/siw_verbs.h | 5 +- include/rdma/ib_verbs.h | 30 ++++++-- 53 files changed, 404 insertions(+), 549 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h index 647cca4e0240..fa2e0bbaf8c7 100644 --- a/drivers/infiniband/core/core_priv.h +++ b/drivers/infiniband/core/core_priv.h @@ -322,13 +322,14 @@ _ib_create_qp(struct ib_device *dev, struct ib_pd *pd, struct ib_uqp_object *uobj, const char *caller) { struct ib_qp *qp; + int ret; if (!dev->ops.create_qp) return ERR_PTR(-EOPNOTSUPP); - qp = dev->ops.create_qp(pd, attr, udata); - if (IS_ERR(qp)) - return qp; + qp = rdma_zalloc_drv_obj_numa(dev, ib_qp); + if (!qp) + return ERR_PTR(-ENOMEM); qp->device = dev; qp->pd = pd; @@ -337,14 +338,10 @@ _ib_create_qp(struct ib_device *dev, struct ib_pd *pd, qp->qp_type = attr->qp_type; qp->rwq_ind_tbl = attr->rwq_ind_tbl; - qp->send_cq = attr->send_cq; - qp->recv_cq = attr->recv_cq; qp->srq = attr->srq; - qp->rwq_ind_tbl = attr->rwq_ind_tbl; qp->event_handler = attr->event_handler; qp->port = attr->port_num; - atomic_set(&qp->usecnt, 0); spin_lock_init(&qp->mr_lock); INIT_LIST_HEAD(&qp->rdma_mrs); INIT_LIST_HEAD(&qp->sig_mrs); @@ -352,8 +349,25 @@ _ib_create_qp(struct ib_device *dev, struct ib_pd *pd, rdma_restrack_new(&qp->res, RDMA_RESTRACK_QP); WARN_ONCE(!udata && !caller, "Missing kernel QP owner"); rdma_restrack_set_name(&qp->res, udata ? NULL : caller); + ret = dev->ops.create_qp(qp, attr, udata); + if (ret) + goto err_create; + + /* + * TODO: The mlx4 internally overwrites send_cq and recv_cq. + * Unfortunately, it is not an easy task to fix that driver. + */ + qp->send_cq = attr->send_cq; + qp->recv_cq = attr->recv_cq; + rdma_restrack_add(&qp->res); return qp; + +err_create: + rdma_restrack_put(&qp->res); + kfree(qp); + return ERR_PTR(ret); + } struct rdma_dev_addr; diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index 9056f48bdca6..f4814bb7f082 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -2654,6 +2654,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) SET_DEVICE_OP(dev_ops, get_hw_stats); SET_DEVICE_OP(dev_ops, get_link_layer); SET_DEVICE_OP(dev_ops, get_netdev); + SET_DEVICE_OP(dev_ops, get_numa_node); SET_DEVICE_OP(dev_ops, get_port_immutable); SET_DEVICE_OP(dev_ops, get_vector_affinity); SET_DEVICE_OP(dev_ops, get_vf_config); @@ -2710,6 +2711,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) SET_OBJ_SIZE(dev_ops, ib_cq); SET_OBJ_SIZE(dev_ops, ib_mw); SET_OBJ_SIZE(dev_ops, ib_pd); + SET_OBJ_SIZE(dev_ops, ib_qp); SET_OBJ_SIZE(dev_ops, ib_rwq_ind_table); SET_OBJ_SIZE(dev_ops, ib_srq); SET_OBJ_SIZE(dev_ops, ib_ucontext); diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c index 033207882c82..1f935d9f6178 100644 --- a/drivers/infiniband/core/restrack.c +++ b/drivers/infiniband/core/restrack.c @@ -343,7 +343,7 @@ void rdma_restrack_del(struct rdma_restrack_entry *res) rt = &dev->res[res->type]; old = xa_erase(&rt->xa, res->id); - if (res->type == RDMA_RESTRACK_MR || res->type == RDMA_RESTRACK_QP) + if (res->type == RDMA_RESTRACK_MR) return; WARN_ON(old != res); diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index 7036967e4c0b..a164609c2ee7 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -1963,30 +1963,32 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata) rdma_rw_cleanup_mrs(qp); rdma_counter_unbind_qp(qp, true); - rdma_restrack_del(&qp->res); ret = qp->device->ops.destroy_qp(qp, udata); - if (!ret) { - if (alt_path_sgid_attr) - rdma_put_gid_attr(alt_path_sgid_attr); - if (av_sgid_attr) - rdma_put_gid_attr(av_sgid_attr); - if (pd) - atomic_dec(&pd->usecnt); - if (scq) - atomic_dec(&scq->usecnt); - if (rcq) - atomic_dec(&rcq->usecnt); - if (srq) - atomic_dec(&srq->usecnt); - if (ind_tbl) - atomic_dec(&ind_tbl->usecnt); - if (sec) - ib_destroy_qp_security_end(sec); - } else { + if (ret) { if (sec) ib_destroy_qp_security_abort(sec); + return ret; } + if (alt_path_sgid_attr) + rdma_put_gid_attr(alt_path_sgid_attr); + if (av_sgid_attr) + rdma_put_gid_attr(av_sgid_attr); + if (pd) + atomic_dec(&pd->usecnt); + if (scq) + atomic_dec(&scq->usecnt); + if (rcq) + atomic_dec(&rcq->usecnt); + if (srq) + atomic_dec(&srq->usecnt); + if (ind_tbl) + atomic_dec(&ind_tbl->usecnt); + if (sec) + ib_destroy_qp_security_end(sec); + + rdma_restrack_del(&qp->res); + kfree(qp); return ret; } EXPORT_SYMBOL(ib_destroy_qp_user); diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 283b6b81563c..634d1586a1fa 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -815,7 +815,7 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp) { rc = bnxt_re_destroy_gsi_sqp(qp); if (rc) - goto sh_fail; + return rc; } mutex_lock(&rdev->qp_lock); @@ -826,10 +826,7 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) ib_umem_release(qp->rumem); ib_umem_release(qp->sumem); - kfree(qp); return 0; -sh_fail: - return rc; } static u8 __from_ib_qp_type(enum ib_qp_type type) @@ -1402,27 +1399,22 @@ static bool bnxt_re_test_qp_limits(struct bnxt_re_dev *rdev, return rc; } -struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd, - struct ib_qp_init_attr *qp_init_attr, - struct ib_udata *udata) +int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, + struct ib_udata *udata) { + struct ib_pd *ib_pd = ib_qp->pd; struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); struct bnxt_re_dev *rdev = pd->rdev; struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; - struct bnxt_re_qp *qp; + struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); int rc; rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr); if (!rc) { rc = -EINVAL; - goto exit; + goto fail; } - qp = kzalloc(sizeof(*qp), GFP_KERNEL); - if (!qp) { - rc = -ENOMEM; - goto exit; - } qp->rdev = rdev; rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, udata); if (rc) @@ -1465,16 +1457,14 @@ struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd, mutex_unlock(&rdev->qp_lock); atomic_inc(&rdev->qp_count); - return &qp->ib_qp; + return 0; qp_destroy: bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); free_umem: ib_umem_release(qp->rumem); ib_umem_release(qp->sumem); fail: - kfree(qp); -exit: - return ERR_PTR(rc); + return rc; } static u8 __from_ib_qp_state(enum ib_qp_state state) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h index d68671cc6173..b5c6e0f4f877 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h @@ -78,9 +78,9 @@ struct bnxt_re_srq { }; struct bnxt_re_qp { + struct ib_qp ib_qp; struct list_head list; struct bnxt_re_dev *rdev; - struct ib_qp ib_qp; spinlock_t sq_lock; /* protect sq */ spinlock_t rq_lock; /* protect rq */ struct bnxt_qplib_qp qplib_qp; @@ -179,9 +179,8 @@ int bnxt_re_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr); int bnxt_re_destroy_srq(struct ib_srq *srq, struct ib_udata *udata); int bnxt_re_post_srq_recv(struct ib_srq *srq, const struct ib_recv_wr *recv_wr, const struct ib_recv_wr **bad_recv_wr); -struct ib_qp *bnxt_re_create_qp(struct ib_pd *pd, - struct ib_qp_init_attr *qp_init_attr, - struct ib_udata *udata); +int bnxt_re_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *qp_init_attr, + struct ib_udata *udata); int bnxt_re_modify_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_udata *udata); int bnxt_re_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c index d5674026512a..3edf66818e4b 100644 --- a/drivers/infiniband/hw/bnxt_re/main.c +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -709,6 +709,7 @@ static const struct ib_device_ops bnxt_re_dev_ops = { INIT_RDMA_OBJ_SIZE(ib_ah, bnxt_re_ah, ib_ah), INIT_RDMA_OBJ_SIZE(ib_cq, bnxt_re_cq, ib_cq), INIT_RDMA_OBJ_SIZE(ib_pd, bnxt_re_pd, ib_pd), + INIT_RDMA_OBJ_SIZE(ib_qp, bnxt_re_qp, ib_qp), INIT_RDMA_OBJ_SIZE(ib_srq, bnxt_re_srq, ib_srq), INIT_RDMA_OBJ_SIZE(ib_ucontext, bnxt_re_ucontext, ib_uctx), }; diff --git a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h index 3883af3d2312..6a2a415ec791 100644 --- a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h +++ b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h @@ -989,9 +989,8 @@ int c4iw_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata); int c4iw_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *attrs, struct ib_udata *udata); int c4iw_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata); -struct ib_qp *c4iw_create_qp(struct ib_pd *pd, - struct ib_qp_init_attr *attrs, - struct ib_udata *udata); +int c4iw_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *attrs, + struct ib_udata *udata); int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata); int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c index 881d515eb15a..e7337662aff8 100644 --- a/drivers/infiniband/hw/cxgb4/provider.c +++ b/drivers/infiniband/hw/cxgb4/provider.c @@ -499,6 +499,7 @@ static const struct ib_device_ops c4iw_dev_ops = { INIT_RDMA_OBJ_SIZE(ib_cq, c4iw_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_mw, c4iw_mw, ibmw), INIT_RDMA_OBJ_SIZE(ib_pd, c4iw_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_qp, c4iw_qp, ibqp), INIT_RDMA_OBJ_SIZE(ib_srq, c4iw_srq, ibsrq), INIT_RDMA_OBJ_SIZE(ib_ucontext, c4iw_ucontext, ibucontext), }; diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c index a81fa7a56edb..d20b4ef2c853 100644 --- a/drivers/infiniband/hw/cxgb4/qp.c +++ b/drivers/infiniband/hw/cxgb4/qp.c @@ -2103,16 +2103,15 @@ int c4iw_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) ucontext ? &ucontext->uctx : &rhp->rdev.uctx, !qhp->srq); c4iw_put_wr_wait(qhp->wr_waitp); - - kfree(qhp); return 0; } -struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, - struct ib_udata *udata) +int c4iw_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *attrs, + struct ib_udata *udata) { + struct ib_pd *pd = qp->pd; struct c4iw_dev *rhp; - struct c4iw_qp *qhp; + struct c4iw_qp *qhp = to_c4iw_qp(qp); struct c4iw_pd *php; struct c4iw_cq *schp; struct c4iw_cq *rchp; @@ -2124,44 +2123,36 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, struct c4iw_mm_entry *sq_key_mm, *rq_key_mm = NULL, *sq_db_key_mm; struct c4iw_mm_entry *rq_db_key_mm = NULL, *ma_sync_key_mm = NULL; - pr_debug("ib_pd %p\n", pd); - if (attrs->qp_type != IB_QPT_RC || attrs->create_flags) - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; php = to_c4iw_pd(pd); rhp = php->rhp; schp = get_chp(rhp, ((struct c4iw_cq *)attrs->send_cq)->cq.cqid); rchp = get_chp(rhp, ((struct c4iw_cq *)attrs->recv_cq)->cq.cqid); if (!schp || !rchp) - return ERR_PTR(-EINVAL); + return -EINVAL; if (attrs->cap.max_inline_data > T4_MAX_SEND_INLINE) - return ERR_PTR(-EINVAL); + return -EINVAL; if (!attrs->srq) { if (attrs->cap.max_recv_wr > rhp->rdev.hw_queue.t4_max_rq_size) - return ERR_PTR(-E2BIG); + return -E2BIG; rqsize = attrs->cap.max_recv_wr + 1; if (rqsize < 8) rqsize = 8; } if (attrs->cap.max_send_wr > rhp->rdev.hw_queue.t4_max_sq_size) - return ERR_PTR(-E2BIG); + return -E2BIG; sqsize = attrs->cap.max_send_wr + 1; if (sqsize < 8) sqsize = 8; - qhp = kzalloc(sizeof(*qhp), GFP_KERNEL); - if (!qhp) - return ERR_PTR(-ENOMEM); - qhp->wr_waitp = c4iw_alloc_wr_wait(GFP_KERNEL); - if (!qhp->wr_waitp) { - ret = -ENOMEM; - goto err_free_qhp; - } + if (!qhp->wr_waitp) + return -ENOMEM; qhp->wq.sq.size = sqsize; qhp->wq.sq.memsize = @@ -2339,7 +2330,7 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, qhp->wq.sq.qid, qhp->wq.sq.size, qhp->wq.sq.memsize, attrs->cap.max_send_wr, qhp->wq.rq.qid, qhp->wq.rq.size, qhp->wq.rq.memsize, attrs->cap.max_recv_wr); - return &qhp->ibqp; + return 0; err_free_ma_sync_key: kfree(ma_sync_key_mm); err_free_rq_db_key: @@ -2359,9 +2350,7 @@ err_destroy_qp: ucontext ? &ucontext->uctx : &rhp->rdev.uctx, !attrs->srq); err_free_wr_wait: c4iw_put_wr_wait(qhp->wr_waitp); -err_free_qhp: - kfree(qhp); - return ERR_PTR(ret); + return ret; } int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, diff --git a/drivers/infiniband/hw/efa/efa.h b/drivers/infiniband/hw/efa/efa.h index 2b8ca099b381..1a1e60eee1dc 100644 --- a/drivers/infiniband/hw/efa/efa.h +++ b/drivers/infiniband/hw/efa/efa.h @@ -132,9 +132,8 @@ int efa_query_pkey(struct ib_device *ibdev, u32 port, u16 index, int efa_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); int efa_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); int efa_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata); -struct ib_qp *efa_create_qp(struct ib_pd *ibpd, - struct ib_qp_init_attr *init_attr, - struct ib_udata *udata); +int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata); int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct ib_udata *udata); diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c index 203e6ddcacbc..997947d77de6 100644 --- a/drivers/infiniband/hw/efa/efa_main.c +++ b/drivers/infiniband/hw/efa/efa_main.c @@ -271,6 +271,7 @@ static const struct ib_device_ops efa_dev_ops = { INIT_RDMA_OBJ_SIZE(ib_ah, efa_ah, ibah), INIT_RDMA_OBJ_SIZE(ib_cq, efa_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, efa_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_qp, efa_qp, ibqp), INIT_RDMA_OBJ_SIZE(ib_ucontext, efa_ucontext, ibucontext), }; diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index b1c4780e86be..e5f9d90aad5e 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -450,7 +450,6 @@ int efa_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) qp->rq_size, DMA_TO_DEVICE); } - kfree(qp); return 0; } @@ -609,17 +608,16 @@ static int efa_qp_validate_attr(struct efa_dev *dev, return 0; } -struct ib_qp *efa_create_qp(struct ib_pd *ibpd, - struct ib_qp_init_attr *init_attr, - struct ib_udata *udata) +int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) { struct efa_com_create_qp_params create_qp_params = {}; struct efa_com_create_qp_result create_qp_resp; - struct efa_dev *dev = to_edev(ibpd->device); + struct efa_dev *dev = to_edev(ibqp->device); struct efa_ibv_create_qp_resp resp = {}; struct efa_ibv_create_qp cmd = {}; + struct efa_qp *qp = to_eqp(ibqp); struct efa_ucontext *ucontext; - struct efa_qp *qp; int err; ucontext = rdma_udata_to_drv_context(udata, struct efa_ucontext, @@ -664,14 +662,8 @@ struct ib_qp *efa_create_qp(struct ib_pd *ibpd, goto err_out; } - qp = kzalloc(sizeof(*qp), GFP_KERNEL); - if (!qp) { - err = -ENOMEM; - goto err_out; - } - create_qp_params.uarn = ucontext->uarn; - create_qp_params.pd = to_epd(ibpd)->pdn; + create_qp_params.pd = to_epd(ibqp->pd)->pdn; if (init_attr->qp_type == IB_QPT_UD) { create_qp_params.qp_type = EFA_ADMIN_QP_TYPE_UD; @@ -682,7 +674,7 @@ struct ib_qp *efa_create_qp(struct ib_pd *ibpd, "Unsupported qp type %d driver qp type %d\n", init_attr->qp_type, cmd.driver_qp_type); err = -EOPNOTSUPP; - goto err_free_qp; + goto err_out; } ibdev_dbg(&dev->ibdev, "Create QP: qp type %d driver qp type %#x\n", @@ -700,7 +692,7 @@ struct ib_qp *efa_create_qp(struct ib_pd *ibpd, qp->rq_size, DMA_TO_DEVICE); if (!qp->rq_cpu_addr) { err = -ENOMEM; - goto err_free_qp; + goto err_out; } ibdev_dbg(&dev->ibdev, @@ -746,7 +738,7 @@ struct ib_qp *efa_create_qp(struct ib_pd *ibpd, ibdev_dbg(&dev->ibdev, "Created qp[%d]\n", qp->ibqp.qp_num); - return &qp->ibqp; + return 0; err_remove_mmap_entries: efa_qp_user_mmap_entries_remove(qp); @@ -756,11 +748,9 @@ err_free_mapped: if (qp->rq_size) efa_free_mapped(dev, qp->rq_cpu_addr, qp->rq_dma_addr, qp->rq_size, DMA_TO_DEVICE); -err_free_qp: - kfree(qp); err_out: atomic64_inc(&dev->stats.create_qp_err); - return ERR_PTR(err); + return err; } static const struct { diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 991f65269fa6..0c3eb1163977 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -1216,9 +1216,8 @@ int hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata); int hns_roce_alloc_xrcd(struct ib_xrcd *ib_xrcd, struct ib_udata *udata); int hns_roce_dealloc_xrcd(struct ib_xrcd *ib_xrcd, struct ib_udata *udata); -struct ib_qp *hns_roce_create_qp(struct ib_pd *ib_pd, - struct ib_qp_init_attr *init_attr, - struct ib_udata *udata); +int hns_roce_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata); int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata); void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp); diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index 078a97193f0e..23b88a5a372f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -454,6 +454,7 @@ static const struct ib_device_ops hns_roce_dev_ops = { INIT_RDMA_OBJ_SIZE(ib_ah, hns_roce_ah, ibah), INIT_RDMA_OBJ_SIZE(ib_cq, hns_roce_cq, ib_cq), INIT_RDMA_OBJ_SIZE(ib_pd, hns_roce_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_qp, hns_roce_qp, ibqp), INIT_RDMA_OBJ_SIZE(ib_ucontext, hns_roce_ucontext, ibucontext), }; diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index c3e2fee16c0e..fd0f71acd470 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -959,8 +959,6 @@ static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, struct ib_device *ibdev = &hr_dev->ib_dev; int ret; - hr_qp->ibqp.qp_type = init_attr->qp_type; - if (init_attr->cap.max_inline_data > hr_dev->caps.max_sq_inline) init_attr->cap.max_inline_data = hr_dev->caps.max_sq_inline; @@ -1121,8 +1119,6 @@ void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, free_qp_buf(hr_dev, hr_qp); free_kernel_wrid(hr_qp); free_qp_db(hr_dev, hr_qp, udata); - - kfree(hr_qp); } static int check_qp_type(struct hns_roce_dev *hr_dev, enum ib_qp_type type, @@ -1154,22 +1150,18 @@ out: return -EOPNOTSUPP; } -struct ib_qp *hns_roce_create_qp(struct ib_pd *pd, - struct ib_qp_init_attr *init_attr, - struct ib_udata *udata) +int hns_roce_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) { - struct ib_device *ibdev = pd ? pd->device : init_attr->xrcd->device; + struct ib_device *ibdev = qp->device; struct hns_roce_dev *hr_dev = to_hr_dev(ibdev); - struct hns_roce_qp *hr_qp; + struct hns_roce_qp *hr_qp = to_hr_qp(qp); + struct ib_pd *pd = qp->pd; int ret; ret = check_qp_type(hr_dev, init_attr->qp_type, !!udata); if (ret) - return ERR_PTR(ret); - - hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL); - if (!hr_qp) - return ERR_PTR(-ENOMEM); + return ret; if (init_attr->qp_type == IB_QPT_XRC_TGT) hr_qp->xrcdn = to_hr_xrcd(init_attr->xrcd)->xrcdn; @@ -1180,15 +1172,11 @@ struct ib_qp *hns_roce_create_qp(struct ib_pd *pd, } ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, hr_qp); - if (ret) { + if (ret) ibdev_err(ibdev, "Create QP type 0x%x failed(%d)\n", init_attr->qp_type, ret); - kfree(hr_qp); - return ERR_PTR(ret); - } - - return &hr_qp->ibqp; + return ret; } int to_hr_qp_type(int qp_type) diff --git a/drivers/infiniband/hw/irdma/utils.c b/drivers/infiniband/hw/irdma/utils.c index 5bbe44e54f9a..e94470991fe0 100644 --- a/drivers/infiniband/hw/irdma/utils.c +++ b/drivers/infiniband/hw/irdma/utils.c @@ -1141,10 +1141,7 @@ void irdma_free_qp_rsrc(struct irdma_qp *iwqp) iwqp->kqp.dma_mem.va, iwqp->kqp.dma_mem.pa); iwqp->kqp.dma_mem.va = NULL; kfree(iwqp->kqp.sq_wrid_mem); - iwqp->kqp.sq_wrid_mem = NULL; kfree(iwqp->kqp.rq_wrid_mem); - iwqp->kqp.rq_wrid_mem = NULL; - kfree(iwqp); } /** diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index 9712f6902ba8..9b8c451e2426 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -792,18 +792,19 @@ static int irdma_validate_qp_attrs(struct ib_qp_init_attr *init_attr, /** * irdma_create_qp - create qp - * @ibpd: ptr of pd + * @ibqp: ptr of qp * @init_attr: attributes for qp * @udata: user data for create qp */ -static struct ib_qp *irdma_create_qp(struct ib_pd *ibpd, - struct ib_qp_init_attr *init_attr, - struct ib_udata *udata) +static int irdma_create_qp(struct ib_qp *ibqp, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) { + struct ib_pd *ibpd = ibqp->pd; struct irdma_pd *iwpd = to_iwpd(ibpd); struct irdma_device *iwdev = to_iwdev(ibpd->device); struct irdma_pci_f *rf = iwdev->rf; - struct irdma_qp *iwqp; + struct irdma_qp *iwqp = to_iwqp(ibqp); struct irdma_create_qp_req req; struct irdma_create_qp_resp uresp = {}; u32 qp_num = 0; @@ -820,7 +821,7 @@ static struct ib_qp *irdma_create_qp(struct ib_pd *ibpd, err_code = irdma_validate_qp_attrs(init_attr, iwdev); if (err_code) - return ERR_PTR(err_code); + return err_code; sq_size = init_attr->cap.max_send_wr; rq_size = init_attr->cap.max_recv_wr; @@ -833,10 +834,6 @@ static struct ib_qp *irdma_create_qp(struct ib_pd *ibpd, init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge; init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data; - iwqp = kzalloc(sizeof(*iwqp), GFP_KERNEL); - if (!iwqp) - return ERR_PTR(-ENOMEM); - qp = &iwqp->sc_qp; qp->qp_uk.back_qp = iwqp; qp->qp_uk.lock = &iwqp->lock; @@ -849,10 +846,8 @@ static struct ib_qp *irdma_create_qp(struct ib_pd *ibpd, iwqp->q2_ctx_mem.size, &iwqp->q2_ctx_mem.pa, GFP_KERNEL); - if (!iwqp->q2_ctx_mem.va) { - err_code = -ENOMEM; - goto error; - } + if (!iwqp->q2_ctx_mem.va) + return -ENOMEM; init_info.q2 = iwqp->q2_ctx_mem.va; init_info.q2_pa = iwqp->q2_ctx_mem.pa; @@ -1001,17 +996,16 @@ static struct ib_qp *irdma_create_qp(struct ib_pd *ibpd, if (err_code) { ibdev_dbg(&iwdev->ibdev, "VERBS: copy_to_udata failed\n"); irdma_destroy_qp(&iwqp->ibqp, udata); - return ERR_PTR(err_code); + return err_code; } } init_completion(&iwqp->free_qp); - return &iwqp->ibqp; + return 0; error: irdma_free_qp_rsrc(iwqp); - - return ERR_PTR(err_code); + return err_code; } static int irdma_get_ib_acc_flags(struct irdma_qp *iwqp) @@ -4406,6 +4400,7 @@ static const struct ib_device_ops irdma_dev_ops = { INIT_RDMA_OBJ_SIZE(ib_ah, irdma_ah, ibah), INIT_RDMA_OBJ_SIZE(ib_cq, irdma_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_mw, irdma_mr, ibmw), + INIT_RDMA_OBJ_SIZE(ib_qp, irdma_qp, ibqp), }; /** diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index ae4c91b612ce..f367f4a4abff 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -2577,6 +2577,7 @@ static const struct ib_device_ops mlx4_ib_dev_ops = { INIT_RDMA_OBJ_SIZE(ib_ah, mlx4_ib_ah, ibah), INIT_RDMA_OBJ_SIZE(ib_cq, mlx4_ib_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, mlx4_ib_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_qp, mlx4_ib_qp, ibqp), INIT_RDMA_OBJ_SIZE(ib_srq, mlx4_ib_srq, ibsrq), INIT_RDMA_OBJ_SIZE(ib_ucontext, mlx4_ib_ucontext, ibucontext), }; diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h index e856cf23a0a1..c60f6e9ac640 100644 --- a/drivers/infiniband/hw/mlx4/mlx4_ib.h +++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h @@ -792,9 +792,8 @@ void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index); int mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr); -struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd, - struct ib_qp_init_attr *init_attr, - struct ib_udata *udata); +int mlx4_ib_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata); int mlx4_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata); void mlx4_ib_drain_sq(struct ib_qp *qp); void mlx4_ib_drain_rq(struct ib_qp *qp); diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c index 4a2ef7daaded..8662f462e2a5 100644 --- a/drivers/infiniband/hw/mlx4/qp.c +++ b/drivers/infiniband/hw/mlx4/qp.c @@ -1578,24 +1578,19 @@ static int _mlx4_ib_create_qp(struct ib_pd *pd, struct mlx4_ib_qp *qp, return 0; } -struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd, - struct ib_qp_init_attr *init_attr, - struct ib_udata *udata) { - struct ib_device *device = pd ? pd->device : init_attr->xrcd->device; +int mlx4_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) +{ + struct ib_device *device = ibqp->device; struct mlx4_ib_dev *dev = to_mdev(device); - struct mlx4_ib_qp *qp; + struct mlx4_ib_qp *qp = to_mqp(ibqp); + struct ib_pd *pd = ibqp->pd; int ret; - qp = kzalloc(sizeof(*qp), GFP_KERNEL); - if (!qp) - return ERR_PTR(-ENOMEM); - mutex_init(&qp->mutex); ret = _mlx4_ib_create_qp(pd, qp, init_attr, udata); - if (ret) { - kfree(qp); - return ERR_PTR(ret); - } + if (ret) + return ret; if (init_attr->qp_type == IB_QPT_GSI && !(init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI)) { @@ -1618,7 +1613,7 @@ struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd, init_attr->create_flags &= ~MLX4_IB_QP_CREATE_ROCE_V2_GSI; } } - return &qp->ibqp; + return 0; } static int _mlx4_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) @@ -1646,8 +1641,6 @@ static int _mlx4_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) } kfree(mqp->sqp); - kfree(mqp); - return 0; } diff --git a/drivers/infiniband/hw/mlx5/gsi.c b/drivers/infiniband/hw/mlx5/gsi.c index 541da52470cb..3ad8f637c589 100644 --- a/drivers/infiniband/hw/mlx5/gsi.c +++ b/drivers/infiniband/hw/mlx5/gsi.c @@ -193,8 +193,6 @@ int mlx5_ib_destroy_gsi(struct mlx5_ib_qp *mqp) kfree(gsi->outstanding_wrs); kfree(gsi->tx_qps); - kfree(mqp); - return 0; } diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 7aa513edc6db..46f28514cf3a 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -3805,6 +3805,7 @@ static const struct ib_device_ops mlx5_ib_dev_ops = { INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs), INIT_RDMA_OBJ_SIZE(ib_cq, mlx5_ib_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, mlx5_ib_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_qp, mlx5_ib_qp, ibqp), INIT_RDMA_OBJ_SIZE(ib_srq, mlx5_ib_srq, ibsrq), INIT_RDMA_OBJ_SIZE(ib_ucontext, mlx5_ib_ucontext, ibucontext), }; diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 0aa19cd90a57..bf20a388eabe 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -1219,9 +1219,8 @@ int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr); int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp); void mlx5_ib_disable_lb(struct mlx5_ib_dev *dev, bool td, bool qp); -struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd, - struct ib_qp_init_attr *init_attr, - struct ib_udata *udata); +int mlx5_ib_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata); int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata); int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 297aacc5d7f9..9d20c838974f 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -3114,7 +3114,6 @@ static int mlx5_ib_destroy_dct(struct mlx5_ib_qp *mqp) } kfree(mqp->dct.in); - kfree(mqp); return 0; } @@ -3152,25 +3151,23 @@ static int check_ucmd_data(struct mlx5_ib_dev *dev, return ret ? 0 : -EINVAL; } -struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attr, - struct ib_udata *udata) +int mlx5_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, + struct ib_udata *udata) { struct mlx5_create_qp_params params = {}; - struct mlx5_ib_dev *dev; - struct mlx5_ib_qp *qp; + struct mlx5_ib_dev *dev = to_mdev(ibqp->device); + struct mlx5_ib_qp *qp = to_mqp(ibqp); + struct ib_pd *pd = ibqp->pd; enum ib_qp_type type; int err; - dev = pd ? to_mdev(pd->device) : - to_mdev(to_mxrcd(attr->xrcd)->ibxrcd.device); - err = check_qp_type(dev, attr, &type); if (err) - return ERR_PTR(err); + return err; err = check_valid_flow(dev, pd, attr, udata); if (err) - return ERR_PTR(err); + return err; params.udata = udata; params.uidx = MLX5_IB_DEFAULT_UIDX; @@ -3180,49 +3177,43 @@ struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attr, if (udata) { err = process_udata_size(dev, ¶ms); if (err) - return ERR_PTR(err); + return err; err = check_ucmd_data(dev, ¶ms); if (err) - return ERR_PTR(err); + return err; params.ucmd = kzalloc(params.ucmd_size, GFP_KERNEL); if (!params.ucmd) - return ERR_PTR(-ENOMEM); + return -ENOMEM; err = ib_copy_from_udata(params.ucmd, udata, params.inlen); if (err) goto free_ucmd; } - qp = kzalloc(sizeof(*qp), GFP_KERNEL); - if (!qp) { - err = -ENOMEM; - goto free_ucmd; - } - mutex_init(&qp->mutex); qp->type = type; if (udata) { err = process_vendor_flags(dev, qp, params.ucmd, attr); if (err) - goto free_qp; + goto free_ucmd; err = get_qp_uidx(qp, ¶ms); if (err) - goto free_qp; + goto free_ucmd; } err = process_create_flags(dev, qp, attr); if (err) - goto free_qp; + goto free_ucmd; err = check_qp_attr(dev, qp, attr); if (err) - goto free_qp; + goto free_ucmd; err = create_qp(dev, pd, qp, ¶ms); if (err) - goto free_qp; + goto free_ucmd; kfree(params.ucmd); params.ucmd = NULL; @@ -3237,7 +3228,7 @@ struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attr, if (err) goto destroy_qp; - return &qp->ibqp; + return 0; destroy_qp: switch (qp->type) { @@ -3248,22 +3239,12 @@ destroy_qp: mlx5_ib_destroy_gsi(qp); break; default: - /* - * These lines below are temp solution till QP allocation - * will be moved to be under IB/core responsiblity. - */ - qp->ibqp.send_cq = attr->send_cq; - qp->ibqp.recv_cq = attr->recv_cq; - qp->ibqp.pd = pd; destroy_qp_common(dev, qp, udata); } - qp = NULL; -free_qp: - kfree(qp); free_ucmd: kfree(params.ucmd); - return ERR_PTR(err); + return err; } int mlx5_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) @@ -3278,9 +3259,6 @@ int mlx5_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) return mlx5_ib_destroy_dct(mqp); destroy_qp_common(dev, mqp, udata); - - kfree(mqp); - return 0; } diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c index adf4fcf0fee4..ceee23ebc0f2 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.c +++ b/drivers/infiniband/hw/mthca/mthca_provider.c @@ -459,52 +459,45 @@ static int mthca_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) return 0; } -static struct ib_qp *mthca_create_qp(struct ib_pd *pd, - struct ib_qp_init_attr *init_attr, - struct ib_udata *udata) +static int mthca_create_qp(struct ib_qp *ibqp, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) { struct mthca_ucontext *context = rdma_udata_to_drv_context( udata, struct mthca_ucontext, ibucontext); struct mthca_create_qp ucmd; - struct mthca_qp *qp; + struct mthca_qp *qp = to_mqp(ibqp); + struct mthca_dev *dev = to_mdev(ibqp->device); int err; if (init_attr->create_flags) - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; switch (init_attr->qp_type) { case IB_QPT_RC: case IB_QPT_UC: case IB_QPT_UD: { - qp = kzalloc(sizeof(*qp), GFP_KERNEL); - if (!qp) - return ERR_PTR(-ENOMEM); - if (udata) { - if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { - kfree(qp); - return ERR_PTR(-EFAULT); - } + if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) + return -EFAULT; - err = mthca_map_user_db(to_mdev(pd->device), &context->uar, + err = mthca_map_user_db(dev, &context->uar, context->db_tab, - ucmd.sq_db_index, ucmd.sq_db_page); - if (err) { - kfree(qp); - return ERR_PTR(err); - } + ucmd.sq_db_index, + ucmd.sq_db_page); + if (err) + return err; - err = mthca_map_user_db(to_mdev(pd->device), &context->uar, + err = mthca_map_user_db(dev, &context->uar, context->db_tab, - ucmd.rq_db_index, ucmd.rq_db_page); + ucmd.rq_db_index, + ucmd.rq_db_page); if (err) { - mthca_unmap_user_db(to_mdev(pd->device), - &context->uar, + mthca_unmap_user_db(dev, &context->uar, context->db_tab, ucmd.sq_db_index); - kfree(qp); - return ERR_PTR(err); + return err; } qp->mr.ibmr.lkey = ucmd.lkey; @@ -512,20 +505,16 @@ static struct ib_qp *mthca_create_qp(struct ib_pd *pd, qp->rq.db_index = ucmd.rq_db_index; } - err = mthca_alloc_qp(to_mdev(pd->device), to_mpd(pd), + err = mthca_alloc_qp(dev, to_mpd(ibqp->pd), to_mcq(init_attr->send_cq), to_mcq(init_attr->recv_cq), init_attr->qp_type, init_attr->sq_sig_type, &init_attr->cap, qp, udata); if (err && udata) { - mthca_unmap_user_db(to_mdev(pd->device), - &context->uar, - context->db_tab, + mthca_unmap_user_db(dev, &context->uar, context->db_tab, ucmd.sq_db_index); - mthca_unmap_user_db(to_mdev(pd->device), - &context->uar, - context->db_tab, + mthca_unmap_user_db(dev, &context->uar, context->db_tab, ucmd.rq_db_index); } @@ -535,34 +524,28 @@ static struct ib_qp *mthca_create_qp(struct ib_pd *pd, case IB_QPT_SMI: case IB_QPT_GSI: { - qp = kzalloc(sizeof(*qp), GFP_KERNEL); - if (!qp) - return ERR_PTR(-ENOMEM); qp->sqp = kzalloc(sizeof(struct mthca_sqp), GFP_KERNEL); - if (!qp->sqp) { - kfree(qp); - return ERR_PTR(-ENOMEM); - } + if (!qp->sqp) + return -ENOMEM; qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1; - err = mthca_alloc_sqp(to_mdev(pd->device), to_mpd(pd), + err = mthca_alloc_sqp(dev, to_mpd(ibqp->pd), to_mcq(init_attr->send_cq), to_mcq(init_attr->recv_cq), init_attr->sq_sig_type, &init_attr->cap, - qp->ibqp.qp_num, init_attr->port_num, - qp, udata); + qp->ibqp.qp_num, init_attr->port_num, qp, + udata); break; } default: /* Don't support raw QPs */ - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; } if (err) { kfree(qp->sqp); - kfree(qp); - return ERR_PTR(err); + return err; } init_attr->cap.max_send_wr = qp->sq.max; @@ -571,7 +554,7 @@ static struct ib_qp *mthca_create_qp(struct ib_pd *pd, init_attr->cap.max_recv_sge = qp->rq.max_gs; init_attr->cap.max_inline_data = qp->max_inline_data; - return &qp->ibqp; + return 0; } static int mthca_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) @@ -594,7 +577,6 @@ static int mthca_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) } mthca_free_qp(to_mdev(qp->device), to_mqp(qp)); kfree(to_mqp(qp)->sqp); - kfree(to_mqp(qp)); return 0; } @@ -1121,6 +1103,7 @@ static const struct ib_device_ops mthca_dev_ops = { INIT_RDMA_OBJ_SIZE(ib_ah, mthca_ah, ibah), INIT_RDMA_OBJ_SIZE(ib_cq, mthca_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, mthca_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_qp, mthca_qp, ibqp), INIT_RDMA_OBJ_SIZE(ib_ucontext, mthca_ucontext, ibucontext), }; diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_main.c b/drivers/infiniband/hw/ocrdma/ocrdma_main.c index f329db0c591f..7abf6cf1e937 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_main.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_main.c @@ -185,6 +185,7 @@ static const struct ib_device_ops ocrdma_dev_ops = { INIT_RDMA_OBJ_SIZE(ib_ah, ocrdma_ah, ibah), INIT_RDMA_OBJ_SIZE(ib_cq, ocrdma_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, ocrdma_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_qp, ocrdma_qp, ibqp), INIT_RDMA_OBJ_SIZE(ib_ucontext, ocrdma_ucontext, ibucontext), }; diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c index 58619ce64d0d..735123d0e9ec 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c @@ -1288,19 +1288,19 @@ static void ocrdma_store_gsi_qp_cq(struct ocrdma_dev *dev, } } -struct ib_qp *ocrdma_create_qp(struct ib_pd *ibpd, - struct ib_qp_init_attr *attrs, - struct ib_udata *udata) +int ocrdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, + struct ib_udata *udata) { int status; + struct ib_pd *ibpd = ibqp->pd; struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); - struct ocrdma_qp *qp; - struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); + struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); + struct ocrdma_dev *dev = get_ocrdma_dev(ibqp->device); struct ocrdma_create_qp_ureq ureq; u16 dpp_credit_lmt, dpp_offset; if (attrs->create_flags) - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; status = ocrdma_check_qp_params(ibpd, dev, attrs, udata); if (status) @@ -1309,12 +1309,7 @@ struct ib_qp *ocrdma_create_qp(struct ib_pd *ibpd, memset(&ureq, 0, sizeof(ureq)); if (udata) { if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) - return ERR_PTR(-EFAULT); - } - qp = kzalloc(sizeof(*qp), GFP_KERNEL); - if (!qp) { - status = -ENOMEM; - goto gen_err; + return -EFAULT; } ocrdma_set_qp_init_params(qp, pd, attrs); if (udata == NULL) @@ -1349,7 +1344,7 @@ struct ib_qp *ocrdma_create_qp(struct ib_pd *ibpd, ocrdma_store_gsi_qp_cq(dev, attrs); qp->ibqp.qp_num = qp->id; mutex_unlock(&dev->dev_lock); - return &qp->ibqp; + return 0; cpy_err: ocrdma_del_qpn_map(dev, qp); @@ -1359,10 +1354,9 @@ mbx_err: mutex_unlock(&dev->dev_lock); kfree(qp->wqe_wr_id_tbl); kfree(qp->rqe_wr_id_tbl); - kfree(qp); pr_err("%s(%d) error=%d\n", __func__, dev->id, status); gen_err: - return ERR_PTR(status); + return status; } int _ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, @@ -1731,7 +1725,6 @@ int ocrdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) kfree(qp->wqe_wr_id_tbl); kfree(qp->rqe_wr_id_tbl); - kfree(qp); return 0; } diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h index b1c5fad81603..b73d742a520c 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h @@ -75,9 +75,8 @@ int ocrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, int ocrdma_resize_cq(struct ib_cq *, int cqe, struct ib_udata *); int ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); -struct ib_qp *ocrdma_create_qp(struct ib_pd *, - struct ib_qp_init_attr *attrs, - struct ib_udata *); +int ocrdma_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *attrs, + struct ib_udata *udata); int _ocrdma_modify_qp(struct ib_qp *, struct ib_qp_attr *attr, int attr_mask); int ocrdma_modify_qp(struct ib_qp *, struct ib_qp_attr *attr, diff --git a/drivers/infiniband/hw/qedr/main.c b/drivers/infiniband/hw/qedr/main.c index de98e0604f91..755930be01b8 100644 --- a/drivers/infiniband/hw/qedr/main.c +++ b/drivers/infiniband/hw/qedr/main.c @@ -233,6 +233,7 @@ static const struct ib_device_ops qedr_dev_ops = { INIT_RDMA_OBJ_SIZE(ib_ah, qedr_ah, ibah), INIT_RDMA_OBJ_SIZE(ib_cq, qedr_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, qedr_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_qp, qedr_qp, ibqp), INIT_RDMA_OBJ_SIZE(ib_srq, qedr_srq, ibsrq), INIT_RDMA_OBJ_SIZE(ib_xrcd, qedr_xrcd, ibxrcd), INIT_RDMA_OBJ_SIZE(ib_ucontext, qedr_ucontext, ibucontext), diff --git a/drivers/infiniband/hw/qedr/qedr_roce_cm.c b/drivers/infiniband/hw/qedr/qedr_roce_cm.c index 13e5e6bbec99..05307c1488b8 100644 --- a/drivers/infiniband/hw/qedr/qedr_roce_cm.c +++ b/drivers/infiniband/hw/qedr/qedr_roce_cm.c @@ -319,20 +319,19 @@ err1: return rc; } -struct ib_qp *qedr_create_gsi_qp(struct qedr_dev *dev, - struct ib_qp_init_attr *attrs, - struct qedr_qp *qp) +int qedr_create_gsi_qp(struct qedr_dev *dev, struct ib_qp_init_attr *attrs, + struct qedr_qp *qp) { int rc; rc = qedr_check_gsi_qp_attrs(dev, attrs); if (rc) - return ERR_PTR(rc); + return rc; rc = qedr_ll2_start(dev, attrs, qp); if (rc) { DP_ERR(dev, "create gsi qp: failed on ll2 start. rc=%d\n", rc); - return ERR_PTR(rc); + return rc; } /* create QP */ @@ -359,7 +358,7 @@ struct ib_qp *qedr_create_gsi_qp(struct qedr_dev *dev, DP_DEBUG(dev, QEDR_MSG_GSI, "created GSI QP %p\n", qp); - return &qp->ibqp; + return 0; err: kfree(qp->rqe_wr_id); @@ -368,7 +367,7 @@ err: if (rc) DP_ERR(dev, "create gsi qp: failed destroy on create\n"); - return ERR_PTR(-ENOMEM); + return -ENOMEM; } int qedr_destroy_gsi_qp(struct qedr_dev *dev) diff --git a/drivers/infiniband/hw/qedr/qedr_roce_cm.h b/drivers/infiniband/hw/qedr/qedr_roce_cm.h index d46dcd3f6424..f3432f035ec6 100644 --- a/drivers/infiniband/hw/qedr/qedr_roce_cm.h +++ b/drivers/infiniband/hw/qedr/qedr_roce_cm.h @@ -50,9 +50,8 @@ int qedr_gsi_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr); int qedr_gsi_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr); -struct ib_qp *qedr_create_gsi_qp(struct qedr_dev *dev, - struct ib_qp_init_attr *attrs, - struct qedr_qp *qp); +int qedr_create_gsi_qp(struct qedr_dev *dev, struct ib_qp_init_attr *attrs, + struct qedr_qp *qp); void qedr_store_gsi_qp_cq(struct qedr_dev *dev, struct qedr_qp *qp, struct ib_qp_init_attr *attrs); int qedr_destroy_gsi_qp(struct qedr_dev *dev); diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c index b72ef24db657..16d4c0228d76 100644 --- a/drivers/infiniband/hw/qedr/verbs.c +++ b/drivers/infiniband/hw/qedr/verbs.c @@ -2239,34 +2239,30 @@ static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp, return 0; } -struct ib_qp *qedr_create_qp(struct ib_pd *ibpd, - struct ib_qp_init_attr *attrs, - struct ib_udata *udata) +int qedr_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, + struct ib_udata *udata) { struct qedr_xrcd *xrcd = NULL; - struct qedr_pd *pd = NULL; - struct qedr_dev *dev; - struct qedr_qp *qp; - struct ib_qp *ibqp; + struct ib_pd *ibpd = ibqp->pd; + struct qedr_pd *pd = get_qedr_pd(ibpd); + struct qedr_dev *dev = get_qedr_dev(ibqp->device); + struct qedr_qp *qp = get_qedr_qp(ibqp); int rc = 0; if (attrs->create_flags) - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; - if (attrs->qp_type == IB_QPT_XRC_TGT) { + if (attrs->qp_type == IB_QPT_XRC_TGT) xrcd = get_qedr_xrcd(attrs->xrcd); - dev = get_qedr_dev(xrcd->ibxrcd.device); - } else { + else pd = get_qedr_pd(ibpd); - dev = get_qedr_dev(ibpd->device); - } DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n", udata ? "user library" : "kernel", pd); rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata); if (rc) - return ERR_PTR(rc); + return rc; DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n", @@ -2276,20 +2272,10 @@ struct ib_qp *qedr_create_qp(struct ib_pd *ibpd, get_qedr_cq(attrs->recv_cq), attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0); - qp = kzalloc(sizeof(*qp), GFP_KERNEL); - if (!qp) { - DP_ERR(dev, "create qp: failed allocating memory\n"); - return ERR_PTR(-ENOMEM); - } - qedr_set_common_qp_params(dev, qp, pd, attrs); - if (attrs->qp_type == IB_QPT_GSI) { - ibqp = qedr_create_gsi_qp(dev, attrs, qp); - if (IS_ERR(ibqp)) - kfree(qp); - return ibqp; - } + if (attrs->qp_type == IB_QPT_GSI) + return qedr_create_gsi_qp(dev, attrs, qp); if (udata || xrcd) rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs); @@ -2297,7 +2283,7 @@ struct ib_qp *qedr_create_qp(struct ib_pd *ibpd, rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs); if (rc) - goto out_free_qp; + return rc; qp->ibqp.qp_num = qp->qp_id; @@ -2307,14 +2293,11 @@ struct ib_qp *qedr_create_qp(struct ib_pd *ibpd, goto out_free_qp_resources; } - return &qp->ibqp; + return 0; out_free_qp_resources: qedr_free_qp_resources(dev, qp, udata); -out_free_qp: - kfree(qp); - - return ERR_PTR(-EFAULT); + return -EFAULT; } static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state) @@ -2874,8 +2857,6 @@ int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) if (rdma_protocol_iwarp(&dev->ibdev, 1)) qedr_iw_qp_rem_ref(&qp->ibqp); - else - kfree(qp); return 0; } diff --git a/drivers/infiniband/hw/qedr/verbs.h b/drivers/infiniband/hw/qedr/verbs.h index 34ad47515861..031687dafc61 100644 --- a/drivers/infiniband/hw/qedr/verbs.h +++ b/drivers/infiniband/hw/qedr/verbs.h @@ -56,8 +56,8 @@ int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, int qedr_resize_cq(struct ib_cq *, int cqe, struct ib_udata *); int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); -struct ib_qp *qedr_create_qp(struct ib_pd *, struct ib_qp_init_attr *attrs, - struct ib_udata *); +int qedr_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *attrs, + struct ib_udata *udata); int qedr_modify_qp(struct ib_qp *, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata); int qedr_query_qp(struct ib_qp *, struct ib_qp_attr *qp_attr, diff --git a/drivers/infiniband/hw/usnic/usnic_ib_main.c b/drivers/infiniband/hw/usnic/usnic_ib_main.c index c49f9e19d926..228e9a36dad0 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_main.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_main.c @@ -360,6 +360,7 @@ static const struct ib_device_ops usnic_dev_ops = { .reg_user_mr = usnic_ib_reg_mr, INIT_RDMA_OBJ_SIZE(ib_pd, usnic_ib_pd, ibpd), INIT_RDMA_OBJ_SIZE(ib_cq, usnic_ib_cq, ibcq), + INIT_RDMA_OBJ_SIZE(ib_qp, usnic_ib_qp_grp, ibqp), INIT_RDMA_OBJ_SIZE(ib_ucontext, usnic_ib_ucontext, ibucontext), }; diff --git a/drivers/infiniband/hw/usnic/usnic_ib_qp_grp.c b/drivers/infiniband/hw/usnic/usnic_ib_qp_grp.c index 0cdb156e165e..3b60fa9cb58d 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_qp_grp.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_qp_grp.c @@ -665,13 +665,12 @@ static int qp_grp_id_from_flow(struct usnic_ib_qp_grp_flow *qp_flow, return 0; } -struct usnic_ib_qp_grp * -usnic_ib_qp_grp_create(struct usnic_fwd_dev *ufdev, struct usnic_ib_vf *vf, - struct usnic_ib_pd *pd, - struct usnic_vnic_res_spec *res_spec, - struct usnic_transport_spec *transport_spec) +int usnic_ib_qp_grp_create(struct usnic_ib_qp_grp *qp_grp, + struct usnic_fwd_dev *ufdev, struct usnic_ib_vf *vf, + struct usnic_ib_pd *pd, + struct usnic_vnic_res_spec *res_spec, + struct usnic_transport_spec *transport_spec) { - struct usnic_ib_qp_grp *qp_grp; int err; enum usnic_transport_type transport = transport_spec->trans_type; struct usnic_ib_qp_grp_flow *qp_flow; @@ -684,20 +683,15 @@ usnic_ib_qp_grp_create(struct usnic_fwd_dev *ufdev, struct usnic_ib_vf *vf, usnic_err("Spec does not meet minimum req for transport %d\n", transport); log_spec(res_spec); - return ERR_PTR(err); + return err; } - qp_grp = kzalloc(sizeof(*qp_grp), GFP_ATOMIC); - if (!qp_grp) - return NULL; - qp_grp->res_chunk_list = alloc_res_chunk_list(vf->vnic, res_spec, qp_grp); - if (IS_ERR_OR_NULL(qp_grp->res_chunk_list)) { - err = qp_grp->res_chunk_list ? - PTR_ERR(qp_grp->res_chunk_list) : -ENOMEM; - goto out_free_qp_grp; - } + if (IS_ERR_OR_NULL(qp_grp->res_chunk_list)) + return qp_grp->res_chunk_list ? + PTR_ERR(qp_grp->res_chunk_list) : + -ENOMEM; err = qp_grp_and_vf_bind(vf, pd, qp_grp); if (err) @@ -724,7 +718,7 @@ usnic_ib_qp_grp_create(struct usnic_fwd_dev *ufdev, struct usnic_ib_vf *vf, usnic_ib_sysfs_qpn_add(qp_grp); - return qp_grp; + return 0; out_release_flow: release_and_remove_flow(qp_flow); @@ -732,10 +726,7 @@ out_qp_grp_vf_unbind: qp_grp_and_vf_unbind(qp_grp); out_free_res: free_qp_grp_res(qp_grp->res_chunk_list); -out_free_qp_grp: - kfree(qp_grp); - - return ERR_PTR(err); + return err; } void usnic_ib_qp_grp_destroy(struct usnic_ib_qp_grp *qp_grp) @@ -748,7 +739,6 @@ void usnic_ib_qp_grp_destroy(struct usnic_ib_qp_grp *qp_grp) usnic_ib_sysfs_qpn_remove(qp_grp); qp_grp_and_vf_unbind(qp_grp); free_qp_grp_res(qp_grp->res_chunk_list); - kfree(qp_grp); } struct usnic_vnic_res_chunk* diff --git a/drivers/infiniband/hw/usnic/usnic_ib_qp_grp.h b/drivers/infiniband/hw/usnic/usnic_ib_qp_grp.h index a8a2314c9531..62e732be6736 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_qp_grp.h +++ b/drivers/infiniband/hw/usnic/usnic_ib_qp_grp.h @@ -89,11 +89,11 @@ extern const struct usnic_vnic_res_spec min_transport_spec[USNIC_TRANSPORT_MAX]; const char *usnic_ib_qp_grp_state_to_string(enum ib_qp_state state); int usnic_ib_qp_grp_dump_hdr(char *buf, int buf_sz); int usnic_ib_qp_grp_dump_rows(void *obj, char *buf, int buf_sz); -struct usnic_ib_qp_grp * -usnic_ib_qp_grp_create(struct usnic_fwd_dev *ufdev, struct usnic_ib_vf *vf, - struct usnic_ib_pd *pd, - struct usnic_vnic_res_spec *res_spec, - struct usnic_transport_spec *trans_spec); +int usnic_ib_qp_grp_create(struct usnic_ib_qp_grp *qp, + struct usnic_fwd_dev *ufdev, struct usnic_ib_vf *vf, + struct usnic_ib_pd *pd, + struct usnic_vnic_res_spec *res_spec, + struct usnic_transport_spec *trans_spec); void usnic_ib_qp_grp_destroy(struct usnic_ib_qp_grp *qp_grp); int usnic_ib_qp_grp_modify(struct usnic_ib_qp_grp *qp_grp, enum ib_qp_state new_state, diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c index 57d210ca855a..06a4e9d4545d 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c @@ -168,30 +168,31 @@ static int usnic_ib_fill_create_qp_resp(struct usnic_ib_qp_grp *qp_grp, return 0; } -static struct usnic_ib_qp_grp* -find_free_vf_and_create_qp_grp(struct usnic_ib_dev *us_ibdev, - struct usnic_ib_pd *pd, - struct usnic_transport_spec *trans_spec, - struct usnic_vnic_res_spec *res_spec) +static int +find_free_vf_and_create_qp_grp(struct ib_qp *qp, + struct usnic_transport_spec *trans_spec, + struct usnic_vnic_res_spec *res_spec) { + struct usnic_ib_dev *us_ibdev = to_usdev(qp->device); + struct usnic_ib_pd *pd = to_upd(qp->pd); struct usnic_ib_vf *vf; struct usnic_vnic *vnic; - struct usnic_ib_qp_grp *qp_grp; + struct usnic_ib_qp_grp *qp_grp = to_uqp_grp(qp); struct device *dev, **dev_list; - int i; + int i, ret; BUG_ON(!mutex_is_locked(&us_ibdev->usdev_lock)); if (list_empty(&us_ibdev->vf_dev_list)) { usnic_info("No vfs to allocate\n"); - return NULL; + return -ENOMEM; } if (usnic_ib_share_vf) { /* Try to find resouces on a used vf which is in pd */ dev_list = usnic_uiom_get_dev_list(pd->umem_pd); if (IS_ERR(dev_list)) - return ERR_CAST(dev_list); + return PTR_ERR(dev_list); for (i = 0; dev_list[i]; i++) { dev = dev_list[i]; vf = dev_get_drvdata(dev); @@ -202,10 +203,10 @@ find_free_vf_and_create_qp_grp(struct usnic_ib_dev *us_ibdev, dev_name(&us_ibdev->ib_dev.dev), pci_name(usnic_vnic_get_pdev( vnic))); - qp_grp = usnic_ib_qp_grp_create(us_ibdev->ufdev, - vf, pd, - res_spec, - trans_spec); + ret = usnic_ib_qp_grp_create(qp_grp, + us_ibdev->ufdev, + vf, pd, res_spec, + trans_spec); spin_unlock(&vf->lock); goto qp_grp_check; @@ -223,9 +224,9 @@ find_free_vf_and_create_qp_grp(struct usnic_ib_dev *us_ibdev, vnic = vf->vnic; if (vf->qp_grp_ref_cnt == 0 && usnic_vnic_check_room(vnic, res_spec) == 0) { - qp_grp = usnic_ib_qp_grp_create(us_ibdev->ufdev, vf, - pd, res_spec, - trans_spec); + ret = usnic_ib_qp_grp_create(qp_grp, us_ibdev->ufdev, + vf, pd, res_spec, + trans_spec); spin_unlock(&vf->lock); goto qp_grp_check; @@ -235,16 +236,15 @@ find_free_vf_and_create_qp_grp(struct usnic_ib_dev *us_ibdev, usnic_info("No free qp grp found on %s\n", dev_name(&us_ibdev->ib_dev.dev)); - return ERR_PTR(-ENOMEM); + return -ENOMEM; qp_grp_check: - if (IS_ERR_OR_NULL(qp_grp)) { + if (ret) { usnic_err("Failed to allocate qp_grp\n"); if (usnic_ib_share_vf) usnic_uiom_free_dev_list(dev_list); - return ERR_PTR(qp_grp ? PTR_ERR(qp_grp) : -ENOMEM); } - return qp_grp; + return ret; } static void qp_grp_destroy(struct usnic_ib_qp_grp *qp_grp) @@ -458,13 +458,12 @@ int usnic_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) return 0; } -struct ib_qp *usnic_ib_create_qp(struct ib_pd *pd, - struct ib_qp_init_attr *init_attr, - struct ib_udata *udata) +int usnic_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) { int err; struct usnic_ib_dev *us_ibdev; - struct usnic_ib_qp_grp *qp_grp; + struct usnic_ib_qp_grp *qp_grp = to_uqp_grp(ibqp); struct usnic_ib_ucontext *ucontext = rdma_udata_to_drv_context( udata, struct usnic_ib_ucontext, ibucontext); int cq_cnt; @@ -474,29 +473,29 @@ struct ib_qp *usnic_ib_create_qp(struct ib_pd *pd, usnic_dbg("\n"); - us_ibdev = to_usdev(pd->device); + us_ibdev = to_usdev(ibqp->device); if (init_attr->create_flags) - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; err = ib_copy_from_udata(&cmd, udata, sizeof(cmd)); if (err) { usnic_err("%s: cannot copy udata for create_qp\n", dev_name(&us_ibdev->ib_dev.dev)); - return ERR_PTR(-EINVAL); + return -EINVAL; } err = create_qp_validate_user_data(cmd); if (err) { usnic_err("%s: Failed to validate user data\n", dev_name(&us_ibdev->ib_dev.dev)); - return ERR_PTR(-EINVAL); + return -EINVAL; } if (init_attr->qp_type != IB_QPT_UD) { usnic_err("%s asked to make a non-UD QP: %d\n", dev_name(&us_ibdev->ib_dev.dev), init_attr->qp_type); - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; } trans_spec = cmd.spec; @@ -504,13 +503,9 @@ struct ib_qp *usnic_ib_create_qp(struct ib_pd *pd, cq_cnt = (init_attr->send_cq == init_attr->recv_cq) ? 1 : 2; res_spec = min_transport_spec[trans_spec.trans_type]; usnic_vnic_res_spec_update(&res_spec, USNIC_VNIC_RES_TYPE_CQ, cq_cnt); - qp_grp = find_free_vf_and_create_qp_grp(us_ibdev, to_upd(pd), - &trans_spec, - &res_spec); - if (IS_ERR_OR_NULL(qp_grp)) { - err = qp_grp ? PTR_ERR(qp_grp) : -ENOMEM; + err = find_free_vf_and_create_qp_grp(ibqp, &trans_spec, &res_spec); + if (err) goto out_release_mutex; - } err = usnic_ib_fill_create_qp_resp(qp_grp, udata); if (err) { @@ -522,13 +517,13 @@ struct ib_qp *usnic_ib_create_qp(struct ib_pd *pd, list_add_tail(&qp_grp->link, &ucontext->qp_grp_list); usnic_ib_log_vf(qp_grp->vf); mutex_unlock(&us_ibdev->usdev_lock); - return &qp_grp->ibqp; + return 0; out_release_qp_grp: qp_grp_destroy(qp_grp); out_release_mutex: mutex_unlock(&us_ibdev->usdev_lock); - return ERR_PTR(err); + return err; } int usnic_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.h b/drivers/infiniband/hw/usnic/usnic_ib_verbs.h index 6b82d0f2d184..6ca9ee0dddbe 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.h +++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.h @@ -50,9 +50,8 @@ int usnic_ib_query_gid(struct ib_device *ibdev, u32 port, int index, union ib_gid *gid); int usnic_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); int usnic_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata); -struct ib_qp *usnic_ib_create_qp(struct ib_pd *pd, - struct ib_qp_init_attr *init_attr, - struct ib_udata *udata); +int usnic_ib_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata); int usnic_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata); int usnic_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata); diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c index 8ed8bc24c69f..b39175837d58 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c @@ -185,6 +185,7 @@ static const struct ib_device_ops pvrdma_dev_ops = { INIT_RDMA_OBJ_SIZE(ib_ah, pvrdma_ah, ibah), INIT_RDMA_OBJ_SIZE(ib_cq, pvrdma_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, pvrdma_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_qp, pvrdma_qp, ibqp), INIT_RDMA_OBJ_SIZE(ib_ucontext, pvrdma_ucontext, ibucontext), }; diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c index 67769b715126..f83cd4a9d992 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c @@ -182,18 +182,17 @@ static int pvrdma_set_sq_size(struct pvrdma_dev *dev, struct ib_qp_cap *req_cap, /** * pvrdma_create_qp - create queue pair - * @pd: protection domain + * @ibqp: queue pair * @init_attr: queue pair attributes * @udata: user data * - * @return: the ib_qp pointer on success, otherwise returns an errno. + * @return: the 0 on success, otherwise returns an errno. */ -struct ib_qp *pvrdma_create_qp(struct ib_pd *pd, - struct ib_qp_init_attr *init_attr, - struct ib_udata *udata) +int pvrdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) { - struct pvrdma_qp *qp = NULL; - struct pvrdma_dev *dev = to_vdev(pd->device); + struct pvrdma_qp *qp = to_vqp(ibqp); + struct pvrdma_dev *dev = to_vdev(ibqp->device); union pvrdma_cmd_req req; union pvrdma_cmd_resp rsp; struct pvrdma_cmd_create_qp *cmd = &req.create_qp; @@ -209,7 +208,7 @@ struct ib_qp *pvrdma_create_qp(struct ib_pd *pd, dev_warn(&dev->pdev->dev, "invalid create queuepair flags %#x\n", init_attr->create_flags); - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; } if (init_attr->qp_type != IB_QPT_RC && @@ -217,22 +216,22 @@ struct ib_qp *pvrdma_create_qp(struct ib_pd *pd, init_attr->qp_type != IB_QPT_GSI) { dev_warn(&dev->pdev->dev, "queuepair type %d not supported\n", init_attr->qp_type); - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; } if (is_srq && !dev->dsr->caps.max_srq) { dev_warn(&dev->pdev->dev, "SRQs not supported by device\n"); - return ERR_PTR(-EINVAL); + return -EINVAL; } if (!atomic_add_unless(&dev->num_qps, 1, dev->dsr->caps.max_qp)) - return ERR_PTR(-ENOMEM); + return -ENOMEM; switch (init_attr->qp_type) { case IB_QPT_GSI: if (init_attr->port_num == 0 || - init_attr->port_num > pd->device->phys_port_cnt) { + init_attr->port_num > ibqp->device->phys_port_cnt) { dev_warn(&dev->pdev->dev, "invalid queuepair attrs\n"); ret = -EINVAL; goto err_qp; @@ -240,12 +239,6 @@ struct ib_qp *pvrdma_create_qp(struct ib_pd *pd, fallthrough; case IB_QPT_RC: case IB_QPT_UD: - qp = kzalloc(sizeof(*qp), GFP_KERNEL); - if (!qp) { - ret = -ENOMEM; - goto err_qp; - } - spin_lock_init(&qp->sq.lock); spin_lock_init(&qp->rq.lock); mutex_init(&qp->mutex); @@ -275,9 +268,9 @@ struct ib_qp *pvrdma_create_qp(struct ib_pd *pd, if (!is_srq) { /* set qp->sq.wqe_cnt, shift, buf_size.. */ - qp->rumem = - ib_umem_get(pd->device, ucmd.rbuf_addr, - ucmd.rbuf_size, 0); + qp->rumem = ib_umem_get(ibqp->device, + ucmd.rbuf_addr, + ucmd.rbuf_size, 0); if (IS_ERR(qp->rumem)) { ret = PTR_ERR(qp->rumem); goto err_qp; @@ -288,7 +281,7 @@ struct ib_qp *pvrdma_create_qp(struct ib_pd *pd, qp->srq = to_vsrq(init_attr->srq); } - qp->sumem = ib_umem_get(pd->device, ucmd.sbuf_addr, + qp->sumem = ib_umem_get(ibqp->device, ucmd.sbuf_addr, ucmd.sbuf_size, 0); if (IS_ERR(qp->sumem)) { if (!is_srq) @@ -306,12 +299,12 @@ struct ib_qp *pvrdma_create_qp(struct ib_pd *pd, qp->npages_recv = 0; qp->npages = qp->npages_send + qp->npages_recv; } else { - ret = pvrdma_set_sq_size(to_vdev(pd->device), + ret = pvrdma_set_sq_size(to_vdev(ibqp->device), &init_attr->cap, qp); if (ret) goto err_qp; - ret = pvrdma_set_rq_size(to_vdev(pd->device), + ret = pvrdma_set_rq_size(to_vdev(ibqp->device), &init_attr->cap, qp); if (ret) goto err_qp; @@ -362,7 +355,7 @@ struct ib_qp *pvrdma_create_qp(struct ib_pd *pd, memset(cmd, 0, sizeof(*cmd)); cmd->hdr.cmd = PVRDMA_CMD_CREATE_QP; - cmd->pd_handle = to_vpd(pd)->pd_handle; + cmd->pd_handle = to_vpd(ibqp->pd)->pd_handle; cmd->send_cq_handle = to_vcq(init_attr->send_cq)->cq_handle; cmd->recv_cq_handle = to_vcq(init_attr->recv_cq)->cq_handle; if (is_srq) @@ -418,11 +411,11 @@ struct ib_qp *pvrdma_create_qp(struct ib_pd *pd, dev_warn(&dev->pdev->dev, "failed to copy back udata\n"); __pvrdma_destroy_qp(dev, qp); - return ERR_PTR(-EINVAL); + return -EINVAL; } } - return &qp->ibqp; + return 0; err_pdir: pvrdma_page_dir_cleanup(dev, &qp->pdir); @@ -430,10 +423,8 @@ err_umem: ib_umem_release(qp->rumem); ib_umem_release(qp->sumem); err_qp: - kfree(qp); atomic_dec(&dev->num_qps); - - return ERR_PTR(ret); + return ret; } static void _pvrdma_free_qp(struct pvrdma_qp *qp) @@ -454,8 +445,6 @@ static void _pvrdma_free_qp(struct pvrdma_qp *qp) pvrdma_page_dir_cleanup(dev, &qp->pdir); - kfree(qp); - atomic_dec(&dev->num_qps); } diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h index 544b94d97c3a..78807b23d831 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h @@ -390,9 +390,8 @@ int pvrdma_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, int pvrdma_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr); int pvrdma_destroy_srq(struct ib_srq *srq, struct ib_udata *udata); -struct ib_qp *pvrdma_create_qp(struct ib_pd *pd, - struct ib_qp_init_attr *init_attr, - struct ib_udata *udata); +int pvrdma_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata); int pvrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata); int pvrdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c index 14900860985c..da2d94a5a9c2 100644 --- a/drivers/infiniband/sw/rdmavt/qp.c +++ b/drivers/infiniband/sw/rdmavt/qp.c @@ -1058,7 +1058,7 @@ static int alloc_ud_wq_attr(struct rvt_qp *qp, int node) /** * rvt_create_qp - create a queue pair for a device - * @ibpd: the protection domain who's device we create the queue pair for + * @ibqp: the queue pair * @init_attr: the attributes of the queue pair * @udata: user data for libibverbs.so * @@ -1066,47 +1066,45 @@ static int alloc_ud_wq_attr(struct rvt_qp *qp, int node) * unique idea of what queue pair numbers mean. For instance there is a reserved * range for PSM. * - * Return: the queue pair on success, otherwise returns an errno. + * Return: 0 on success, otherwise returns an errno. * * Called by the ib_create_qp() core verbs function. */ -struct ib_qp *rvt_create_qp(struct ib_pd *ibpd, - struct ib_qp_init_attr *init_attr, - struct ib_udata *udata) +int rvt_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) { - struct rvt_qp *qp; - int err; + struct rvt_qp *qp = ibqp_to_rvtqp(ibqp); + int ret = -ENOMEM; struct rvt_swqe *swq = NULL; size_t sz; size_t sg_list_sz = 0; - struct ib_qp *ret = ERR_PTR(-ENOMEM); - struct rvt_dev_info *rdi = ib_to_rvt(ibpd->device); + struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device); void *priv = NULL; size_t sqsize; u8 exclude_prefix = 0; if (!rdi) - return ERR_PTR(-EINVAL); + return -EINVAL; if (init_attr->create_flags & ~IB_QP_CREATE_NETDEV_USE) - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; if (init_attr->cap.max_send_sge > rdi->dparms.props.max_send_sge || init_attr->cap.max_send_wr > rdi->dparms.props.max_qp_wr) - return ERR_PTR(-EINVAL); + return -EINVAL; /* Check receive queue parameters if no SRQ is specified. */ if (!init_attr->srq) { if (init_attr->cap.max_recv_sge > rdi->dparms.props.max_recv_sge || init_attr->cap.max_recv_wr > rdi->dparms.props.max_qp_wr) - return ERR_PTR(-EINVAL); + return -EINVAL; if (init_attr->cap.max_send_sge + init_attr->cap.max_send_wr + init_attr->cap.max_recv_sge + init_attr->cap.max_recv_wr == 0) - return ERR_PTR(-EINVAL); + return -EINVAL; } sqsize = init_attr->cap.max_send_wr + 1 + @@ -1115,8 +1113,8 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd, case IB_QPT_SMI: case IB_QPT_GSI: if (init_attr->port_num == 0 || - init_attr->port_num > ibpd->device->phys_port_cnt) - return ERR_PTR(-EINVAL); + init_attr->port_num > ibqp->device->phys_port_cnt) + return -EINVAL; fallthrough; case IB_QPT_UC: case IB_QPT_RC: @@ -1124,7 +1122,7 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd, sz = struct_size(swq, sg_list, init_attr->cap.max_send_sge); swq = vzalloc_node(array_size(sz, sqsize), rdi->dparms.node); if (!swq) - return ERR_PTR(-ENOMEM); + return -ENOMEM; if (init_attr->srq) { struct rvt_srq *srq = ibsrq_to_rvtsrq(init_attr->srq); @@ -1135,9 +1133,6 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd, } else if (init_attr->cap.max_recv_sge > 1) sg_list_sz = sizeof(*qp->r_sg_list) * (init_attr->cap.max_recv_sge - 1); - qp = kzalloc_node(sizeof(*qp), GFP_KERNEL, rdi->dparms.node); - if (!qp) - goto bail_swq; qp->r_sg_list = kzalloc_node(sg_list_sz, GFP_KERNEL, rdi->dparms.node); if (!qp->r_sg_list) @@ -1166,7 +1161,7 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd, */ priv = rdi->driver_f.qp_priv_alloc(rdi, qp); if (IS_ERR(priv)) { - ret = priv; + ret = PTR_ERR(priv); goto bail_qp; } qp->priv = priv; @@ -1180,12 +1175,10 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd, qp->r_rq.max_sge = init_attr->cap.max_recv_sge; sz = (sizeof(struct ib_sge) * qp->r_rq.max_sge) + sizeof(struct rvt_rwqe); - err = rvt_alloc_rq(&qp->r_rq, qp->r_rq.size * sz, + ret = rvt_alloc_rq(&qp->r_rq, qp->r_rq.size * sz, rdi->dparms.node, udata); - if (err) { - ret = ERR_PTR(err); + if (ret) goto bail_driver_priv; - } } /* @@ -1206,40 +1199,35 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd, qp->s_max_sge = init_attr->cap.max_send_sge; if (init_attr->sq_sig_type == IB_SIGNAL_REQ_WR) qp->s_flags = RVT_S_SIGNAL_REQ_WR; - err = alloc_ud_wq_attr(qp, rdi->dparms.node); - if (err) { - ret = (ERR_PTR(err)); + ret = alloc_ud_wq_attr(qp, rdi->dparms.node); + if (ret) goto bail_rq_rvt; - } if (init_attr->create_flags & IB_QP_CREATE_NETDEV_USE) exclude_prefix = RVT_AIP_QP_PREFIX; - err = alloc_qpn(rdi, &rdi->qp_dev->qpn_table, + ret = alloc_qpn(rdi, &rdi->qp_dev->qpn_table, init_attr->qp_type, init_attr->port_num, exclude_prefix); - if (err < 0) { - ret = ERR_PTR(err); + if (ret < 0) goto bail_rq_wq; - } - qp->ibqp.qp_num = err; + + qp->ibqp.qp_num = ret; if (init_attr->create_flags & IB_QP_CREATE_NETDEV_USE) qp->ibqp.qp_num |= RVT_AIP_QP_BASE; qp->port_num = init_attr->port_num; rvt_init_qp(rdi, qp, init_attr->qp_type); if (rdi->driver_f.qp_priv_init) { - err = rdi->driver_f.qp_priv_init(rdi, qp, init_attr); - if (err) { - ret = ERR_PTR(err); + ret = rdi->driver_f.qp_priv_init(rdi, qp, init_attr); + if (ret) goto bail_rq_wq; - } } break; default: /* Don't support raw QPs */ - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; } init_attr->cap.max_inline_data = 0; @@ -1252,28 +1240,24 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd, if (!qp->r_rq.wq) { __u64 offset = 0; - err = ib_copy_to_udata(udata, &offset, + ret = ib_copy_to_udata(udata, &offset, sizeof(offset)); - if (err) { - ret = ERR_PTR(err); + if (ret) goto bail_qpn; - } } else { u32 s = sizeof(struct rvt_rwq) + qp->r_rq.size * sz; qp->ip = rvt_create_mmap_info(rdi, s, udata, qp->r_rq.wq); if (IS_ERR(qp->ip)) { - ret = ERR_CAST(qp->ip); + ret = PTR_ERR(qp->ip); goto bail_qpn; } - err = ib_copy_to_udata(udata, &qp->ip->offset, + ret = ib_copy_to_udata(udata, &qp->ip->offset, sizeof(qp->ip->offset)); - if (err) { - ret = ERR_PTR(err); + if (ret) goto bail_ip; - } } qp->pid = current->pid; } @@ -1281,7 +1265,7 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd, spin_lock(&rdi->n_qps_lock); if (rdi->n_qps_allocated == rdi->dparms.props.max_qp) { spin_unlock(&rdi->n_qps_lock); - ret = ERR_PTR(-ENOMEM); + ret = ENOMEM; goto bail_ip; } @@ -1307,9 +1291,7 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd, spin_unlock_irq(&rdi->pending_lock); } - ret = &qp->ibqp; - - return ret; + return 0; bail_ip: if (qp->ip) @@ -1330,11 +1312,7 @@ bail_driver_priv: bail_qp: kfree(qp->s_ack_queue); kfree(qp->r_sg_list); - kfree(qp); - -bail_swq: vfree(swq); - return ret; } @@ -1769,7 +1747,6 @@ int rvt_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) rdma_destroy_ah_attr(&qp->alt_ah_attr); free_ud_wq_attr(qp); vfree(qp->s_wq); - kfree(qp); return 0; } diff --git a/drivers/infiniband/sw/rdmavt/qp.h b/drivers/infiniband/sw/rdmavt/qp.h index 2cdba1283bf6..bceb77c28c71 100644 --- a/drivers/infiniband/sw/rdmavt/qp.h +++ b/drivers/infiniband/sw/rdmavt/qp.h @@ -52,9 +52,8 @@ int rvt_driver_qp_init(struct rvt_dev_info *rdi); void rvt_qp_exit(struct rvt_dev_info *rdi); -struct ib_qp *rvt_create_qp(struct ib_pd *ibpd, - struct ib_qp_init_attr *init_attr, - struct ib_udata *udata); +int rvt_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, + struct ib_udata *udata); int rvt_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata); int rvt_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata); diff --git a/drivers/infiniband/sw/rdmavt/vt.c b/drivers/infiniband/sw/rdmavt/vt.c index ac17209816cd..d4526f38427e 100644 --- a/drivers/infiniband/sw/rdmavt/vt.c +++ b/drivers/infiniband/sw/rdmavt/vt.c @@ -131,6 +131,13 @@ static int rvt_query_device(struct ib_device *ibdev, return 0; } +static int rvt_get_numa_node(struct ib_device *ibdev) +{ + struct rvt_dev_info *rdi = ib_to_rvt(ibdev); + + return rdi->dparms.node; +} + static int rvt_modify_device(struct ib_device *device, int device_modify_mask, struct ib_device_modify *device_modify) @@ -380,6 +387,7 @@ static const struct ib_device_ops rvt_dev_ops = { .destroy_srq = rvt_destroy_srq, .detach_mcast = rvt_detach_mcast, .get_dma_mr = rvt_get_dma_mr, + .get_numa_node = rvt_get_numa_node, .get_port_immutable = rvt_get_port_immutable, .map_mr_sg = rvt_map_mr_sg, .mmap = rvt_mmap, @@ -406,6 +414,7 @@ static const struct ib_device_ops rvt_dev_ops = { INIT_RDMA_OBJ_SIZE(ib_ah, rvt_ah, ibah), INIT_RDMA_OBJ_SIZE(ib_cq, rvt_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, rvt_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_qp, rvt_qp, ibqp), INIT_RDMA_OBJ_SIZE(ib_srq, rvt_srq, ibsrq), INIT_RDMA_OBJ_SIZE(ib_ucontext, rvt_ucontext, ibucontext), }; diff --git a/drivers/infiniband/sw/rxe/rxe_pool.c b/drivers/infiniband/sw/rxe/rxe_pool.c index 0b8e7c6255a2..ffa8420b4765 100644 --- a/drivers/infiniband/sw/rxe/rxe_pool.c +++ b/drivers/infiniband/sw/rxe/rxe_pool.c @@ -41,7 +41,7 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = { .size = sizeof(struct rxe_qp), .elem_offset = offsetof(struct rxe_qp, pelem), .cleanup = rxe_qp_cleanup, - .flags = RXE_POOL_INDEX, + .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC, .min_index = RXE_MIN_QP_INDEX, .max_index = RXE_MAX_QP_INDEX, }, diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c index f7b1a1f64c13..267b5a9c345d 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.c +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c @@ -391,59 +391,52 @@ static int rxe_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, return err; } -static struct ib_qp *rxe_create_qp(struct ib_pd *ibpd, - struct ib_qp_init_attr *init, - struct ib_udata *udata) +static int rxe_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init, + struct ib_udata *udata) { int err; - struct rxe_dev *rxe = to_rdev(ibpd->device); - struct rxe_pd *pd = to_rpd(ibpd); - struct rxe_qp *qp; + struct rxe_dev *rxe = to_rdev(ibqp->device); + struct rxe_pd *pd = to_rpd(ibqp->pd); + struct rxe_qp *qp = to_rqp(ibqp); struct rxe_create_qp_resp __user *uresp = NULL; if (udata) { if (udata->outlen < sizeof(*uresp)) - return ERR_PTR(-EINVAL); + return -EINVAL; uresp = udata->outbuf; } if (init->create_flags) - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; err = rxe_qp_chk_init(rxe, init); if (err) - goto err1; - - qp = rxe_alloc(&rxe->qp_pool); - if (!qp) { - err = -ENOMEM; - goto err1; - } + return err; if (udata) { - if (udata->inlen) { - err = -EINVAL; - goto err2; - } + if (udata->inlen) + return -EINVAL; + qp->is_user = true; } else { qp->is_user = false; } - rxe_add_index(qp); + err = rxe_add_to_pool(&rxe->qp_pool, qp); + if (err) + return err; - err = rxe_qp_from_init(rxe, qp, pd, init, uresp, ibpd, udata); + rxe_add_index(qp); + err = rxe_qp_from_init(rxe, qp, pd, init, uresp, ibqp->pd, udata); if (err) - goto err3; + goto qp_init; - return &qp->ibqp; + return 0; -err3: +qp_init: rxe_drop_index(qp); -err2: rxe_drop_ref(qp); -err1: - return ERR_PTR(err); + return err; } static int rxe_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, @@ -1145,6 +1138,7 @@ static const struct ib_device_ops rxe_dev_ops = { INIT_RDMA_OBJ_SIZE(ib_ah, rxe_ah, ibah), INIT_RDMA_OBJ_SIZE(ib_cq, rxe_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, rxe_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_qp, rxe_qp, ibqp), INIT_RDMA_OBJ_SIZE(ib_srq, rxe_srq, ibsrq), INIT_RDMA_OBJ_SIZE(ib_ucontext, rxe_ucontext, ibuc), INIT_RDMA_OBJ_SIZE(ib_mw, rxe_mw, ibmw), diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h index 959a3260fcab..ac2a2148027f 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.h +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h @@ -210,8 +210,8 @@ struct rxe_resp_info { }; struct rxe_qp { - struct rxe_pool_entry pelem; struct ib_qp ibqp; + struct rxe_pool_entry pelem; struct ib_qp_attr attr; unsigned int valid; unsigned int mtu; diff --git a/drivers/infiniband/sw/siw/siw_main.c b/drivers/infiniband/sw/siw/siw_main.c index cf55326f2ab4..9093e6a80b26 100644 --- a/drivers/infiniband/sw/siw/siw_main.c +++ b/drivers/infiniband/sw/siw/siw_main.c @@ -297,6 +297,7 @@ static const struct ib_device_ops siw_device_ops = { INIT_RDMA_OBJ_SIZE(ib_cq, siw_cq, base_cq), INIT_RDMA_OBJ_SIZE(ib_pd, siw_pd, base_pd), + INIT_RDMA_OBJ_SIZE(ib_qp, siw_qp, base_qp), INIT_RDMA_OBJ_SIZE(ib_srq, siw_srq, base_srq), INIT_RDMA_OBJ_SIZE(ib_ucontext, siw_ucontext, base_ucontext), }; diff --git a/drivers/infiniband/sw/siw/siw_qp.c b/drivers/infiniband/sw/siw/siw_qp.c index ddb2e66f9f13..7e01f2438afc 100644 --- a/drivers/infiniband/sw/siw/siw_qp.c +++ b/drivers/infiniband/sw/siw/siw_qp.c @@ -1344,6 +1344,4 @@ void siw_free_qp(struct kref *ref) siw_put_tx_cpu(qp->tx_cpu); atomic_dec(&sdev->num_qp); - siw_dbg_qp(qp, "free QP\n"); - kfree_rcu(qp, rcu); } diff --git a/drivers/infiniband/sw/siw/siw_verbs.c b/drivers/infiniband/sw/siw/siw_verbs.c index 3f175f220a22..1b36350601fa 100644 --- a/drivers/infiniband/sw/siw/siw_verbs.c +++ b/drivers/infiniband/sw/siw/siw_verbs.c @@ -285,16 +285,16 @@ siw_mmap_entry_insert(struct siw_ucontext *uctx, * * Create QP of requested size on given device. * - * @pd: Protection Domain + * @qp: Queue pait * @attrs: Initial QP attributes. * @udata: used to provide QP ID, SQ and RQ size back to user. */ -struct ib_qp *siw_create_qp(struct ib_pd *pd, - struct ib_qp_init_attr *attrs, - struct ib_udata *udata) +int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, + struct ib_udata *udata) { - struct siw_qp *qp = NULL; + struct ib_pd *pd = ibqp->pd; + struct siw_qp *qp = to_siw_qp(ibqp); struct ib_device *base_dev = pd->device; struct siw_device *sdev = to_siw_dev(base_dev); struct siw_ucontext *uctx = @@ -307,17 +307,16 @@ struct ib_qp *siw_create_qp(struct ib_pd *pd, siw_dbg(base_dev, "create new QP\n"); if (attrs->create_flags) - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; if (atomic_inc_return(&sdev->num_qp) > SIW_MAX_QP) { siw_dbg(base_dev, "too many QP's\n"); - rv = -ENOMEM; - goto err_out; + return -ENOMEM; } if (attrs->qp_type != IB_QPT_RC) { siw_dbg(base_dev, "only RC QP's supported\n"); rv = -EOPNOTSUPP; - goto err_out; + goto err_atomic; } if ((attrs->cap.max_send_wr > SIW_MAX_QP_WR) || (attrs->cap.max_recv_wr > SIW_MAX_QP_WR) || @@ -325,13 +324,13 @@ struct ib_qp *siw_create_qp(struct ib_pd *pd, (attrs->cap.max_recv_sge > SIW_MAX_SGE)) { siw_dbg(base_dev, "QP size error\n"); rv = -EINVAL; - goto err_out; + goto err_atomic; } if (attrs->cap.max_inline_data > SIW_MAX_INLINE) { siw_dbg(base_dev, "max inline send: %d > %d\n", attrs->cap.max_inline_data, (int)SIW_MAX_INLINE); rv = -EINVAL; - goto err_out; + goto err_atomic; } /* * NOTE: we allow for zero element SQ and RQ WQE's SGL's @@ -340,19 +339,15 @@ struct ib_qp *siw_create_qp(struct ib_pd *pd, if (attrs->cap.max_send_wr + attrs->cap.max_recv_wr == 0) { siw_dbg(base_dev, "QP must have send or receive queue\n"); rv = -EINVAL; - goto err_out; + goto err_atomic; } if (!attrs->send_cq || (!attrs->recv_cq && !attrs->srq)) { siw_dbg(base_dev, "send CQ or receive CQ invalid\n"); rv = -EINVAL; - goto err_out; - } - qp = kzalloc(sizeof(*qp), GFP_KERNEL); - if (!qp) { - rv = -ENOMEM; - goto err_out; + goto err_atomic; } + init_rwsem(&qp->state_lock); spin_lock_init(&qp->sq_lock); spin_lock_init(&qp->rq_lock); @@ -360,7 +355,7 @@ struct ib_qp *siw_create_qp(struct ib_pd *pd, rv = siw_qp_add(sdev, qp); if (rv) - goto err_out; + goto err_atomic; num_sqe = attrs->cap.max_send_wr; num_rqe = attrs->cap.max_recv_wr; @@ -482,23 +477,20 @@ struct ib_qp *siw_create_qp(struct ib_pd *pd, list_add_tail(&qp->devq, &sdev->qp_list); spin_unlock_irqrestore(&sdev->lock, flags); - return &qp->base_qp; + return 0; err_out_xa: xa_erase(&sdev->qp_xa, qp_id(qp)); -err_out: - if (qp) { - if (uctx) { - rdma_user_mmap_entry_remove(qp->sq_entry); - rdma_user_mmap_entry_remove(qp->rq_entry); - } - vfree(qp->sendq); - vfree(qp->recvq); - kfree(qp); + if (uctx) { + rdma_user_mmap_entry_remove(qp->sq_entry); + rdma_user_mmap_entry_remove(qp->rq_entry); } - atomic_dec(&sdev->num_qp); + vfree(qp->sendq); + vfree(qp->recvq); - return ERR_PTR(rv); +err_atomic: + atomic_dec(&sdev->num_qp); + return rv; } /* diff --git a/drivers/infiniband/sw/siw/siw_verbs.h b/drivers/infiniband/sw/siw/siw_verbs.h index 67ac08886a70..09964234f8d3 100644 --- a/drivers/infiniband/sw/siw/siw_verbs.h +++ b/drivers/infiniband/sw/siw/siw_verbs.h @@ -50,9 +50,8 @@ int siw_query_gid(struct ib_device *base_dev, u32 port, int idx, union ib_gid *gid); int siw_alloc_pd(struct ib_pd *base_pd, struct ib_udata *udata); int siw_dealloc_pd(struct ib_pd *base_pd, struct ib_udata *udata); -struct ib_qp *siw_create_qp(struct ib_pd *base_pd, - struct ib_qp_init_attr *attr, - struct ib_udata *udata); +int siw_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *attr, + struct ib_udata *udata); int siw_query_qp(struct ib_qp *base_qp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr); int siw_verbs_modify_qp(struct ib_qp *base_qp, struct ib_qp_attr *attr, diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 371df1c80aeb..6737582e9e2e 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -2268,8 +2268,13 @@ struct iw_cm_conn_param; !__same_type(((struct drv_struct *)NULL)->member, \ struct ib_struct))) -#define rdma_zalloc_drv_obj_gfp(ib_dev, ib_type, gfp) \ - ((struct ib_type *)kzalloc(ib_dev->ops.size_##ib_type, gfp)) +#define rdma_zalloc_drv_obj_gfp(ib_dev, ib_type, gfp) \ + ((struct ib_type *)rdma_zalloc_obj(ib_dev, ib_dev->ops.size_##ib_type, \ + gfp, false)) + +#define rdma_zalloc_drv_obj_numa(ib_dev, ib_type) \ + ((struct ib_type *)rdma_zalloc_obj(ib_dev, ib_dev->ops.size_##ib_type, \ + GFP_KERNEL, true)) #define rdma_zalloc_drv_obj(ib_dev, ib_type) \ rdma_zalloc_drv_obj_gfp(ib_dev, ib_type, GFP_KERNEL) @@ -2435,9 +2440,8 @@ struct ib_device_ops { struct ib_udata *udata); int (*query_srq)(struct ib_srq *srq, struct ib_srq_attr *srq_attr); int (*destroy_srq)(struct ib_srq *srq, struct ib_udata *udata); - struct ib_qp *(*create_qp)(struct ib_pd *pd, - struct ib_qp_init_attr *qp_init_attr, - struct ib_udata *udata); + int (*create_qp)(struct ib_qp *qp, struct ib_qp_init_attr *qp_init_attr, + struct ib_udata *udata); int (*modify_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_udata *udata); int (*query_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr, @@ -2635,11 +2639,18 @@ struct ib_device_ops { int (*query_ucontext)(struct ib_ucontext *context, struct uverbs_attr_bundle *attrs); + /* + * Provide NUMA node. This API exists for rdmavt/hfi1 only. + * Everyone else relies on Linux memory management model. + */ + int (*get_numa_node)(struct ib_device *dev); + DECLARE_RDMA_OBJ_SIZE(ib_ah); DECLARE_RDMA_OBJ_SIZE(ib_counters); DECLARE_RDMA_OBJ_SIZE(ib_cq); DECLARE_RDMA_OBJ_SIZE(ib_mw); DECLARE_RDMA_OBJ_SIZE(ib_pd); + DECLARE_RDMA_OBJ_SIZE(ib_qp); DECLARE_RDMA_OBJ_SIZE(ib_rwq_ind_table); DECLARE_RDMA_OBJ_SIZE(ib_srq); DECLARE_RDMA_OBJ_SIZE(ib_ucontext); @@ -2746,6 +2757,15 @@ struct ib_device { u32 lag_flags; }; +static inline void *rdma_zalloc_obj(struct ib_device *dev, size_t size, + gfp_t gfp, bool is_numa_aware) +{ + if (is_numa_aware && dev->ops.get_numa_node) + return kzalloc_node(size, gfp, dev->ops.get_numa_node(dev)); + + return kzalloc(size, gfp); +} + struct ib_client_nl_info; struct ib_client { const char *name; -- cgit v1.2.3-59-g8ed1b From 20da44dfe8eff5b61685e394dec690a5d9dc36ce Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 23 Jul 2021 14:39:51 +0300 Subject: RDMA/mlx5: Drop in-driver verbs object creations There is no real value in bypassing IB/core APIs for creating standard objects with standard types. The open-coded variant didn't have any restrack task management calls and caused to such objects to be not present when running rdmatoool. Link: https://lore.kernel.org/r/f745590e5fb7d56f90fdb25f64ee3983ba17e1e4.1627040189.git.leonro@nvidia.com Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/verbs.c | 7 +-- drivers/infiniband/hw/mlx5/main.c | 92 +++++++++------------------------------ 2 files changed, 25 insertions(+), 74 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index a164609c2ee7..89c6987cb5eb 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -1035,7 +1035,8 @@ struct ib_srq *ib_create_srq_user(struct ib_pd *pd, } if (srq->srq_type == IB_SRQT_XRC) { srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd; - atomic_inc(&srq->ext.xrc.xrcd->usecnt); + if (srq->ext.xrc.xrcd) + atomic_inc(&srq->ext.xrc.xrcd->usecnt); } atomic_inc(&pd->usecnt); @@ -1046,7 +1047,7 @@ struct ib_srq *ib_create_srq_user(struct ib_pd *pd, if (ret) { rdma_restrack_put(&srq->res); atomic_dec(&srq->pd->usecnt); - if (srq->srq_type == IB_SRQT_XRC) + if (srq->srq_type == IB_SRQT_XRC && srq->ext.xrc.xrcd) atomic_dec(&srq->ext.xrc.xrcd->usecnt); if (ib_srq_has_cq(srq->srq_type)) atomic_dec(&srq->ext.cq->usecnt); @@ -1090,7 +1091,7 @@ int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata) return ret; atomic_dec(&srq->pd->usecnt); - if (srq->srq_type == IB_SRQT_XRC) + if (srq->srq_type == IB_SRQT_XRC && srq->ext.xrc.xrcd) atomic_dec(&srq->ext.xrc.xrcd->usecnt); if (ib_srq_has_cq(srq->srq_type)) atomic_dec(&srq->ext.cq->usecnt); diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 46f28514cf3a..60dfc666d33a 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -2802,31 +2802,16 @@ static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev) if (!MLX5_CAP_GEN(dev->mdev, xrc)) return -EOPNOTSUPP; - devr->p0 = rdma_zalloc_drv_obj(ibdev, ib_pd); - if (!devr->p0) - return -ENOMEM; - - devr->p0->device = ibdev; - devr->p0->uobject = NULL; - atomic_set(&devr->p0->usecnt, 0); + devr->p0 = ib_alloc_pd(ibdev, 0); + if (IS_ERR(devr->p0)) + return PTR_ERR(devr->p0); - ret = mlx5_ib_alloc_pd(devr->p0, NULL); - if (ret) - goto error0; - - devr->c0 = rdma_zalloc_drv_obj(ibdev, ib_cq); - if (!devr->c0) { - ret = -ENOMEM; + devr->c0 = ib_create_cq(ibdev, NULL, NULL, NULL, &cq_attr); + if (IS_ERR(devr->c0)) { + ret = PTR_ERR(devr->c0); goto error1; } - devr->c0->device = &dev->ib_dev; - atomic_set(&devr->c0->usecnt, 0); - - ret = mlx5_ib_create_cq(devr->c0, &cq_attr, NULL); - if (ret) - goto err_create_cq; - ret = mlx5_cmd_xrcd_alloc(dev->mdev, &devr->xrcdn0, 0); if (ret) goto error2; @@ -2841,45 +2826,22 @@ static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev) attr.srq_type = IB_SRQT_XRC; attr.ext.cq = devr->c0; - devr->s0 = rdma_zalloc_drv_obj(ibdev, ib_srq); - if (!devr->s0) { - ret = -ENOMEM; - goto error4; - } - - devr->s0->device = &dev->ib_dev; - devr->s0->pd = devr->p0; - devr->s0->srq_type = IB_SRQT_XRC; - devr->s0->ext.cq = devr->c0; - ret = mlx5_ib_create_srq(devr->s0, &attr, NULL); - if (ret) + devr->s0 = ib_create_srq(devr->p0, &attr); + if (IS_ERR(devr->s0)) { + ret = PTR_ERR(devr->s0); goto err_create; - - atomic_inc(&devr->s0->ext.cq->usecnt); - atomic_inc(&devr->p0->usecnt); - atomic_set(&devr->s0->usecnt, 0); + } memset(&attr, 0, sizeof(attr)); attr.attr.max_sge = 1; attr.attr.max_wr = 1; attr.srq_type = IB_SRQT_BASIC; - devr->s1 = rdma_zalloc_drv_obj(ibdev, ib_srq); - if (!devr->s1) { - ret = -ENOMEM; - goto error5; - } - - devr->s1->device = &dev->ib_dev; - devr->s1->pd = devr->p0; - devr->s1->srq_type = IB_SRQT_BASIC; - devr->s1->ext.cq = devr->c0; - ret = mlx5_ib_create_srq(devr->s1, &attr, NULL); - if (ret) + devr->s1 = ib_create_srq(devr->p0, &attr); + if (IS_ERR(devr->s1)) { + ret = PTR_ERR(devr->s1); goto error6; - - atomic_inc(&devr->p0->usecnt); - atomic_set(&devr->s1->usecnt, 0); + } for (port = 0; port < ARRAY_SIZE(devr->ports); ++port) INIT_WORK(&devr->ports[port].pkey_change_work, @@ -2888,23 +2850,15 @@ static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev) return 0; error6: - kfree(devr->s1); -error5: - mlx5_ib_destroy_srq(devr->s0, NULL); + ib_destroy_srq(devr->s0); err_create: - kfree(devr->s0); -error4: mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0); error3: mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0); error2: - mlx5_ib_destroy_cq(devr->c0, NULL); -err_create_cq: - kfree(devr->c0); + ib_destroy_cq(devr->c0); error1: - mlx5_ib_dealloc_pd(devr->p0, NULL); -error0: - kfree(devr->p0); + ib_dealloc_pd(devr->p0); return ret; } @@ -2922,16 +2876,12 @@ static void mlx5_ib_dev_res_cleanup(struct mlx5_ib_dev *dev) for (port = 0; port < ARRAY_SIZE(devr->ports); ++port) cancel_work_sync(&devr->ports[port].pkey_change_work); - mlx5_ib_destroy_srq(devr->s1, NULL); - kfree(devr->s1); - mlx5_ib_destroy_srq(devr->s0, NULL); - kfree(devr->s0); + ib_destroy_srq(devr->s1); + ib_destroy_srq(devr->s0); mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0); mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0); - mlx5_ib_destroy_cq(devr->c0, NULL); - kfree(devr->c0); - mlx5_ib_dealloc_pd(devr->p0, NULL); - kfree(devr->p0); + ib_destroy_cq(devr->c0); + ib_dealloc_pd(devr->p0); } static u32 get_core_cap_flags(struct ib_device *ibdev, -- cgit v1.2.3-59-g8ed1b From 8fc3beebf623092e446f4c88fca1699c868ca86d Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Tue, 3 Aug 2021 21:20:33 +0300 Subject: RDMA/core: Delete duplicated and unreachable code The ib_create_named_qp() is kernel verb and no kernel users exist that use XRC_INI QP. Hence such QP path is not reachable. In addition, delete duplicated assignments of QP attributes from the initialization structure. Link: https://lore.kernel.org/r/1b4c0d1def5f8f6d26839e14d19da950cc4a0b05.1628014762.git.leonro@nvidia.com Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/core_priv.h | 1 + drivers/infiniband/core/verbs.c | 22 ++++------------------ 2 files changed, 5 insertions(+), 18 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h index fa2e0bbaf8c7..c870adecd3a4 100644 --- a/drivers/infiniband/core/core_priv.h +++ b/drivers/infiniband/core/core_priv.h @@ -341,6 +341,7 @@ _ib_create_qp(struct ib_device *dev, struct ib_pd *pd, qp->srq = attr->srq; qp->event_handler = attr->event_handler; qp->port = attr->port_num; + qp->qp_context = attr->qp_context; spin_lock_init(&qp->mr_lock); INIT_LIST_HEAD(&qp->rdma_mrs); diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index 89c6987cb5eb..635642a3ecbc 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -1257,28 +1257,14 @@ struct ib_qp *ib_create_named_qp(struct ib_pd *pd, return xrc_qp; } - qp->event_handler = qp_init_attr->event_handler; - qp->qp_context = qp_init_attr->qp_context; - if (qp_init_attr->qp_type == IB_QPT_XRC_INI) { - qp->recv_cq = NULL; - qp->srq = NULL; - } else { - qp->recv_cq = qp_init_attr->recv_cq; - if (qp_init_attr->recv_cq) - atomic_inc(&qp_init_attr->recv_cq->usecnt); - qp->srq = qp_init_attr->srq; - if (qp->srq) - atomic_inc(&qp_init_attr->srq->usecnt); - } - - qp->send_cq = qp_init_attr->send_cq; - qp->xrcd = NULL; + if (qp_init_attr->recv_cq) + atomic_inc(&qp_init_attr->recv_cq->usecnt); + if (qp->srq) + atomic_inc(&qp_init_attr->srq->usecnt); atomic_inc(&pd->usecnt); if (qp_init_attr->send_cq) atomic_inc(&qp_init_attr->send_cq->usecnt); - if (qp_init_attr->rwq_ind_tbl) - atomic_inc(&qp->rwq_ind_tbl->usecnt); if (qp_init_attr->cap.max_rdma_ctxs) { ret = rdma_rw_init_mrs(qp, qp_init_attr); -- cgit v1.2.3-59-g8ed1b From 20e2bcc4c2a8ede2fe6e335b9eea357bcfbe79bb Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Tue, 3 Aug 2021 21:20:34 +0300 Subject: RDMA/core: Remove protection from wrong in-kernel API usage The ib_create_named_qp() is kernel verb that is not used for user supplied attributes. In such case, it is ULP responsibility to provide valid QP attributes. In-kernel API shouldn't check it, exactly like other functions that don't check device capabilities. Link: https://lore.kernel.org/r/b9b9e981d1af148b750750196e686199dbbf61f8.1628014762.git.leonro@nvidia.com Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/verbs.c | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index 635642a3ecbc..2090f3c9f689 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -1219,16 +1219,6 @@ struct ib_qp *ib_create_named_qp(struct ib_pd *pd, struct ib_qp *qp; int ret; - if (qp_init_attr->rwq_ind_tbl && - (qp_init_attr->recv_cq || - qp_init_attr->srq || qp_init_attr->cap.max_recv_wr || - qp_init_attr->cap.max_recv_sge)) - return ERR_PTR(-EINVAL); - - if ((qp_init_attr->create_flags & IB_QP_CREATE_INTEGRITY_EN) && - !(device->attrs.device_cap_flags & IB_DEVICE_INTEGRITY_HANDOVER)) - return ERR_PTR(-EINVAL); - /* * If the callers is using the RDMA API calculate the resources * needed for the RDMA READ/WRITE operations. -- cgit v1.2.3-59-g8ed1b From 8da9fe4e4fa7d561df0f3fe65bfa6dbf78aa7590 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Tue, 3 Aug 2021 21:20:35 +0300 Subject: RDMA/core: Reorganize create QP low-level functions The low-level create QP function grew to be larger than any sensible inline function should be. The inline attribute is not really needed for that function and can be implemented as exported symbol. Link: https://lore.kernel.org/r/2c08709d86f876c3dfb77684357b2a939e570ca4.1628014762.git.leonro@nvidia.com Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/core_priv.h | 58 ++--------------------------- drivers/infiniband/core/verbs.c | 74 ++++++++++++++++++++++++++++++++----- include/rdma/ib_verbs.h | 16 ++++++-- 3 files changed, 81 insertions(+), 67 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h index c870adecd3a4..d28ced053222 100644 --- a/drivers/infiniband/core/core_priv.h +++ b/drivers/infiniband/core/core_priv.h @@ -316,60 +316,10 @@ struct ib_device *ib_device_get_by_index(const struct net *net, u32 index); void nldev_init(void); void nldev_exit(void); -static inline struct ib_qp * -_ib_create_qp(struct ib_device *dev, struct ib_pd *pd, - struct ib_qp_init_attr *attr, struct ib_udata *udata, - struct ib_uqp_object *uobj, const char *caller) -{ - struct ib_qp *qp; - int ret; - - if (!dev->ops.create_qp) - return ERR_PTR(-EOPNOTSUPP); - - qp = rdma_zalloc_drv_obj_numa(dev, ib_qp); - if (!qp) - return ERR_PTR(-ENOMEM); - - qp->device = dev; - qp->pd = pd; - qp->uobject = uobj; - qp->real_qp = qp; - - qp->qp_type = attr->qp_type; - qp->rwq_ind_tbl = attr->rwq_ind_tbl; - qp->srq = attr->srq; - qp->event_handler = attr->event_handler; - qp->port = attr->port_num; - qp->qp_context = attr->qp_context; - - spin_lock_init(&qp->mr_lock); - INIT_LIST_HEAD(&qp->rdma_mrs); - INIT_LIST_HEAD(&qp->sig_mrs); - - rdma_restrack_new(&qp->res, RDMA_RESTRACK_QP); - WARN_ONCE(!udata && !caller, "Missing kernel QP owner"); - rdma_restrack_set_name(&qp->res, udata ? NULL : caller); - ret = dev->ops.create_qp(qp, attr, udata); - if (ret) - goto err_create; - - /* - * TODO: The mlx4 internally overwrites send_cq and recv_cq. - * Unfortunately, it is not an easy task to fix that driver. - */ - qp->send_cq = attr->send_cq; - qp->recv_cq = attr->recv_cq; - - rdma_restrack_add(&qp->res); - return qp; - -err_create: - rdma_restrack_put(&qp->res); - kfree(qp); - return ERR_PTR(ret); - -} +struct ib_qp *_ib_create_qp(struct ib_device *dev, struct ib_pd *pd, + struct ib_qp_init_attr *attr, + struct ib_udata *udata, struct ib_uqp_object *uobj, + const char *caller); struct rdma_dev_addr; int rdma_resolve_ip_route(struct sockaddr *src_addr, diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index 2090f3c9f689..a7717df83273 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -1201,19 +1201,75 @@ static struct ib_qp *create_xrc_qp_user(struct ib_qp *qp, } /** - * ib_create_named_qp - Creates a kernel QP associated with the specified protection - * domain. + * _ib_create_qp - Creates a QP associated with the specified protection domain + * @dev: IB device * @pd: The protection domain associated with the QP. - * @qp_init_attr: A list of initial attributes required to create the + * @attr: A list of initial attributes required to create the * QP. If QP creation succeeds, then the attributes are updated to * the actual capabilities of the created QP. + * @udata: User data + * @uobj: uverbs obect * @caller: caller's build-time module name - * - * NOTE: for user qp use ib_create_qp_user with valid udata! */ -struct ib_qp *ib_create_named_qp(struct ib_pd *pd, - struct ib_qp_init_attr *qp_init_attr, - const char *caller) +struct ib_qp *_ib_create_qp(struct ib_device *dev, struct ib_pd *pd, + struct ib_qp_init_attr *attr, + struct ib_udata *udata, struct ib_uqp_object *uobj, + const char *caller) +{ + struct ib_qp *qp; + int ret; + + if (!dev->ops.create_qp) + return ERR_PTR(-EOPNOTSUPP); + + qp = rdma_zalloc_drv_obj_numa(dev, ib_qp); + if (!qp) + return ERR_PTR(-ENOMEM); + + qp->device = dev; + qp->pd = pd; + qp->uobject = uobj; + qp->real_qp = qp; + + qp->qp_type = attr->qp_type; + qp->rwq_ind_tbl = attr->rwq_ind_tbl; + qp->srq = attr->srq; + qp->event_handler = attr->event_handler; + qp->port = attr->port_num; + qp->qp_context = attr->qp_context; + + spin_lock_init(&qp->mr_lock); + INIT_LIST_HEAD(&qp->rdma_mrs); + INIT_LIST_HEAD(&qp->sig_mrs); + + rdma_restrack_new(&qp->res, RDMA_RESTRACK_QP); + WARN_ONCE(!udata && !caller, "Missing kernel QP owner"); + rdma_restrack_set_name(&qp->res, udata ? NULL : caller); + ret = dev->ops.create_qp(qp, attr, udata); + if (ret) + goto err_create; + + /* + * TODO: The mlx4 internally overwrites send_cq and recv_cq. + * Unfortunately, it is not an easy task to fix that driver. + */ + qp->send_cq = attr->send_cq; + qp->recv_cq = attr->recv_cq; + + rdma_restrack_add(&qp->res); + return qp; + +err_create: + rdma_restrack_put(&qp->res); + kfree(qp); + return ERR_PTR(ret); + +} +EXPORT_SYMBOL(_ib_create_qp); + +struct ib_qp *ib_create_qp_kernel(struct ib_pd *pd, + struct ib_qp_init_attr *qp_init_attr, + const char *caller) { struct ib_device *device = pd ? pd->device : qp_init_attr->xrcd->device; struct ib_qp *qp; @@ -1280,7 +1336,7 @@ err: return ERR_PTR(ret); } -EXPORT_SYMBOL(ib_create_named_qp); +EXPORT_SYMBOL(ib_create_qp_kernel); static const struct { int valid; diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 6737582e9e2e..aa7806335cba 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -3688,13 +3688,21 @@ static inline int ib_post_srq_recv(struct ib_srq *srq, bad_recv_wr ? : &dummy); } -struct ib_qp *ib_create_named_qp(struct ib_pd *pd, - struct ib_qp_init_attr *qp_init_attr, - const char *caller); +struct ib_qp *ib_create_qp_kernel(struct ib_pd *pd, + struct ib_qp_init_attr *qp_init_attr, + const char *caller); +/** + * ib_create_qp - Creates a kernel QP associated with the specific protection + * domain. + * @pd: The protection domain associated with the QP. + * @init_attr: A list of initial attributes required to create the + * QP. If QP creation succeeds, then the attributes are updated to + * the actual capabilities of the created QP. + */ static inline struct ib_qp *ib_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *init_attr) { - return ib_create_named_qp(pd, init_attr, KBUILD_MODNAME); + return ib_create_qp_kernel(pd, init_attr, KBUILD_MODNAME); } /** -- cgit v1.2.3-59-g8ed1b From 00a79d6b996d46e9077b0d02a19c87b99305b94a Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Tue, 3 Aug 2021 21:20:36 +0300 Subject: RDMA/core: Configure selinux QP during creation All QP creation flows called ib_create_qp_security(), but differently. This caused to the need to provide exclusion conditions for the XRC_TGT, because such QP already had selinux configuration call. In order to fix it, move ib_create_qp_security() to the general QP creation routine. Link: https://lore.kernel.org/r/4d7cd6f5828aca37fb62283e6b126b73ab86b18c.1628014762.git.leonro@nvidia.com Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/uverbs_cmd.c | 7 ------- drivers/infiniband/core/uverbs_std_types_qp.c | 6 ------ drivers/infiniband/core/verbs.c | 11 +++++++---- 3 files changed, 7 insertions(+), 17 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index 8c8ca7bce3ca..b5153200b8a8 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -1447,10 +1447,6 @@ static int create_qp(struct uverbs_attr_bundle *attrs, } if (cmd->qp_type != IB_QPT_XRC_TGT) { - ret = ib_create_qp_security(qp, device); - if (ret) - goto err_cb; - atomic_inc(&pd->usecnt); if (attr.send_cq) atomic_inc(&attr.send_cq->usecnt); @@ -1502,9 +1498,6 @@ static int create_qp(struct uverbs_attr_bundle *attrs, resp.response_length = uverbs_response_length(attrs, sizeof(resp)); return uverbs_response(attrs, &resp, sizeof(resp)); -err_cb: - ib_destroy_qp_user(qp, uverbs_get_cleared_udata(attrs)); - err_put: if (!IS_ERR(xrcd_uobj)) uobj_put_read(xrcd_uobj); diff --git a/drivers/infiniband/core/uverbs_std_types_qp.c b/drivers/infiniband/core/uverbs_std_types_qp.c index c00cfb5ed387..92812f6a21b0 100644 --- a/drivers/infiniband/core/uverbs_std_types_qp.c +++ b/drivers/infiniband/core/uverbs_std_types_qp.c @@ -280,12 +280,6 @@ static int UVERBS_HANDLER(UVERBS_METHOD_QP_CREATE)( obj->uevent.uobject.object = qp; uverbs_finalize_uobj_create(attrs, UVERBS_ATTR_CREATE_QP_HANDLE); - if (attr.qp_type != IB_QPT_XRC_TGT) { - ret = ib_create_qp_security(qp, device); - if (ret) - return ret; - } - set_caps(&attr, &cap, false); ret = uverbs_copy_to_struct_or_zero(attrs, UVERBS_ATTR_CREATE_QP_RESP_CAP, &cap, diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index a7717df83273..1f0f0beebe22 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -1216,6 +1216,7 @@ struct ib_qp *_ib_create_qp(struct ib_device *dev, struct ib_pd *pd, struct ib_udata *udata, struct ib_uqp_object *uobj, const char *caller) { + struct ib_udata dummy = {}; struct ib_qp *qp; int ret; @@ -1256,9 +1257,15 @@ struct ib_qp *_ib_create_qp(struct ib_device *dev, struct ib_pd *pd, qp->send_cq = attr->send_cq; qp->recv_cq = attr->recv_cq; + ret = ib_create_qp_security(qp, dev); + if (ret) + goto err_security; + rdma_restrack_add(&qp->res); return qp; +err_security: + qp->device->ops.destroy_qp(qp, udata ? &dummy : NULL); err_create: rdma_restrack_put(&qp->res); kfree(qp); @@ -1288,10 +1295,6 @@ struct ib_qp *ib_create_qp_kernel(struct ib_pd *pd, if (IS_ERR(qp)) return qp; - ret = ib_create_qp_security(qp, device); - if (ret) - goto err; - if (qp_init_attr->qp_type == IB_QPT_XRC_TGT) { struct ib_qp *xrc_qp = create_xrc_qp_user(qp, qp_init_attr); -- cgit v1.2.3-59-g8ed1b From 5507f67d08cdd947714647caa5c60f96b719fcb7 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Tue, 3 Aug 2021 21:20:37 +0300 Subject: RDMA/core: Properly increment and decrement QP usecnts The QP usecnts were incremented through QP attributes structure while decreased through QP itself. Rely on the ib_creat_qp_user() code that initialized all QP parameters prior returning to the user and increment exactly like destroy does. Link: https://lore.kernel.org/r/25d256a3bb1fc480b77d7fe439817b993de48610.1628014762.git.leonro@nvidia.com Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/core_priv.h | 2 + drivers/infiniband/core/uverbs_cmd.c | 13 +----- drivers/infiniband/core/uverbs_std_types_qp.c | 13 +----- drivers/infiniband/core/verbs.c | 60 +++++++++++++++------------ 4 files changed, 39 insertions(+), 49 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h index d28ced053222..d8f464b43dbc 100644 --- a/drivers/infiniband/core/core_priv.h +++ b/drivers/infiniband/core/core_priv.h @@ -320,6 +320,8 @@ struct ib_qp *_ib_create_qp(struct ib_device *dev, struct ib_pd *pd, struct ib_qp_init_attr *attr, struct ib_udata *udata, struct ib_uqp_object *uobj, const char *caller); +void ib_qp_usecnt_inc(struct ib_qp *qp); +void ib_qp_usecnt_dec(struct ib_qp *qp); struct rdma_dev_addr; int rdma_resolve_ip_route(struct sockaddr *src_addr, diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index b5153200b8a8..62cafd768d89 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -1445,18 +1445,9 @@ static int create_qp(struct uverbs_attr_bundle *attrs, ret = PTR_ERR(qp); goto err_put; } + ib_qp_usecnt_inc(qp); - if (cmd->qp_type != IB_QPT_XRC_TGT) { - atomic_inc(&pd->usecnt); - if (attr.send_cq) - atomic_inc(&attr.send_cq->usecnt); - if (attr.recv_cq) - atomic_inc(&attr.recv_cq->usecnt); - if (attr.srq) - atomic_inc(&attr.srq->usecnt); - if (ind_tbl) - atomic_inc(&ind_tbl->usecnt); - } else { + if (cmd->qp_type == IB_QPT_XRC_TGT) { /* It is done in _ib_create_qp for other QP types */ qp->uobject = obj; } diff --git a/drivers/infiniband/core/uverbs_std_types_qp.c b/drivers/infiniband/core/uverbs_std_types_qp.c index 92812f6a21b0..a0e734735ba5 100644 --- a/drivers/infiniband/core/uverbs_std_types_qp.c +++ b/drivers/infiniband/core/uverbs_std_types_qp.c @@ -258,18 +258,9 @@ static int UVERBS_HANDLER(UVERBS_METHOD_QP_CREATE)( ret = PTR_ERR(qp); goto err_put; } + ib_qp_usecnt_inc(qp); - if (attr.qp_type != IB_QPT_XRC_TGT) { - atomic_inc(&pd->usecnt); - if (attr.send_cq) - atomic_inc(&attr.send_cq->usecnt); - if (attr.recv_cq) - atomic_inc(&attr.recv_cq->usecnt); - if (attr.srq) - atomic_inc(&attr.srq->usecnt); - if (attr.rwq_ind_tbl) - atomic_inc(&attr.rwq_ind_tbl->usecnt); - } else { + if (attr.qp_type == IB_QPT_XRC_TGT) { obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject); atomic_inc(&obj->uxrcd->refcnt); diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index 1f0f0beebe22..a568155b63f5 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -1274,6 +1274,36 @@ err_create: } EXPORT_SYMBOL(_ib_create_qp); +void ib_qp_usecnt_inc(struct ib_qp *qp) +{ + if (qp->pd) + atomic_inc(&qp->pd->usecnt); + if (qp->send_cq) + atomic_inc(&qp->send_cq->usecnt); + if (qp->recv_cq) + atomic_inc(&qp->recv_cq->usecnt); + if (qp->srq) + atomic_inc(&qp->srq->usecnt); + if (qp->rwq_ind_tbl) + atomic_inc(&qp->rwq_ind_tbl->usecnt); +} +EXPORT_SYMBOL(ib_qp_usecnt_inc); + +void ib_qp_usecnt_dec(struct ib_qp *qp) +{ + if (qp->rwq_ind_tbl) + atomic_dec(&qp->rwq_ind_tbl->usecnt); + if (qp->srq) + atomic_dec(&qp->srq->usecnt); + if (qp->recv_cq) + atomic_dec(&qp->recv_cq->usecnt); + if (qp->send_cq) + atomic_dec(&qp->send_cq->usecnt); + if (qp->pd) + atomic_dec(&qp->pd->usecnt); +} +EXPORT_SYMBOL(ib_qp_usecnt_dec); + struct ib_qp *ib_create_qp_kernel(struct ib_pd *pd, struct ib_qp_init_attr *qp_init_attr, const char *caller) @@ -1306,14 +1336,7 @@ struct ib_qp *ib_create_qp_kernel(struct ib_pd *pd, return xrc_qp; } - if (qp_init_attr->recv_cq) - atomic_inc(&qp_init_attr->recv_cq->usecnt); - if (qp->srq) - atomic_inc(&qp_init_attr->srq->usecnt); - - atomic_inc(&pd->usecnt); - if (qp_init_attr->send_cq) - atomic_inc(&qp_init_attr->send_cq->usecnt); + ib_qp_usecnt_inc(qp); if (qp_init_attr->cap.max_rdma_ctxs) { ret = rdma_rw_init_mrs(qp, qp_init_attr); @@ -1971,10 +1994,6 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata) { const struct ib_gid_attr *alt_path_sgid_attr = qp->alt_path_sgid_attr; const struct ib_gid_attr *av_sgid_attr = qp->av_sgid_attr; - struct ib_pd *pd; - struct ib_cq *scq, *rcq; - struct ib_srq *srq; - struct ib_rwq_ind_table *ind_tbl; struct ib_qp_security *sec; int ret; @@ -1986,11 +2005,6 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata) if (qp->real_qp != qp) return __ib_destroy_shared_qp(qp); - pd = qp->pd; - scq = qp->send_cq; - rcq = qp->recv_cq; - srq = qp->srq; - ind_tbl = qp->rwq_ind_tbl; sec = qp->qp_sec; if (sec) ib_destroy_qp_security_begin(sec); @@ -2010,16 +2024,8 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata) rdma_put_gid_attr(alt_path_sgid_attr); if (av_sgid_attr) rdma_put_gid_attr(av_sgid_attr); - if (pd) - atomic_dec(&pd->usecnt); - if (scq) - atomic_dec(&scq->usecnt); - if (rcq) - atomic_dec(&rcq->usecnt); - if (srq) - atomic_dec(&srq->usecnt); - if (ind_tbl) - atomic_dec(&ind_tbl->usecnt); + + ib_qp_usecnt_dec(qp); if (sec) ib_destroy_qp_security_end(sec); -- cgit v1.2.3-59-g8ed1b From d2b10794fc1312f856d67d0a6454aaa3ae96c595 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Tue, 3 Aug 2021 21:20:38 +0300 Subject: RDMA/core: Create clean QP creations interface for uverbs Unify create QP creation interface to make clean approach to create XRC_TGT and regular QPs. Link: https://lore.kernel.org/r/5cd50e7d8ad9112545a1a61dea62799a5cb3224a.1628014762.git.leonro@nvidia.com Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/core_priv.h | 9 ++-- drivers/infiniband/core/uverbs_cmd.c | 13 +---- drivers/infiniband/core/uverbs_std_types_qp.c | 10 +--- drivers/infiniband/core/verbs.c | 72 ++++++++++++++++----------- 4 files changed, 52 insertions(+), 52 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h index d8f464b43dbc..f66f48d860ec 100644 --- a/drivers/infiniband/core/core_priv.h +++ b/drivers/infiniband/core/core_priv.h @@ -316,10 +316,11 @@ struct ib_device *ib_device_get_by_index(const struct net *net, u32 index); void nldev_init(void); void nldev_exit(void); -struct ib_qp *_ib_create_qp(struct ib_device *dev, struct ib_pd *pd, - struct ib_qp_init_attr *attr, - struct ib_udata *udata, struct ib_uqp_object *uobj, - const char *caller); +struct ib_qp *ib_create_qp_user(struct ib_device *dev, struct ib_pd *pd, + struct ib_qp_init_attr *attr, + struct ib_udata *udata, + struct ib_uqp_object *uobj, const char *caller); + void ib_qp_usecnt_inc(struct ib_qp *qp); void ib_qp_usecnt_dec(struct ib_qp *qp); diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index 62cafd768d89..740e6b2efe0e 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -1435,23 +1435,14 @@ static int create_qp(struct uverbs_attr_bundle *attrs, attr.source_qpn = cmd->source_qpn; } - if (cmd->qp_type == IB_QPT_XRC_TGT) - qp = ib_create_qp(pd, &attr); - else - qp = _ib_create_qp(device, pd, &attr, &attrs->driver_udata, obj, - NULL); - + qp = ib_create_qp_user(device, pd, &attr, &attrs->driver_udata, obj, + KBUILD_MODNAME); if (IS_ERR(qp)) { ret = PTR_ERR(qp); goto err_put; } ib_qp_usecnt_inc(qp); - if (cmd->qp_type == IB_QPT_XRC_TGT) { - /* It is done in _ib_create_qp for other QP types */ - qp->uobject = obj; - } - obj->uevent.uobject.object = qp; obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file); if (obj->uevent.event_file) diff --git a/drivers/infiniband/core/uverbs_std_types_qp.c b/drivers/infiniband/core/uverbs_std_types_qp.c index a0e734735ba5..dd1075466f61 100644 --- a/drivers/infiniband/core/uverbs_std_types_qp.c +++ b/drivers/infiniband/core/uverbs_std_types_qp.c @@ -248,12 +248,8 @@ static int UVERBS_HANDLER(UVERBS_METHOD_QP_CREATE)( set_caps(&attr, &cap, true); mutex_init(&obj->mcast_lock); - if (attr.qp_type == IB_QPT_XRC_TGT) - qp = ib_create_qp(pd, &attr); - else - qp = _ib_create_qp(device, pd, &attr, &attrs->driver_udata, obj, - NULL); - + qp = ib_create_qp_user(device, pd, &attr, &attrs->driver_udata, obj, + KBUILD_MODNAME); if (IS_ERR(qp)) { ret = PTR_ERR(qp); goto err_put; @@ -264,8 +260,6 @@ static int UVERBS_HANDLER(UVERBS_METHOD_QP_CREATE)( obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject); atomic_inc(&obj->uxrcd->refcnt); - /* It is done in _ib_create_qp for other QP types */ - qp->uobject = obj; } obj->uevent.uobject.object = qp; diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index a568155b63f5..89a2b21976d6 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -1200,21 +1200,10 @@ static struct ib_qp *create_xrc_qp_user(struct ib_qp *qp, return qp; } -/** - * _ib_create_qp - Creates a QP associated with the specified protection domain - * @dev: IB device - * @pd: The protection domain associated with the QP. - * @attr: A list of initial attributes required to create the - * QP. If QP creation succeeds, then the attributes are updated to - * the actual capabilities of the created QP. - * @udata: User data - * @uobj: uverbs obect - * @caller: caller's build-time module name - */ -struct ib_qp *_ib_create_qp(struct ib_device *dev, struct ib_pd *pd, - struct ib_qp_init_attr *attr, - struct ib_udata *udata, struct ib_uqp_object *uobj, - const char *caller) +static struct ib_qp *create_qp(struct ib_device *dev, struct ib_pd *pd, + struct ib_qp_init_attr *attr, + struct ib_udata *udata, + struct ib_uqp_object *uobj, const char *caller) { struct ib_udata dummy = {}; struct ib_qp *qp; @@ -1272,7 +1261,43 @@ err_create: return ERR_PTR(ret); } -EXPORT_SYMBOL(_ib_create_qp); + +/** + * ib_create_qp_user - Creates a QP associated with the specified protection + * domain. + * @dev: IB device + * @pd: The protection domain associated with the QP. + * @attr: A list of initial attributes required to create the + * QP. If QP creation succeeds, then the attributes are updated to + * the actual capabilities of the created QP. + * @udata: User data + * @uobj: uverbs obect + * @caller: caller's build-time module name + */ +struct ib_qp *ib_create_qp_user(struct ib_device *dev, struct ib_pd *pd, + struct ib_qp_init_attr *attr, + struct ib_udata *udata, + struct ib_uqp_object *uobj, const char *caller) +{ + struct ib_qp *qp, *xrc_qp; + + if (attr->qp_type == IB_QPT_XRC_TGT) + qp = create_qp(dev, pd, attr, NULL, NULL, caller); + else + qp = create_qp(dev, pd, attr, udata, uobj, NULL); + if (attr->qp_type != IB_QPT_XRC_TGT || IS_ERR(qp)) + return qp; + + xrc_qp = create_xrc_qp_user(qp, attr); + if (IS_ERR(xrc_qp)) { + ib_destroy_qp(qp); + return xrc_qp; + } + + xrc_qp->uobject = uobj; + return xrc_qp; +} +EXPORT_SYMBOL(ib_create_qp_user); void ib_qp_usecnt_inc(struct ib_qp *qp) { @@ -1308,7 +1333,7 @@ struct ib_qp *ib_create_qp_kernel(struct ib_pd *pd, struct ib_qp_init_attr *qp_init_attr, const char *caller) { - struct ib_device *device = pd ? pd->device : qp_init_attr->xrcd->device; + struct ib_device *device = pd->device; struct ib_qp *qp; int ret; @@ -1321,21 +1346,10 @@ struct ib_qp *ib_create_qp_kernel(struct ib_pd *pd, if (qp_init_attr->cap.max_rdma_ctxs) rdma_rw_init_qp(device, qp_init_attr); - qp = _ib_create_qp(device, pd, qp_init_attr, NULL, NULL, caller); + qp = create_qp(device, pd, qp_init_attr, NULL, NULL, caller); if (IS_ERR(qp)) return qp; - if (qp_init_attr->qp_type == IB_QPT_XRC_TGT) { - struct ib_qp *xrc_qp = - create_xrc_qp_user(qp, qp_init_attr); - - if (IS_ERR(xrc_qp)) { - ret = PTR_ERR(xrc_qp); - goto err; - } - return xrc_qp; - } - ib_qp_usecnt_inc(qp); if (qp_init_attr->cap.max_rdma_ctxs) { -- cgit v1.2.3-59-g8ed1b From bfeababd5141a110ddffff0fa1e7fd988d76292d Mon Sep 17 00:00:00 2001 From: Håkon Bugge Date: Wed, 11 Aug 2021 19:25:36 +0200 Subject: RDMA/core/sa_query: Remove unused function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ib_sa_service_rec_query() was introduced in kernel v2.6.13 by commit cbae32c56314 ("[PATCH] IB: Add Service Record support to SA client") in 2005. It was not used then and have never been used since. Removing it and related functions/structs. Link: https://lore.kernel.org/r/1628702736-12651-1-git-send-email-haakon.bugge@oracle.com Signed-off-by: Håkon Bugge Reviewed-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/sa_query.c | 177 ------------------------------------- include/rdma/ib_sa.h | 24 ----- 2 files changed, 201 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/sa_query.c b/drivers/infiniband/core/sa_query.c index b61576f702b8..740499bd4935 100644 --- a/drivers/infiniband/core/sa_query.c +++ b/drivers/infiniband/core/sa_query.c @@ -123,12 +123,6 @@ struct ib_sa_query { #define IB_SA_CANCEL 0x00000002 #define IB_SA_QUERY_OPA 0x00000004 -struct ib_sa_service_query { - void (*callback)(int, struct ib_sa_service_rec *, void *); - void *context; - struct ib_sa_query sa_query; -}; - struct ib_sa_path_query { void (*callback)(int, struct sa_path_rec *, void *); void *context; @@ -502,54 +496,6 @@ static const struct ib_field mcmember_rec_table[] = { .size_bits = 23 }, }; -#define SERVICE_REC_FIELD(field) \ - .struct_offset_bytes = offsetof(struct ib_sa_service_rec, field), \ - .struct_size_bytes = sizeof_field(struct ib_sa_service_rec, field), \ - .field_name = "sa_service_rec:" #field - -static const struct ib_field service_rec_table[] = { - { SERVICE_REC_FIELD(id), - .offset_words = 0, - .offset_bits = 0, - .size_bits = 64 }, - { SERVICE_REC_FIELD(gid), - .offset_words = 2, - .offset_bits = 0, - .size_bits = 128 }, - { SERVICE_REC_FIELD(pkey), - .offset_words = 6, - .offset_bits = 0, - .size_bits = 16 }, - { SERVICE_REC_FIELD(lease), - .offset_words = 7, - .offset_bits = 0, - .size_bits = 32 }, - { SERVICE_REC_FIELD(key), - .offset_words = 8, - .offset_bits = 0, - .size_bits = 128 }, - { SERVICE_REC_FIELD(name), - .offset_words = 12, - .offset_bits = 0, - .size_bits = 64*8 }, - { SERVICE_REC_FIELD(data8), - .offset_words = 28, - .offset_bits = 0, - .size_bits = 16*8 }, - { SERVICE_REC_FIELD(data16), - .offset_words = 32, - .offset_bits = 0, - .size_bits = 8*16 }, - { SERVICE_REC_FIELD(data32), - .offset_words = 36, - .offset_bits = 0, - .size_bits = 4*32 }, - { SERVICE_REC_FIELD(data64), - .offset_words = 40, - .offset_bits = 0, - .size_bits = 2*64 }, -}; - #define CLASSPORTINFO_REC_FIELD(field) \ .struct_offset_bytes = offsetof(struct ib_class_port_info, field), \ .struct_size_bytes = sizeof_field(struct ib_class_port_info, field), \ @@ -1634,129 +1580,6 @@ err1: } EXPORT_SYMBOL(ib_sa_path_rec_get); -static void ib_sa_service_rec_callback(struct ib_sa_query *sa_query, - int status, - struct ib_sa_mad *mad) -{ - struct ib_sa_service_query *query = - container_of(sa_query, struct ib_sa_service_query, sa_query); - - if (mad) { - struct ib_sa_service_rec rec; - - ib_unpack(service_rec_table, ARRAY_SIZE(service_rec_table), - mad->data, &rec); - query->callback(status, &rec, query->context); - } else - query->callback(status, NULL, query->context); -} - -static void ib_sa_service_rec_release(struct ib_sa_query *sa_query) -{ - kfree(container_of(sa_query, struct ib_sa_service_query, sa_query)); -} - -/** - * ib_sa_service_rec_query - Start Service Record operation - * @client:SA client - * @device:device to send request on - * @port_num: port number to send request on - * @method:SA method - should be get, set, or delete - * @rec:Service Record to send in request - * @comp_mask:component mask to send in request - * @timeout_ms:time to wait for response - * @gfp_mask:GFP mask to use for internal allocations - * @callback:function called when request completes, times out or is - * canceled - * @context:opaque user context passed to callback - * @sa_query:request context, used to cancel request - * - * Send a Service Record set/get/delete to the SA to register, - * unregister or query a service record. - * The callback function will be called when the request completes (or - * fails); status is 0 for a successful response, -EINTR if the query - * is canceled, -ETIMEDOUT is the query timed out, or -EIO if an error - * occurred sending the query. The resp parameter of the callback is - * only valid if status is 0. - * - * If the return value of ib_sa_service_rec_query() is negative, it is an - * error code. Otherwise it is a request ID that can be used to cancel - * the query. - */ -int ib_sa_service_rec_query(struct ib_sa_client *client, - struct ib_device *device, u32 port_num, u8 method, - struct ib_sa_service_rec *rec, - ib_sa_comp_mask comp_mask, - unsigned long timeout_ms, gfp_t gfp_mask, - void (*callback)(int status, - struct ib_sa_service_rec *resp, - void *context), - void *context, - struct ib_sa_query **sa_query) -{ - struct ib_sa_service_query *query; - struct ib_sa_device *sa_dev = ib_get_client_data(device, &sa_client); - struct ib_sa_port *port; - struct ib_mad_agent *agent; - struct ib_sa_mad *mad; - int ret; - - if (!sa_dev) - return -ENODEV; - - port = &sa_dev->port[port_num - sa_dev->start_port]; - agent = port->agent; - - if (method != IB_MGMT_METHOD_GET && - method != IB_MGMT_METHOD_SET && - method != IB_SA_METHOD_DELETE) - return -EINVAL; - - query = kzalloc(sizeof(*query), gfp_mask); - if (!query) - return -ENOMEM; - - query->sa_query.port = port; - ret = alloc_mad(&query->sa_query, gfp_mask); - if (ret) - goto err1; - - ib_sa_client_get(client); - query->sa_query.client = client; - query->callback = callback; - query->context = context; - - mad = query->sa_query.mad_buf->mad; - init_mad(&query->sa_query, agent); - - query->sa_query.callback = callback ? ib_sa_service_rec_callback : NULL; - query->sa_query.release = ib_sa_service_rec_release; - mad->mad_hdr.method = method; - mad->mad_hdr.attr_id = cpu_to_be16(IB_SA_ATTR_SERVICE_REC); - mad->sa_hdr.comp_mask = comp_mask; - - ib_pack(service_rec_table, ARRAY_SIZE(service_rec_table), - rec, mad->data); - - *sa_query = &query->sa_query; - - ret = send_mad(&query->sa_query, timeout_ms, gfp_mask); - if (ret < 0) - goto err2; - - return ret; - -err2: - *sa_query = NULL; - ib_sa_client_put(query->sa_query.client); - free_mad(&query->sa_query); - -err1: - kfree(query); - return ret; -} -EXPORT_SYMBOL(ib_sa_service_rec_query); - static void ib_sa_mcmember_rec_callback(struct ib_sa_query *sa_query, int status, struct ib_sa_mad *mad) diff --git a/include/rdma/ib_sa.h b/include/rdma/ib_sa.h index ba3c808a3789..3634d4cc7a56 100644 --- a/include/rdma/ib_sa.h +++ b/include/rdma/ib_sa.h @@ -366,20 +366,6 @@ struct ib_sa_mcmember_rec { #define IB_DEFAULT_SERVICE_LEASE 0xFFFFFFFF -struct ib_sa_service_rec { - u64 id; - union ib_gid gid; - __be16 pkey; - /* reserved */ - u32 lease; - u8 key[16]; - u8 name[64]; - u8 data8[16]; - u16 data16[8]; - u32 data32[4]; - u64 data64[2]; -}; - #define IB_SA_GUIDINFO_REC_LID IB_SA_COMP_MASK(0) #define IB_SA_GUIDINFO_REC_BLOCK_NUM IB_SA_COMP_MASK(1) #define IB_SA_GUIDINFO_REC_RES1 IB_SA_COMP_MASK(2) @@ -430,16 +416,6 @@ int ib_sa_path_rec_get(struct ib_sa_client *client, struct ib_device *device, void *context), void *context, struct ib_sa_query **query); -int ib_sa_service_rec_query(struct ib_sa_client *client, - struct ib_device *device, u32 port_num, u8 method, - struct ib_sa_service_rec *rec, - ib_sa_comp_mask comp_mask, unsigned long timeout_ms, - gfp_t gfp_mask, - void (*callback)(int status, - struct ib_sa_service_rec *resp, - void *context), - void *context, struct ib_sa_query **sa_query); - struct ib_sa_multicast { struct ib_sa_mcmember_rec rec; ib_sa_comp_mask comp_mask; -- cgit v1.2.3-59-g8ed1b From 03da1b26fa13b6911ef09e909dad164249410901 Mon Sep 17 00:00:00 2001 From: Li Zhijian Date: Mon, 23 Aug 2021 11:52:46 +0800 Subject: IB/core: Remove deprecated current_seq comments current_seq was removed since the commit below. Fixes: 36f30e486dce ("IB/core: Improve ODP to use hmm_range_fault()") Link: https://lore.kernel.org/r/20210823035246.3506-1-lizhijian@cn.fujitsu.com Signed-off-by: Li Zhijian Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/umem_odp.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c index 9462dbe66014..7a47343d11f9 100644 --- a/drivers/infiniband/core/umem_odp.c +++ b/drivers/infiniband/core/umem_odp.c @@ -292,9 +292,6 @@ EXPORT_SYMBOL(ib_umem_odp_release); * @dma_index: index in the umem to add the dma to. * @page: the page struct to map and add. * @access_mask: access permissions needed for this page. - * @current_seq: sequence number for synchronization with invalidations. - * the sequence number is taken from - * umem_odp->notifiers_seq. * * The function returns -EFAULT if the DMA mapping operation fails. * -- cgit v1.2.3-59-g8ed1b From 90e7a6de62781c27d6a111fccfb19b807f9b6887 Mon Sep 17 00:00:00 2001 From: Maor Gottlieb Date: Tue, 24 Aug 2021 17:25:29 +0300 Subject: lib/scatterlist: Provide a dedicated function to support table append RDMA is the only in-kernel user that uses __sg_alloc_table_from_pages to append pages dynamically. In the next patch. That mode will be extended and that function will get more parameters. So separate it into a unique function to make such change more clear. Link: https://lore.kernel.org/r/20210824142531.3877007-2-maorg@nvidia.com Signed-off-by: Maor Gottlieb Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/gpu/drm/drm_prime.c | 13 +++++----- drivers/gpu/drm/i915/gem/i915_gem_userptr.c | 11 +++----- drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c | 14 ++++------- drivers/infiniband/core/umem.c | 4 +-- include/linux/scatterlist.h | 39 ++++++++++++++++++++++++++--- lib/scatterlist.c | 36 ++++++++++++++------------ tools/testing/scatterlist/main.c | 25 ++++++++++++------ 7 files changed, 90 insertions(+), 52 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c index 2a54f86856af..cf3278041f9c 100644 --- a/drivers/gpu/drm/drm_prime.c +++ b/drivers/gpu/drm/drm_prime.c @@ -807,8 +807,8 @@ struct sg_table *drm_prime_pages_to_sg(struct drm_device *dev, struct page **pages, unsigned int nr_pages) { struct sg_table *sg; - struct scatterlist *sge; size_t max_segment = 0; + int err; sg = kmalloc(sizeof(struct sg_table), GFP_KERNEL); if (!sg) @@ -818,13 +818,12 @@ struct sg_table *drm_prime_pages_to_sg(struct drm_device *dev, max_segment = dma_max_mapping_size(dev->dev); if (max_segment == 0) max_segment = UINT_MAX; - sge = __sg_alloc_table_from_pages(sg, pages, nr_pages, 0, - nr_pages << PAGE_SHIFT, - max_segment, - NULL, 0, GFP_KERNEL); - if (IS_ERR(sge)) { + err = sg_alloc_table_from_pages_segment(sg, pages, nr_pages, 0, + nr_pages << PAGE_SHIFT, + max_segment, GFP_KERNEL); + if (err) { kfree(sg); - sg = ERR_CAST(sge); + sg = ERR_PTR(err); } return sg; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c index 7487bab11f0b..458f797a9e1e 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c @@ -133,7 +133,6 @@ static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) unsigned int max_segment = i915_sg_segment_size(); struct sg_table *st; unsigned int sg_page_sizes; - struct scatterlist *sg; struct page **pvec; int ret; @@ -153,13 +152,11 @@ static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) spin_unlock(&i915->mm.notifier_lock); alloc_table: - sg = __sg_alloc_table_from_pages(st, pvec, num_pages, 0, - num_pages << PAGE_SHIFT, max_segment, - NULL, 0, GFP_KERNEL); - if (IS_ERR(sg)) { - ret = PTR_ERR(sg); + ret = sg_alloc_table_from_pages_segment(st, pvec, num_pages, 0, + num_pages << PAGE_SHIFT, + max_segment, GFP_KERNEL); + if (ret) goto err; - } ret = i915_gem_gtt_prepare_pages(obj, st); if (ret) { diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c index 0488042fb287..fc372d2e52a1 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c @@ -363,7 +363,6 @@ static int vmw_ttm_map_dma(struct vmw_ttm_tt *vmw_tt) int ret = 0; static size_t sgl_size; static size_t sgt_size; - struct scatterlist *sg; if (vmw_tt->mapped) return 0; @@ -386,15 +385,12 @@ static int vmw_ttm_map_dma(struct vmw_ttm_tt *vmw_tt) if (unlikely(ret != 0)) return ret; - sg = __sg_alloc_table_from_pages(&vmw_tt->sgt, vsgt->pages, - vsgt->num_pages, 0, - (unsigned long) vsgt->num_pages << PAGE_SHIFT, - dma_get_max_seg_size(dev_priv->drm.dev), - NULL, 0, GFP_KERNEL); - if (IS_ERR(sg)) { - ret = PTR_ERR(sg); + ret = sg_alloc_table_from_pages_segment( + &vmw_tt->sgt, vsgt->pages, vsgt->num_pages, 0, + (unsigned long)vsgt->num_pages << PAGE_SHIFT, + dma_get_max_seg_size(dev_priv->drm.dev), GFP_KERNEL); + if (ret) goto out_sg_alloc_fail; - } if (vsgt->num_pages > vmw_tt->sgt.orig_nents) { uint64_t over_alloc = diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index 0eb40025075f..b741758e528f 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -226,8 +226,8 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, cur_base += ret * PAGE_SIZE; npages -= ret; - sg = __sg_alloc_table_from_pages(&umem->sg_head, page_list, ret, - 0, ret << PAGE_SHIFT, + sg = sg_alloc_append_table_from_pages(&umem->sg_head, page_list, + ret, 0, ret << PAGE_SHIFT, ib_dma_max_seg_size(device), sg, npages, GFP_KERNEL); umem->sg_nents = umem->sg_head.nents; diff --git a/include/linux/scatterlist.h b/include/linux/scatterlist.h index ecf87484814f..5c700f2a0d18 100644 --- a/include/linux/scatterlist.h +++ b/include/linux/scatterlist.h @@ -285,14 +285,45 @@ void sg_free_table(struct sg_table *); int __sg_alloc_table(struct sg_table *, unsigned int, unsigned int, struct scatterlist *, unsigned int, gfp_t, sg_alloc_fn *); int sg_alloc_table(struct sg_table *, unsigned int, gfp_t); -struct scatterlist *__sg_alloc_table_from_pages(struct sg_table *sgt, +struct scatterlist *sg_alloc_append_table_from_pages(struct sg_table *sgt, struct page **pages, unsigned int n_pages, unsigned int offset, unsigned long size, unsigned int max_segment, struct scatterlist *prv, unsigned int left_pages, gfp_t gfp_mask); -int sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages, - unsigned int n_pages, unsigned int offset, - unsigned long size, gfp_t gfp_mask); +int sg_alloc_table_from_pages_segment(struct sg_table *sgt, struct page **pages, + unsigned int n_pages, unsigned int offset, + unsigned long size, + unsigned int max_segment, gfp_t gfp_mask); + +/** + * sg_alloc_table_from_pages - Allocate and initialize an sg table from + * an array of pages + * @sgt: The sg table header to use + * @pages: Pointer to an array of page pointers + * @n_pages: Number of pages in the pages array + * @offset: Offset from start of the first page to the start of a buffer + * @size: Number of valid bytes in the buffer (after offset) + * @gfp_mask: GFP allocation mask + * + * Description: + * Allocate and initialize an sg table from a list of pages. Contiguous + * ranges of the pages are squashed into a single scatterlist node. A user + * may provide an offset at a start and a size of valid data in a buffer + * specified by the page array. The returned sg table is released by + * sg_free_table. + * + * Returns: + * 0 on success, negative error on failure + */ +static inline int sg_alloc_table_from_pages(struct sg_table *sgt, + struct page **pages, + unsigned int n_pages, + unsigned int offset, + unsigned long size, gfp_t gfp_mask) +{ + return sg_alloc_table_from_pages_segment(sgt, pages, n_pages, offset, + size, UINT_MAX, gfp_mask); +} #ifdef CONFIG_SGL_ALLOC struct scatterlist *sgl_alloc_order(unsigned long long length, diff --git a/lib/scatterlist.c b/lib/scatterlist.c index 27efa6178153..611c63d4a958 100644 --- a/lib/scatterlist.c +++ b/lib/scatterlist.c @@ -397,7 +397,7 @@ static struct scatterlist *get_next_sg(struct sg_table *table, } /** - * __sg_alloc_table_from_pages - Allocate and initialize an sg table from + * sg_alloc_append_table_from_pages - Allocate and initialize an sg table from * an array of pages * @sgt: The sg table header to use * @pages: Pointer to an array of page pointers @@ -425,7 +425,7 @@ static struct scatterlist *get_next_sg(struct sg_table *table, * If this function returns non-0 (eg failure), the caller must call * sg_free_table() to cleanup any leftover allocations. */ -struct scatterlist *__sg_alloc_table_from_pages(struct sg_table *sgt, +struct scatterlist *sg_alloc_append_table_from_pages(struct sg_table *sgt, struct page **pages, unsigned int n_pages, unsigned int offset, unsigned long size, unsigned int max_segment, struct scatterlist *prv, unsigned int left_pages, @@ -520,36 +520,40 @@ out: sg_mark_end(s); return s; } -EXPORT_SYMBOL(__sg_alloc_table_from_pages); +EXPORT_SYMBOL(sg_alloc_append_table_from_pages); /** - * sg_alloc_table_from_pages - Allocate and initialize an sg table from - * an array of pages + * sg_alloc_table_from_pages_segment - Allocate and initialize an sg table from + * an array of pages and given maximum + * segment. * @sgt: The sg table header to use * @pages: Pointer to an array of page pointers * @n_pages: Number of pages in the pages array * @offset: Offset from start of the first page to the start of a buffer * @size: Number of valid bytes in the buffer (after offset) + * @max_segment: Maximum size of a scatterlist element in bytes * @gfp_mask: GFP allocation mask * * Description: * Allocate and initialize an sg table from a list of pages. Contiguous - * ranges of the pages are squashed into a single scatterlist node. A user - * may provide an offset at a start and a size of valid data in a buffer - * specified by the page array. The returned sg table is released by - * sg_free_table. + * ranges of the pages are squashed into a single scatterlist node up to the + * maximum size specified in @max_segment. A user may provide an offset at a + * start and a size of valid data in a buffer specified by the page array. * - * Returns: + * The returned sg table is released by sg_free_table. + * + * Returns: * 0 on success, negative error on failure */ -int sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages, - unsigned int n_pages, unsigned int offset, - unsigned long size, gfp_t gfp_mask) +int sg_alloc_table_from_pages_segment(struct sg_table *sgt, struct page **pages, + unsigned int n_pages, unsigned int offset, + unsigned long size, unsigned int max_segment, + gfp_t gfp_mask) { - return PTR_ERR_OR_ZERO(__sg_alloc_table_from_pages(sgt, pages, n_pages, - offset, size, UINT_MAX, NULL, 0, gfp_mask)); + return PTR_ERR_OR_ZERO(sg_alloc_append_table_from_pages(sgt, pages, + n_pages, offset, size, max_segment, NULL, 0, gfp_mask)); } -EXPORT_SYMBOL(sg_alloc_table_from_pages); +EXPORT_SYMBOL(sg_alloc_table_from_pages_segment); #ifdef CONFIG_SGL_ALLOC diff --git a/tools/testing/scatterlist/main.c b/tools/testing/scatterlist/main.c index 652254754b4c..c2ff9179c2cc 100644 --- a/tools/testing/scatterlist/main.c +++ b/tools/testing/scatterlist/main.c @@ -87,28 +87,39 @@ int main(void) int left_pages = test->pfn_app ? test->num_pages : 0; struct page *pages[MAX_PAGES]; struct sg_table st; - struct scatterlist *sg; + struct scatterlist *sg = NULL; + int ret; set_pages(pages, test->pfn, test->num_pages); - sg = __sg_alloc_table_from_pages(&st, pages, test->num_pages, 0, - test->size, test->max_seg, NULL, left_pages, GFP_KERNEL); - assert(PTR_ERR_OR_ZERO(sg) == test->alloc_ret); + if (test->pfn_app) { + sg = sg_alloc_append_table_from_pages( + &st, pages, test->num_pages, 0, test->size, + test->max_seg, NULL, left_pages, GFP_KERNEL); + assert(PTR_ERR_OR_ZERO(sg) == test->alloc_ret); + } else { + ret = sg_alloc_table_from_pages_segment( + &st, pages, test->num_pages, 0, test->size, + test->max_seg, GFP_KERNEL); + assert(ret == test->alloc_ret); + } if (test->alloc_ret) continue; if (test->pfn_app) { set_pages(pages, test->pfn_app, test->num_pages); - sg = __sg_alloc_table_from_pages(&st, pages, test->num_pages, 0, - test->size, test->max_seg, sg, 0, GFP_KERNEL); + sg = sg_alloc_append_table_from_pages( + &st, pages, test->num_pages, 0, test->size, + test->max_seg, sg, 0, GFP_KERNEL); assert(PTR_ERR_OR_ZERO(sg) == test->alloc_ret); } VALIDATE(st.nents == test->expected_segments, &st, test); if (!test->pfn_app) - VALIDATE(st.orig_nents == test->expected_segments, &st, test); + VALIDATE(st.orig_nents == test->expected_segments, &st, + test); sg_free_table(&st); } -- cgit v1.2.3-59-g8ed1b From 3e302dbc6774a27edaea39a1d5107f0c12e35cf2 Mon Sep 17 00:00:00 2001 From: Maor Gottlieb Date: Tue, 24 Aug 2021 17:25:30 +0300 Subject: lib/scatterlist: Fix wrong update of orig_nents orig_nents should represent the number of entries with pages, but __sg_alloc_table_from_pages sets orig_nents as the number of total entries in the table. This is wrong when the API is used for dynamic allocation where not all the table entries are mapped with pages. It wasn't observed until now, since RDMA umem who uses this API in the dynamic form doesn't use orig_nents implicit or explicit by the scatterlist APIs. Fix it by changing the append API to track the SG append table state and have an API to free the append table according to the total number of entries in the table. Now all APIs set orig_nents as number of enries with pages. Fixes: 07da1223ec93 ("lib/scatterlist: Add support in dynamic allocation of SG table from pages") Link: https://lore.kernel.org/r/20210824142531.3877007-3-maorg@nvidia.com Signed-off-by: Maor Gottlieb Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/umem.c | 34 ++++++----- include/linux/scatterlist.h | 19 ++++-- include/rdma/ib_umem.h | 1 + lib/scatterlist.c | 127 ++++++++++++++++++++++++--------------- lib/sg_pool.c | 3 +- tools/testing/scatterlist/main.c | 45 +++++++------- 6 files changed, 136 insertions(+), 93 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index b741758e528f..42481e7a72e8 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -59,7 +59,7 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int d unpin_user_page_range_dirty_lock(sg_page(sg), DIV_ROUND_UP(sg->length, PAGE_SIZE), make_dirty); - sg_free_table(&umem->sg_head); + sg_free_append_table(&umem->sgt_append); } /** @@ -155,8 +155,7 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, unsigned long dma_attr = 0; struct mm_struct *mm; unsigned long npages; - int ret; - struct scatterlist *sg = NULL; + int pinned, ret; unsigned int gup_flags = FOLL_WRITE; /* @@ -216,28 +215,33 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, while (npages) { cond_resched(); - ret = pin_user_pages_fast(cur_base, + pinned = pin_user_pages_fast(cur_base, min_t(unsigned long, npages, PAGE_SIZE / sizeof(struct page *)), gup_flags | FOLL_LONGTERM, page_list); - if (ret < 0) + if (pinned < 0) { + ret = pinned; goto umem_release; + } - cur_base += ret * PAGE_SIZE; - npages -= ret; - sg = sg_alloc_append_table_from_pages(&umem->sg_head, page_list, - ret, 0, ret << PAGE_SHIFT, - ib_dma_max_seg_size(device), sg, npages, - GFP_KERNEL); - umem->sg_nents = umem->sg_head.nents; - if (IS_ERR(sg)) { - unpin_user_pages_dirty_lock(page_list, ret, 0); - ret = PTR_ERR(sg); + cur_base += pinned * PAGE_SIZE; + npages -= pinned; + ret = sg_alloc_append_table_from_pages( + &umem->sgt_append, page_list, pinned, 0, + pinned << PAGE_SHIFT, ib_dma_max_seg_size(device), + npages, GFP_KERNEL); + umem->sg_nents = umem->sgt_append.sgt.nents; + if (ret) { + memcpy(&umem->sg_head.sgl, &umem->sgt_append.sgt, + sizeof(umem->sgt_append.sgt)); + unpin_user_pages_dirty_lock(page_list, pinned, 0); goto umem_release; } } + memcpy(&umem->sg_head.sgl, &umem->sgt_append.sgt, + sizeof(umem->sgt_append.sgt)); if (access & IB_ACCESS_RELAXED_ORDERING) dma_attr |= DMA_ATTR_WEAK_ORDERING; diff --git a/include/linux/scatterlist.h b/include/linux/scatterlist.h index 5c700f2a0d18..266754a55327 100644 --- a/include/linux/scatterlist.h +++ b/include/linux/scatterlist.h @@ -39,6 +39,12 @@ struct sg_table { unsigned int orig_nents; /* original size of list */ }; +struct sg_append_table { + struct sg_table sgt; /* The scatter list table */ + struct scatterlist *prv; /* last populated sge in the table */ + unsigned int total_nents; /* Total entries in the table */ +}; + /* * Notes on SG table design. * @@ -280,16 +286,17 @@ typedef struct scatterlist *(sg_alloc_fn)(unsigned int, gfp_t); typedef void (sg_free_fn)(struct scatterlist *, unsigned int); void __sg_free_table(struct sg_table *, unsigned int, unsigned int, - sg_free_fn *); + sg_free_fn *, unsigned int); void sg_free_table(struct sg_table *); +void sg_free_append_table(struct sg_append_table *sgt); int __sg_alloc_table(struct sg_table *, unsigned int, unsigned int, struct scatterlist *, unsigned int, gfp_t, sg_alloc_fn *); int sg_alloc_table(struct sg_table *, unsigned int, gfp_t); -struct scatterlist *sg_alloc_append_table_from_pages(struct sg_table *sgt, - struct page **pages, unsigned int n_pages, unsigned int offset, - unsigned long size, unsigned int max_segment, - struct scatterlist *prv, unsigned int left_pages, - gfp_t gfp_mask); +int sg_alloc_append_table_from_pages(struct sg_append_table *sgt, + struct page **pages, unsigned int n_pages, + unsigned int offset, unsigned long size, + unsigned int max_segment, + unsigned int left_pages, gfp_t gfp_mask); int sg_alloc_table_from_pages_segment(struct sg_table *sgt, struct page **pages, unsigned int n_pages, unsigned int offset, unsigned long size, diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h index 676c57f5ca80..33cb23b2ee3c 100644 --- a/include/rdma/ib_umem.h +++ b/include/rdma/ib_umem.h @@ -26,6 +26,7 @@ struct ib_umem { u32 is_odp : 1; u32 is_dmabuf : 1; struct work_struct work; + struct sg_append_table sgt_append; struct sg_table sg_head; int nmap; unsigned int sg_nents; diff --git a/lib/scatterlist.c b/lib/scatterlist.c index 611c63d4a958..f4b1ff78ab2d 100644 --- a/lib/scatterlist.c +++ b/lib/scatterlist.c @@ -182,6 +182,7 @@ static void sg_kfree(struct scatterlist *sg, unsigned int nents) * @nents_first_chunk: Number of entries int the (preallocated) first * scatterlist chunk, 0 means no such preallocated first chunk * @free_fn: Free function + * @num_ents: Number of entries in the table * * Description: * Free an sg table previously allocated and setup with @@ -190,7 +191,8 @@ static void sg_kfree(struct scatterlist *sg, unsigned int nents) * **/ void __sg_free_table(struct sg_table *table, unsigned int max_ents, - unsigned int nents_first_chunk, sg_free_fn *free_fn) + unsigned int nents_first_chunk, sg_free_fn *free_fn, + unsigned int num_ents) { struct scatterlist *sgl, *next; unsigned curr_max_ents = nents_first_chunk ?: max_ents; @@ -199,8 +201,8 @@ void __sg_free_table(struct sg_table *table, unsigned int max_ents, return; sgl = table->sgl; - while (table->orig_nents) { - unsigned int alloc_size = table->orig_nents; + while (num_ents) { + unsigned int alloc_size = num_ents; unsigned int sg_size; /* @@ -218,7 +220,7 @@ void __sg_free_table(struct sg_table *table, unsigned int max_ents, next = NULL; } - table->orig_nents -= sg_size; + num_ents -= sg_size; if (nents_first_chunk) nents_first_chunk = 0; else @@ -231,6 +233,19 @@ void __sg_free_table(struct sg_table *table, unsigned int max_ents, } EXPORT_SYMBOL(__sg_free_table); +/** + * sg_free_append_table - Free a previously allocated append sg table. + * @table: The mapped sg append table header + * + **/ +void sg_free_append_table(struct sg_append_table *table) +{ + __sg_free_table(&table->sgt, SG_MAX_SINGLE_ALLOC, false, sg_kfree, + table->total_nents); +} +EXPORT_SYMBOL(sg_free_append_table); + + /** * sg_free_table - Free a previously allocated sg table * @table: The mapped sg table header @@ -238,7 +253,8 @@ EXPORT_SYMBOL(__sg_free_table); **/ void sg_free_table(struct sg_table *table) { - __sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, sg_kfree); + __sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, sg_kfree, + table->orig_nents); } EXPORT_SYMBOL(sg_free_table); @@ -359,13 +375,12 @@ int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC, NULL, 0, gfp_mask, sg_kmalloc); if (unlikely(ret)) - __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); - + sg_free_table(table); return ret; } EXPORT_SYMBOL(sg_alloc_table); -static struct scatterlist *get_next_sg(struct sg_table *table, +static struct scatterlist *get_next_sg(struct sg_append_table *table, struct scatterlist *cur, unsigned long needed_sges, gfp_t gfp_mask) @@ -386,54 +401,52 @@ static struct scatterlist *get_next_sg(struct sg_table *table, return ERR_PTR(-ENOMEM); sg_init_table(new_sg, alloc_size); if (cur) { + table->total_nents += alloc_size - 1; __sg_chain(next_sg, new_sg); - table->orig_nents += alloc_size - 1; } else { - table->sgl = new_sg; - table->orig_nents = alloc_size; - table->nents = 0; + table->sgt.sgl = new_sg; + table->total_nents = alloc_size; } return new_sg; } /** - * sg_alloc_append_table_from_pages - Allocate and initialize an sg table from - * an array of pages - * @sgt: The sg table header to use - * @pages: Pointer to an array of page pointers - * @n_pages: Number of pages in the pages array + * sg_alloc_append_table_from_pages - Allocate and initialize an append sg + * table from an array of pages + * @sgt_append: The sg append table to use + * @pages: Pointer to an array of page pointers + * @n_pages: Number of pages in the pages array * @offset: Offset from start of the first page to the start of a buffer * @size: Number of valid bytes in the buffer (after offset) * @max_segment: Maximum size of a scatterlist element in bytes - * @prv: Last populated sge in sgt * @left_pages: Left pages caller have to set after this call * @gfp_mask: GFP allocation mask * * Description: - * If @prv is NULL, allocate and initialize an sg table from a list of pages, - * else reuse the scatterlist passed in at @prv. - * Contiguous ranges of the pages are squashed into a single scatterlist - * entry up to the maximum size specified in @max_segment. A user may - * provide an offset at a start and a size of valid data in a buffer - * specified by the page array. + * In the first call it allocate and initialize an sg table from a list of + * pages, else reuse the scatterlist from sgt_append. Contiguous ranges of + * the pages are squashed into a single scatterlist entry up to the maximum + * size specified in @max_segment. A user may provide an offset at a start + * and a size of valid data in a buffer specified by the page array. The + * returned sg table is released by sg_free_append_table * * Returns: - * Last SGE in sgt on success, PTR_ERR on otherwise. - * The allocation in @sgt must be released by sg_free_table. + * 0 on success, negative error on failure * * Notes: * If this function returns non-0 (eg failure), the caller must call - * sg_free_table() to cleanup any leftover allocations. + * sg_free_append_table() to cleanup any leftover allocations. + * + * In the fist call, sgt_append must by initialized. */ -struct scatterlist *sg_alloc_append_table_from_pages(struct sg_table *sgt, +int sg_alloc_append_table_from_pages(struct sg_append_table *sgt_append, struct page **pages, unsigned int n_pages, unsigned int offset, unsigned long size, unsigned int max_segment, - struct scatterlist *prv, unsigned int left_pages, - gfp_t gfp_mask) + unsigned int left_pages, gfp_t gfp_mask) { unsigned int chunks, cur_page, seg_len, i, prv_len = 0; unsigned int added_nents = 0; - struct scatterlist *s = prv; + struct scatterlist *s = sgt_append->prv; /* * The algorithm below requires max_segment to be aligned to PAGE_SIZE @@ -441,25 +454,26 @@ struct scatterlist *sg_alloc_append_table_from_pages(struct sg_table *sgt, */ max_segment = ALIGN_DOWN(max_segment, PAGE_SIZE); if (WARN_ON(max_segment < PAGE_SIZE)) - return ERR_PTR(-EINVAL); + return -EINVAL; - if (IS_ENABLED(CONFIG_ARCH_NO_SG_CHAIN) && prv) - return ERR_PTR(-EOPNOTSUPP); + if (IS_ENABLED(CONFIG_ARCH_NO_SG_CHAIN) && sgt_append->prv) + return -EOPNOTSUPP; - if (prv) { - unsigned long paddr = (page_to_pfn(sg_page(prv)) * PAGE_SIZE + - prv->offset + prv->length) / - PAGE_SIZE; + if (sgt_append->prv) { + unsigned long paddr = + (page_to_pfn(sg_page(sgt_append->prv)) * PAGE_SIZE + + sgt_append->prv->offset + sgt_append->prv->length) / + PAGE_SIZE; if (WARN_ON(offset)) - return ERR_PTR(-EINVAL); + return -EINVAL; /* Merge contiguous pages into the last SG */ - prv_len = prv->length; + prv_len = sgt_append->prv->length; while (n_pages && page_to_pfn(pages[0]) == paddr) { - if (prv->length + PAGE_SIZE > max_segment) + if (sgt_append->prv->length + PAGE_SIZE > max_segment) break; - prv->length += PAGE_SIZE; + sgt_append->prv->length += PAGE_SIZE; paddr++; pages++; n_pages--; @@ -496,15 +510,16 @@ struct scatterlist *sg_alloc_append_table_from_pages(struct sg_table *sgt, } /* Pass how many chunks might be left */ - s = get_next_sg(sgt, s, chunks - i + left_pages, gfp_mask); + s = get_next_sg(sgt_append, s, chunks - i + left_pages, + gfp_mask); if (IS_ERR(s)) { /* * Adjust entry length to be as before function was * called. */ - if (prv) - prv->length = prv_len; - return s; + if (sgt_append->prv) + sgt_append->prv->length = prv_len; + return PTR_ERR(s); } chunk_size = ((j - cur_page) << PAGE_SHIFT) - offset; sg_set_page(s, pages[cur_page], @@ -514,11 +529,13 @@ struct scatterlist *sg_alloc_append_table_from_pages(struct sg_table *sgt, offset = 0; cur_page = j; } - sgt->nents += added_nents; + sgt_append->sgt.nents += added_nents; + sgt_append->sgt.orig_nents = sgt_append->sgt.nents; + sgt_append->prv = s; out: if (!left_pages) sg_mark_end(s); - return s; + return 0; } EXPORT_SYMBOL(sg_alloc_append_table_from_pages); @@ -550,8 +567,18 @@ int sg_alloc_table_from_pages_segment(struct sg_table *sgt, struct page **pages, unsigned long size, unsigned int max_segment, gfp_t gfp_mask) { - return PTR_ERR_OR_ZERO(sg_alloc_append_table_from_pages(sgt, pages, - n_pages, offset, size, max_segment, NULL, 0, gfp_mask)); + struct sg_append_table append = {}; + int err; + + err = sg_alloc_append_table_from_pages(&append, pages, n_pages, offset, + size, max_segment, 0, gfp_mask); + if (err) { + sg_free_append_table(&append); + return err; + } + memcpy(sgt, &append.sgt, sizeof(*sgt)); + WARN_ON(append.total_nents != sgt->orig_nents); + return 0; } EXPORT_SYMBOL(sg_alloc_table_from_pages_segment); diff --git a/lib/sg_pool.c b/lib/sg_pool.c index db29e5c1f790..a0b1a52cd6f7 100644 --- a/lib/sg_pool.c +++ b/lib/sg_pool.c @@ -90,7 +90,8 @@ void sg_free_table_chained(struct sg_table *table, if (nents_first_chunk == 1) nents_first_chunk = 0; - __sg_free_table(table, SG_CHUNK_SIZE, nents_first_chunk, sg_pool_free); + __sg_free_table(table, SG_CHUNK_SIZE, nents_first_chunk, sg_pool_free, + table->orig_nents); } EXPORT_SYMBOL_GPL(sg_free_table_chained); diff --git a/tools/testing/scatterlist/main.c b/tools/testing/scatterlist/main.c index c2ff9179c2cc..08465a701cd5 100644 --- a/tools/testing/scatterlist/main.c +++ b/tools/testing/scatterlist/main.c @@ -85,43 +85,46 @@ int main(void) for (i = 0, test = tests; test->expected_segments; test++, i++) { int left_pages = test->pfn_app ? test->num_pages : 0; + struct sg_append_table append = {}; struct page *pages[MAX_PAGES]; - struct sg_table st; - struct scatterlist *sg = NULL; int ret; set_pages(pages, test->pfn, test->num_pages); - if (test->pfn_app) { - sg = sg_alloc_append_table_from_pages( - &st, pages, test->num_pages, 0, test->size, - test->max_seg, NULL, left_pages, GFP_KERNEL); - assert(PTR_ERR_OR_ZERO(sg) == test->alloc_ret); - } else { + if (test->pfn_app) + ret = sg_alloc_append_table_from_pages( + &append, pages, test->num_pages, 0, test->size, + test->max_seg, left_pages, GFP_KERNEL); + else ret = sg_alloc_table_from_pages_segment( - &st, pages, test->num_pages, 0, test->size, - test->max_seg, GFP_KERNEL); - assert(ret == test->alloc_ret); - } + &append.sgt, pages, test->num_pages, 0, + test->size, test->max_seg, GFP_KERNEL); + + assert(ret == test->alloc_ret); if (test->alloc_ret) continue; if (test->pfn_app) { set_pages(pages, test->pfn_app, test->num_pages); - sg = sg_alloc_append_table_from_pages( - &st, pages, test->num_pages, 0, test->size, - test->max_seg, sg, 0, GFP_KERNEL); + ret = sg_alloc_append_table_from_pages( + &append, pages, test->num_pages, 0, test->size, + test->max_seg, 0, GFP_KERNEL); - assert(PTR_ERR_OR_ZERO(sg) == test->alloc_ret); + assert(ret == test->alloc_ret); } - VALIDATE(st.nents == test->expected_segments, &st, test); + VALIDATE(append.sgt.nents == test->expected_segments, + &append.sgt, test); if (!test->pfn_app) - VALIDATE(st.orig_nents == test->expected_segments, &st, - test); - - sg_free_table(&st); + VALIDATE(append.sgt.orig_nents == + test->expected_segments, + &append.sgt, test); + + if (test->pfn_app) + sg_free_append_table(&append); + else + sg_free_table(&append.sgt); } assert(i == (sizeof(tests) / sizeof(tests[0])) - 1); -- cgit v1.2.3-59-g8ed1b From 79fbd3e1241cea83dded06db2b8bcd5893d877d7 Mon Sep 17 00:00:00 2001 From: Maor Gottlieb Date: Tue, 24 Aug 2021 17:25:31 +0300 Subject: RDMA: Use the sg_table directly and remove the opencoded version from umem This allows using the normal sg_table APIs and makes all the code cleaner. Remove sgt, nents and nmapd from ib_umem. Link: https://lore.kernel.org/r/20210824142531.3877007-4-maorg@nvidia.com Signed-off-by: Maor Gottlieb Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/umem.c | 32 +++++++++++--------------------- drivers/infiniband/core/umem_dmabuf.c | 5 ++--- drivers/infiniband/hw/hns/hns_roce_db.c | 4 ++-- drivers/infiniband/hw/irdma/verbs.c | 2 +- drivers/infiniband/hw/mlx4/doorbell.c | 3 ++- drivers/infiniband/hw/mlx4/mr.c | 4 ++-- drivers/infiniband/hw/mlx5/doorbell.c | 3 ++- drivers/infiniband/hw/mlx5/mr.c | 3 ++- drivers/infiniband/hw/qedr/verbs.c | 2 +- drivers/infiniband/sw/rdmavt/mr.c | 2 +- drivers/infiniband/sw/rxe/rxe_mr.c | 2 +- include/rdma/ib_umem.h | 12 +++++------- include/rdma/ib_verbs.h | 28 ++++++++++++++++++++++++++++ 13 files changed, 60 insertions(+), 42 deletions(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index 42481e7a72e8..86d479772fbc 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -51,11 +51,11 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int d struct scatterlist *sg; unsigned int i; - if (umem->nmap > 0) - ib_dma_unmap_sg(dev, umem->sg_head.sgl, umem->sg_nents, - DMA_BIDIRECTIONAL); + if (dirty) + ib_dma_unmap_sgtable_attrs(dev, &umem->sgt_append.sgt, + DMA_BIDIRECTIONAL, 0); - for_each_sg(umem->sg_head.sgl, sg, umem->sg_nents, i) + for_each_sgtable_sg(&umem->sgt_append.sgt, sg, i) unpin_user_page_range_dirty_lock(sg_page(sg), DIV_ROUND_UP(sg->length, PAGE_SIZE), make_dirty); @@ -111,7 +111,7 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, /* offset into first SGL */ pgoff = umem->address & ~PAGE_MASK; - for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i) { + for_each_sgtable_dma_sg(&umem->sgt_append.sgt, sg, i) { /* Walk SGL and reduce max page size if VA/PA bits differ * for any address. */ @@ -121,7 +121,7 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, * the maximum possible page size as the low bits of the iova * must be zero when starting the next chunk. */ - if (i != (umem->nmap - 1)) + if (i != (umem->sgt_append.sgt.nents - 1)) mask |= va; pgoff = 0; } @@ -231,30 +231,19 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, &umem->sgt_append, page_list, pinned, 0, pinned << PAGE_SHIFT, ib_dma_max_seg_size(device), npages, GFP_KERNEL); - umem->sg_nents = umem->sgt_append.sgt.nents; if (ret) { - memcpy(&umem->sg_head.sgl, &umem->sgt_append.sgt, - sizeof(umem->sgt_append.sgt)); unpin_user_pages_dirty_lock(page_list, pinned, 0); goto umem_release; } } - memcpy(&umem->sg_head.sgl, &umem->sgt_append.sgt, - sizeof(umem->sgt_append.sgt)); if (access & IB_ACCESS_RELAXED_ORDERING) dma_attr |= DMA_ATTR_WEAK_ORDERING; - umem->nmap = - ib_dma_map_sg_attrs(device, umem->sg_head.sgl, umem->sg_nents, - DMA_BIDIRECTIONAL, dma_attr); - - if (!umem->nmap) { - ret = -ENOMEM; + ret = ib_dma_map_sgtable_attrs(device, &umem->sgt_append.sgt, + DMA_BIDIRECTIONAL, dma_attr); + if (ret) goto umem_release; - } - - ret = 0; goto out; umem_release: @@ -314,7 +303,8 @@ int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset, return -EINVAL; } - ret = sg_pcopy_to_buffer(umem->sg_head.sgl, umem->sg_nents, dst, length, + ret = sg_pcopy_to_buffer(umem->sgt_append.sgt.sgl, + umem->sgt_append.sgt.orig_nents, dst, length, offset + ib_umem_offset(umem)); if (ret < 0) diff --git a/drivers/infiniband/core/umem_dmabuf.c b/drivers/infiniband/core/umem_dmabuf.c index c6e875619fac..e824baf4640d 100644 --- a/drivers/infiniband/core/umem_dmabuf.c +++ b/drivers/infiniband/core/umem_dmabuf.c @@ -55,9 +55,8 @@ int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf) cur += sg_dma_len(sg); } - umem_dmabuf->umem.sg_head.sgl = umem_dmabuf->first_sg; - umem_dmabuf->umem.sg_head.nents = nmap; - umem_dmabuf->umem.nmap = nmap; + umem_dmabuf->umem.sgt_append.sgt.sgl = umem_dmabuf->first_sg; + umem_dmabuf->umem.sgt_append.sgt.nents = nmap; umem_dmabuf->sgt = sgt; wait_fence: diff --git a/drivers/infiniband/hw/hns/hns_roce_db.c b/drivers/infiniband/hw/hns/hns_roce_db.c index d40ea3d87260..751470c7a2ce 100644 --- a/drivers/infiniband/hw/hns/hns_roce_db.c +++ b/drivers/infiniband/hw/hns/hns_roce_db.c @@ -42,8 +42,8 @@ int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt, found: offset = virt - page_addr; - db->dma = sg_dma_address(page->umem->sg_head.sgl) + offset; - db->virt_addr = sg_virt(page->umem->sg_head.sgl) + offset; + db->dma = sg_dma_address(page->umem->sgt_append.sgt.sgl) + offset; + db->virt_addr = sg_virt(page->umem->sgt_append.sgt.sgl) + offset; db->u.user_page = page; refcount_inc(&page->refcount); diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index 717147ed0519..e2114f2134bb 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -2235,7 +2235,7 @@ static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc->level2.leaf; if (iwmr->type == IRDMA_MEMREG_TYPE_QP) - iwpbl->qp_mr.sq_page = sg_page(region->sg_head.sgl); + iwpbl->qp_mr.sq_page = sg_page(region->sgt_append.sgt.sgl); rdma_umem_for_each_dma_block(region, &biter, iwmr->page_size) { *pbl = rdma_block_iter_dma_address(&biter); diff --git a/drivers/infiniband/hw/mlx4/doorbell.c b/drivers/infiniband/hw/mlx4/doorbell.c index d41f03ccb0e1..9bbd695a9fd5 100644 --- a/drivers/infiniband/hw/mlx4/doorbell.c +++ b/drivers/infiniband/hw/mlx4/doorbell.c @@ -75,7 +75,8 @@ int mlx4_ib_db_map_user(struct ib_udata *udata, unsigned long virt, list_add(&page->list, &context->db_page_list); found: - db->dma = sg_dma_address(page->umem->sg_head.sgl) + (virt & ~PAGE_MASK); + db->dma = sg_dma_address(page->umem->sgt_append.sgt.sgl) + + (virt & ~PAGE_MASK); db->u.user_page = page; ++page->refcnt; diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c index 50becc0e4b62..04a67b481608 100644 --- a/drivers/infiniband/hw/mlx4/mr.c +++ b/drivers/infiniband/hw/mlx4/mr.c @@ -200,7 +200,7 @@ int mlx4_ib_umem_write_mtt(struct mlx4_ib_dev *dev, struct mlx4_mtt *mtt, mtt_shift = mtt->page_shift; mtt_size = 1ULL << mtt_shift; - for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i) { + for_each_sgtable_dma_sg(&umem->sgt_append.sgt, sg, i) { if (cur_start_addr + len == sg_dma_address(sg)) { /* still the same block */ len += sg_dma_len(sg); @@ -273,7 +273,7 @@ int mlx4_ib_umem_calc_optimal_mtt_size(struct ib_umem *umem, u64 start_va, *num_of_mtts = ib_umem_num_dma_blocks(umem, PAGE_SIZE); - for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i) { + for_each_sgtable_dma_sg(&umem->sgt_append.sgt, sg, i) { /* * Initialization - save the first chunk start as the * current_block_start - block means contiguous pages. diff --git a/drivers/infiniband/hw/mlx5/doorbell.c b/drivers/infiniband/hw/mlx5/doorbell.c index 9ca2e61807ec..6398e2f48579 100644 --- a/drivers/infiniband/hw/mlx5/doorbell.c +++ b/drivers/infiniband/hw/mlx5/doorbell.c @@ -78,7 +78,8 @@ int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, unsigned long virt, list_add(&page->list, &context->db_page_list); found: - db->dma = sg_dma_address(page->umem->sg_head.sgl) + (virt & ~PAGE_MASK); + db->dma = sg_dma_address(page->umem->sgt_append.sgt.sgl) + + (virt & ~PAGE_MASK); db->u.user_page = page; ++page->refcnt; diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 3f1c5a4f158b..a520ac8ab68c 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -1226,7 +1226,8 @@ int mlx5_ib_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags) orig_sg_length = sg.length; cur_mtt = mtt; - rdma_for_each_block (mr->umem->sg_head.sgl, &biter, mr->umem->nmap, + rdma_for_each_block (mr->umem->sgt_append.sgt.sgl, &biter, + mr->umem->sgt_append.sgt.nents, BIT(mr->page_shift)) { if (cur_mtt == (void *)mtt + sg.length) { dma_sync_single_for_device(ddev, sg.addr, sg.length, diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c index fdc47ef7d861..f23d324bd5e1 100644 --- a/drivers/infiniband/hw/qedr/verbs.c +++ b/drivers/infiniband/hw/qedr/verbs.c @@ -1481,7 +1481,7 @@ static int qedr_init_srq_user_params(struct ib_udata *udata, return PTR_ERR(srq->prod_umem); } - sg = srq->prod_umem->sg_head.sgl; + sg = srq->prod_umem->sgt_append.sgt.sgl; srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg); return 0; diff --git a/drivers/infiniband/sw/rdmavt/mr.c b/drivers/infiniband/sw/rdmavt/mr.c index 34b7af6ab9c2..dfb99a56d952 100644 --- a/drivers/infiniband/sw/rdmavt/mr.c +++ b/drivers/infiniband/sw/rdmavt/mr.c @@ -410,7 +410,7 @@ struct ib_mr *rvt_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, mr->mr.page_shift = PAGE_SHIFT; m = 0; n = 0; - for_each_sg_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) { + for_each_sgtable_page (&umem->sgt_append.sgt, &sg_iter, 0) { void *vaddr; vaddr = page_address(sg_page_iter_page(&sg_iter)); diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c index be4bcb420fab..b5fcb14350c7 100644 --- a/drivers/infiniband/sw/rxe/rxe_mr.c +++ b/drivers/infiniband/sw/rxe/rxe_mr.c @@ -143,7 +143,7 @@ int rxe_mr_init_user(struct rxe_pd *pd, u64 start, u64 length, u64 iova, if (length > 0) { buf = map[0]->buf; - for_each_sg_page(umem->sg_head.sgl, &sg_iter, umem->nmap, 0) { + for_each_sgtable_page (&umem->sgt_append.sgt, &sg_iter, 0) { if (num_buf >= RXE_BUF_PER_MAP) { map++; buf = map[0]->buf; diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h index 33cb23b2ee3c..5ae9dff74dac 100644 --- a/include/rdma/ib_umem.h +++ b/include/rdma/ib_umem.h @@ -26,10 +26,7 @@ struct ib_umem { u32 is_odp : 1; u32 is_dmabuf : 1; struct work_struct work; - struct sg_append_table sgt_append; - struct sg_table sg_head; - int nmap; - unsigned int sg_nents; + struct sg_append_table sgt_append; }; struct ib_umem_dmabuf { @@ -57,7 +54,7 @@ static inline int ib_umem_offset(struct ib_umem *umem) static inline unsigned long ib_umem_dma_offset(struct ib_umem *umem, unsigned long pgsz) { - return (sg_dma_address(umem->sg_head.sgl) + ib_umem_offset(umem)) & + return (sg_dma_address(umem->sgt_append.sgt.sgl) + ib_umem_offset(umem)) & (pgsz - 1); } @@ -78,7 +75,8 @@ static inline void __rdma_umem_block_iter_start(struct ib_block_iter *biter, struct ib_umem *umem, unsigned long pgsz) { - __rdma_block_iter_start(biter, umem->sg_head.sgl, umem->nmap, pgsz); + __rdma_block_iter_start(biter, umem->sgt_append.sgt.sgl, + umem->sgt_append.sgt.nents, pgsz); } /** @@ -129,7 +127,7 @@ static inline unsigned long ib_umem_find_best_pgoff(struct ib_umem *umem, unsigned long pgsz_bitmap, u64 pgoff_bitmask) { - struct scatterlist *sg = umem->sg_head.sgl; + struct scatterlist *sg = umem->sgt_append.sgt.sgl; dma_addr_t dma_addr; dma_addr = sg_dma_address(sg) + (umem->address & ~PAGE_MASK); diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 371df1c80aeb..2dba30849731 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -4057,6 +4057,34 @@ static inline void ib_dma_unmap_sg_attrs(struct ib_device *dev, dma_attrs); } +/** + * ib_dma_map_sgtable_attrs - Map a scatter/gather table to DMA addresses + * @dev: The device for which the DMA addresses are to be created + * @sg: The sg_table object describing the buffer + * @direction: The direction of the DMA + * @attrs: Optional DMA attributes for the map operation + */ +static inline int ib_dma_map_sgtable_attrs(struct ib_device *dev, + struct sg_table *sgt, + enum dma_data_direction direction, + unsigned long dma_attrs) +{ + if (ib_uses_virt_dma(dev)) { + ib_dma_virt_map_sg(dev, sgt->sgl, sgt->orig_nents); + return 0; + } + return dma_map_sgtable(dev->dma_device, sgt, direction, dma_attrs); +} + +static inline void ib_dma_unmap_sgtable_attrs(struct ib_device *dev, + struct sg_table *sgt, + enum dma_data_direction direction, + unsigned long dma_attrs) +{ + if (!ib_uses_virt_dma(dev)) + dma_unmap_sgtable(dev->dma_device, sgt, direction, dma_attrs); +} + /** * ib_dma_map_sg - Map a scatter/gather list to DMA addresses * @dev: The device for which the DMA addresses are to be created -- cgit v1.2.3-59-g8ed1b From 5f5a650999d5718af766fc70a120230b04235a6f Mon Sep 17 00:00:00 2001 From: Håkon Bugge Date: Thu, 12 Aug 2021 18:12:35 +0200 Subject: RDMA/core/sa_query: Retry SA queries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A MAD packet is sent as an unreliable datagram (UD). SA requests are sent as MAD packets. As such, SA requests or responses may be silently dropped. IB Core's MAD layer has a timeout and retry mechanism, which amongst other, is used by RDMA CM. But it is not used by SA queries. The lack of retries of SA queries leads to long specified timeout, and error being returned in case of packet loss. The ULP or user-land process has to perform the retry. Fix this by taking advantage of the MAD layer's retry mechanism. First, a check against a zero timeout is added in rdma_resolve_route(). In send_mad(), we set the MAD layer timeout to one tenth of the specified timeout and the number of retries to 10. The special case when timeout is less than 10 is handled. With this fix: # ucmatose -c 1000 -S 1024 -C 1 runs stable on an Infiniband fabric. Without this fix, we see an intermittent behavior and it errors out with: cmatose: event: RDMA_CM_EVENT_ROUTE_ERROR, error: -110 (110 is ETIMEDOUT) Link: https://lore.kernel.org/r/1628784755-28316-1-git-send-email-haakon.bugge@oracle.com Signed-off-by: Håkon Bugge Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/cma.c | 3 +++ drivers/infiniband/core/sa_query.c | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) (limited to 'drivers/infiniband/core') diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index 515a7e95a421..c3f2facafd02 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -3117,6 +3117,9 @@ int rdma_resolve_route(struct rdma_cm_id *id, unsigned long timeout_ms) struct rdma_id_private *id_priv; int ret; + if (!timeout_ms) + return -EINVAL; + id_priv = container_of(id, struct rdma_id_private, id); if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, RDMA_CM_ROUTE_QUERY)) return -EINVAL; diff --git a/drivers/infiniband/core/sa_query.c b/drivers/infiniband/core/sa_query.c index 740499bd4935..a20b8108e160 100644 --- a/drivers/infiniband/core/sa_query.c +++ b/drivers/infiniband/core/sa_query.c @@ -1304,6 +1304,7 @@ static int send_mad(struct ib_sa_query *query, unsigned long timeout_ms, { unsigned long flags; int ret, id; + const int nmbr_sa_query_retries = 10; xa_lock_irqsave(&queries, flags); ret = __xa_alloc(&queries, &id, query, xa_limit_32b, gfp_mask); @@ -1311,7 +1312,13 @@ static int send_mad(struct ib_sa_query *query, unsigned long timeout_ms, if (ret < 0) return ret; - query->mad_buf->timeout_ms = timeout_ms; + query->mad_buf->timeout_ms = timeout_ms / nmbr_sa_query_retries; + query->mad_buf->retries = nmbr_sa_query_retries; + if (!query->mad_buf->timeout_ms) { + /* Special case, very small timeout_ms */ + query->mad_buf->timeout_ms = 1; + query->mad_buf->retries = timeout_ms; + } query->mad_buf->context[0] = query; query->id = id; -- cgit v1.2.3-59-g8ed1b