From 1fb7f8973f51ca1a7ffe61a2c779ed15f57f3d82 Mon Sep 17 00:00:00 2001 From: Mark Bloch Date: Mon, 1 Mar 2021 09:04:20 +0200 Subject: RDMA: Support more than 255 rdma ports Current code uses many different types when dealing with a port of a RDMA device: u8, unsigned int and u32. Switch to u32 to clean up the logic. This allows us to make (at least) the core view consistent and use the same type. Unfortunately not all places can be converted. Many uverbs functions expect port to be u8 so keep those places in order not to break UAPIs. HW/Spec defined values must also not be changed. With the switch to u32 we now can support devices with more than 255 ports. U32_MAX is reserved to make control logic a bit easier to deal with. As a device with U32_MAX ports probably isn't going to happen any time soon this seems like a non issue. When a device with more than 255 ports is created uverbs will report the RDMA device as having 255 ports as this is the max currently supported. The verbs interface is not changed yet because the IBTA spec limits the port size in too many places to be u8 and all applications that relies in verbs won't be able to cope with this change. At this stage, we are extending the interfaces that are using vendor channel solely Once the limitation is lifted mlx5 in switchdev mode will be able to have thousands of SFs created by the device. As the only instance of an RDMA device that reports more than 255 ports will be a representor device and it exposes itself as a RAW Ethernet only device CM/MAD/IPoIB and other ULPs aren't effected by this change and their sysfs/interfaces that are exposes to userspace can remain unchanged. While here cleanup some alignment issues and remove unneeded sanity checks (mainly in rdmavt), Link: https://lore.kernel.org/r/20210301070420.439400-1-leon@kernel.org Signed-off-by: Mark Bloch Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/qib/qib.h | 8 ++++---- drivers/infiniband/hw/qib/qib_mad.c | 4 ++-- drivers/infiniband/hw/qib/qib_qp.c | 4 ++-- drivers/infiniband/hw/qib/qib_sysfs.c | 2 +- drivers/infiniband/hw/qib/qib_verbs.c | 6 +++--- drivers/infiniband/hw/qib/qib_verbs.h | 6 +++--- 6 files changed, 15 insertions(+), 15 deletions(-) (limited to 'drivers/infiniband/hw/qib') diff --git a/drivers/infiniband/hw/qib/qib.h b/drivers/infiniband/hw/qib/qib.h index 35801c6cc068..88497739029e 100644 --- a/drivers/infiniband/hw/qib/qib.h +++ b/drivers/infiniband/hw/qib/qib.h @@ -630,7 +630,7 @@ struct qib_pportdata { u8 rx_pol_inv; u8 hw_pidx; /* physical port index */ - u8 port; /* IB port number and index into dd->pports - 1 */ + u32 port; /* IB port number and index into dd->pports - 1 */ u8 delay_mult; @@ -1200,10 +1200,10 @@ static inline struct qib_pportdata *ppd_from_ibp(struct qib_ibport *ibp) return container_of(ibp, struct qib_pportdata, ibport_data); } -static inline struct qib_ibport *to_iport(struct ib_device *ibdev, u8 port) +static inline struct qib_ibport *to_iport(struct ib_device *ibdev, u32 port) { struct qib_devdata *dd = dd_from_ibdev(ibdev); - unsigned pidx = port - 1; /* IB number port from 1, hdw from 0 */ + u32 pidx = port - 1; /* IB number port from 1, hdw from 0 */ WARN_ON(pidx >= dd->num_pports); return &dd->pport[pidx].ibport_data; @@ -1369,7 +1369,7 @@ extern const struct attribute_group qib_attr_group; int qib_device_create(struct qib_devdata *); void qib_device_remove(struct qib_devdata *); -int qib_create_port_files(struct ib_device *ibdev, u8 port_num, +int qib_create_port_files(struct ib_device *ibdev, u32 port_num, struct kobject *kobj); void qib_verbs_unregister_sysfs(struct qib_devdata *); /* Hook for sysfs read of QSFP */ diff --git a/drivers/infiniband/hw/qib/qib_mad.c b/drivers/infiniband/hw/qib/qib_mad.c index 44e2f813024a..ef02f2bfddb2 100644 --- a/drivers/infiniband/hw/qib/qib_mad.c +++ b/drivers/infiniband/hw/qib/qib_mad.c @@ -203,7 +203,7 @@ static void qib_bad_mkey(struct qib_ibport *ibp, struct ib_smp *smp) /* * Send a Port Capability Mask Changed trap (ch. 14.3.11). */ -void qib_cap_mask_chg(struct rvt_dev_info *rdi, u8 port_num) +void qib_cap_mask_chg(struct rvt_dev_info *rdi, u32 port_num) { struct qib_ibdev *ibdev = container_of(rdi, struct qib_ibdev, rdi); struct qib_devdata *dd = dd_from_dev(ibdev); @@ -2360,7 +2360,7 @@ static int process_cc(struct ib_device *ibdev, int mad_flags, * * This is called by the ib_mad module. */ -int qib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port, +int qib_process_mad(struct ib_device *ibdev, int mad_flags, u32 port, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_mad *in, struct ib_mad *out, size_t *out_mad_size, u16 *out_mad_pkey_index) diff --git a/drivers/infiniband/hw/qib/qib_qp.c b/drivers/infiniband/hw/qib/qib_qp.c index ca39a029e4af..1974ceb9d405 100644 --- a/drivers/infiniband/hw/qib/qib_qp.c +++ b/drivers/infiniband/hw/qib/qib_qp.c @@ -125,7 +125,7 @@ static void get_map_page(struct rvt_qpn_table *qpt, struct rvt_qpn_map *map) * zero/one for QP type IB_QPT_SMI/IB_QPT_GSI. */ int qib_alloc_qpn(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt, - enum ib_qp_type type, u8 port) + enum ib_qp_type type, u32 port) { u32 i, offset, max_scan, qpn; struct rvt_qpn_map *map; @@ -136,7 +136,7 @@ int qib_alloc_qpn(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt, u16 qpt_mask = dd->qpn_mask; if (type == IB_QPT_SMI || type == IB_QPT_GSI) { - unsigned n; + u32 n; ret = type == IB_QPT_GSI; n = 1 << (ret + 2 * (port - 1)); diff --git a/drivers/infiniband/hw/qib/qib_sysfs.c b/drivers/infiniband/hw/qib/qib_sysfs.c index 62c179fc764b..5e9e66f27064 100644 --- a/drivers/infiniband/hw/qib/qib_sysfs.c +++ b/drivers/infiniband/hw/qib/qib_sysfs.c @@ -728,7 +728,7 @@ const struct attribute_group qib_attr_group = { .attrs = qib_attributes, }; -int qib_create_port_files(struct ib_device *ibdev, u8 port_num, +int qib_create_port_files(struct ib_device *ibdev, u32 port_num, struct kobject *kobj) { struct qib_pportdata *ppd; diff --git a/drivers/infiniband/hw/qib/qib_verbs.c b/drivers/infiniband/hw/qib/qib_verbs.c index 8e0de265ad57..d17d034ecdfd 100644 --- a/drivers/infiniband/hw/qib/qib_verbs.c +++ b/drivers/infiniband/hw/qib/qib_verbs.c @@ -1188,7 +1188,7 @@ full: } } -static int qib_query_port(struct rvt_dev_info *rdi, u8 port_num, +static int qib_query_port(struct rvt_dev_info *rdi, u32 port_num, struct ib_port_attr *props) { struct qib_ibdev *ibdev = container_of(rdi, struct qib_ibdev, rdi); @@ -1273,7 +1273,7 @@ bail: return ret; } -static int qib_shut_down_port(struct rvt_dev_info *rdi, u8 port_num) +static int qib_shut_down_port(struct rvt_dev_info *rdi, u32 port_num) { struct qib_ibdev *ibdev = container_of(rdi, struct qib_ibdev, rdi); struct qib_devdata *dd = dd_from_dev(ibdev); @@ -1342,7 +1342,7 @@ struct ib_ah *qib_create_qp0_ah(struct qib_ibport *ibp, u16 dlid) struct rvt_qp *qp0; struct qib_pportdata *ppd = ppd_from_ibp(ibp); struct qib_devdata *dd = dd_from_ppd(ppd); - u8 port_num = ppd->port; + u32 port_num = ppd->port; memset(&attr, 0, sizeof(attr)); attr.type = rdma_ah_find_type(&dd->verbs_dev.rdi.ibdev, port_num); diff --git a/drivers/infiniband/hw/qib/qib_verbs.h b/drivers/infiniband/hw/qib/qib_verbs.h index dc0e81f3b6f4..07548fac1d8e 100644 --- a/drivers/infiniband/hw/qib/qib_verbs.h +++ b/drivers/infiniband/hw/qib/qib_verbs.h @@ -239,10 +239,10 @@ static inline int qib_pkey_ok(u16 pkey1, u16 pkey2) void qib_bad_pkey(struct qib_ibport *ibp, u32 key, u32 sl, u32 qp1, u32 qp2, __be16 lid1, __be16 lid2); -void qib_cap_mask_chg(struct rvt_dev_info *rdi, u8 port_num); +void qib_cap_mask_chg(struct rvt_dev_info *rdi, u32 port_num); void qib_sys_guid_chg(struct qib_ibport *ibp); void qib_node_desc_chg(struct qib_ibport *ibp); -int qib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, +int qib_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_mad *in, struct ib_mad *out, size_t *out_mad_size, u16 *out_mad_pkey_index); @@ -273,7 +273,7 @@ void *qib_qp_priv_alloc(struct rvt_dev_info *rdi, struct rvt_qp *qp); void qib_qp_priv_free(struct rvt_dev_info *rdi, struct rvt_qp *qp); void qib_notify_qp_reset(struct rvt_qp *qp); int qib_alloc_qpn(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt, - enum ib_qp_type type, u8 port); + enum ib_qp_type type, u32 port); void qib_restart_rc(struct rvt_qp *qp, u32 psn, int wait); #ifdef CONFIG_DEBUG_FS -- cgit v1.2.3-59-g8ed1b