diff options
Diffstat (limited to 'drivers/infiniband/hw/hns/hns_roce_hem.c')
-rw-r--r-- | drivers/infiniband/hw/hns/hns_roce_hem.c | 1003 |
1 files changed, 456 insertions, 547 deletions
diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c index e82215774032..aa8a08d1c014 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.c +++ b/drivers/infiniband/hw/hns/hns_roce_hem.c @@ -31,13 +31,19 @@ * SOFTWARE. */ -#include <linux/platform_device.h> #include "hns_roce_device.h" #include "hns_roce_hem.h" #include "hns_roce_common.h" -#define DMA_ADDR_T_SHIFT 12 -#define BT_BA_SHIFT 32 +#define HEM_INDEX_BUF BIT(0) +#define HEM_INDEX_L0 BIT(1) +#define HEM_INDEX_L1 BIT(2) +struct hns_roce_hem_index { + u64 buf; + u64 l0; + u64 l1; + u32 inited; /* indicate which index is available */ +}; bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type) { @@ -65,17 +71,8 @@ bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type) case HEM_TYPE_CQC_TIMER: hop_num = hr_dev->caps.cqc_timer_hop_num; break; - case HEM_TYPE_CQE: - hop_num = hr_dev->caps.cqe_hop_num; - break; - case HEM_TYPE_MTT: - hop_num = hr_dev->caps.mtt_hop_num; - break; - case HEM_TYPE_SRQWQE: - hop_num = hr_dev->caps.srqwqe_hop_num; - break; - case HEM_TYPE_IDX: - hop_num = hr_dev->caps.idx_hop_num; + case HEM_TYPE_GMV: + hop_num = hr_dev->caps.gmv_hop_num; break; default: return false; @@ -84,25 +81,27 @@ bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type) return hop_num ? true : false; } -static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 start_idx, - u32 bt_chunk_num, u64 hem_max_num) +static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 hem_idx, + u32 bt_chunk_num, u64 hem_max_num) { + u64 start_idx = round_down(hem_idx, bt_chunk_num); u64 check_max_num = start_idx + bt_chunk_num; u64 i; for (i = start_idx; (i < check_max_num) && (i < hem_max_num); i++) - if (hem[i]) + if (i != hem_idx && hem[i]) return false; return true; } -static bool hns_roce_check_bt_null(u64 **bt, u64 start_idx, u32 bt_chunk_num) +static bool hns_roce_check_bt_null(u64 **bt, u64 ba_idx, u32 bt_chunk_num) { + u64 start_idx = round_down(ba_idx, bt_chunk_num); int i; for (i = 0; i < bt_chunk_num; i++) - if (bt[start_idx + i]) + if (i != ba_idx && bt[start_idx + i]) return false; return true; @@ -183,40 +182,16 @@ static int get_hem_table_config(struct hns_roce_dev *hr_dev, mhop->ba_l0_num = hr_dev->caps.srqc_bt_num; mhop->hop_num = hr_dev->caps.srqc_hop_num; break; - case HEM_TYPE_MTT: - mhop->buf_chunk_size = 1 << (hr_dev->caps.mtt_buf_pg_sz - + PAGE_SHIFT); - mhop->bt_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz - + PAGE_SHIFT); - mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN; - mhop->hop_num = hr_dev->caps.mtt_hop_num; - break; - case HEM_TYPE_CQE: - mhop->buf_chunk_size = 1 << (hr_dev->caps.cqe_buf_pg_sz - + PAGE_SHIFT); - mhop->bt_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz - + PAGE_SHIFT); - mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN; - mhop->hop_num = hr_dev->caps.cqe_hop_num; - break; - case HEM_TYPE_SRQWQE: - mhop->buf_chunk_size = 1 << (hr_dev->caps.srqwqe_buf_pg_sz - + PAGE_SHIFT); - mhop->bt_chunk_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz - + PAGE_SHIFT); - mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN; - mhop->hop_num = hr_dev->caps.srqwqe_hop_num; - break; - case HEM_TYPE_IDX: - mhop->buf_chunk_size = 1 << (hr_dev->caps.idx_buf_pg_sz - + PAGE_SHIFT); - mhop->bt_chunk_size = 1 << (hr_dev->caps.idx_ba_pg_sz - + PAGE_SHIFT); - mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN; - mhop->hop_num = hr_dev->caps.idx_hop_num; + case HEM_TYPE_GMV: + mhop->buf_chunk_size = 1 << (hr_dev->caps.gmv_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (hr_dev->caps.gmv_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = hr_dev->caps.gmv_bt_num; + mhop->hop_num = hr_dev->caps.gmv_hop_num; break; default: - dev_err(dev, "Table %d not support multi-hop addressing!\n", + dev_err(dev, "table %u not support multi-hop addressing!\n", type); return -EINVAL; } @@ -230,9 +205,9 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, { struct device *dev = hr_dev->dev; u32 chunk_ba_num; + u32 chunk_size; u32 table_idx; u32 bt_num; - u32 chunk_size; if (get_hem_table_config(hr_dev, mhop, table->type)) return -EINVAL; @@ -248,8 +223,7 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : mhop->bt_chunk_size; - table_idx = (*obj & (table->num_obj - 1)) / - (chunk_size / table->obj_size); + table_idx = *obj / (chunk_size / table->obj_size); switch (bt_num) { case 3: mhop->l2_idx = table_idx & (chunk_ba_num - 1); @@ -264,8 +238,8 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, mhop->l0_idx = table_idx; break; default: - dev_err(dev, "Table %d not support hop_num = %d!\n", - table->type, mhop->hop_num); + dev_err(dev, "table %u not support hop_num = %u!\n", + table->type, mhop->hop_num); return -EINVAL; } if (mhop->l0_idx >= mhop->ba_l0_num) @@ -292,7 +266,6 @@ static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, if (!hem) return NULL; - hem->refcount = 0; INIT_LIST_HEAD(&hem->chunk_list); order = get_order(hem_alloc_size); @@ -359,253 +332,235 @@ void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem) kfree(hem); } -static int hns_roce_set_hem(struct hns_roce_dev *hr_dev, - struct hns_roce_hem_table *table, unsigned long obj) -{ - spinlock_t *lock = &hr_dev->bt_cmd_lock; - struct device *dev = hr_dev->dev; - long end; - unsigned long flags; - struct hns_roce_hem_iter iter; - void __iomem *bt_cmd; - __le32 bt_cmd_val[2]; - __le32 bt_cmd_h = 0; - __le32 bt_cmd_l = 0; - u64 bt_ba = 0; - int ret = 0; - - /* Find the HEM(Hardware Entry Memory) entry */ - unsigned long i = (obj & (table->num_obj - 1)) / - (table->table_chunk_size / table->obj_size); - - switch (table->type) { - case HEM_TYPE_QPC: - case HEM_TYPE_MTPT: - case HEM_TYPE_CQC: - case HEM_TYPE_SRQC: - roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M, - ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, table->type); - break; - default: - return ret; - } - - roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M, - ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj); - roce_set_bit(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0); - roce_set_bit(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1); - - /* Currently iter only a chunk */ - for (hns_roce_hem_first(table->hem[i], &iter); - !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) { - bt_ba = hns_roce_hem_addr(&iter) >> DMA_ADDR_T_SHIFT; - - spin_lock_irqsave(lock, flags); - - bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG; - - end = HW_SYNC_TIMEOUT_MSECS; - while (end > 0) { - if (!(readl(bt_cmd) >> BT_CMD_SYNC_SHIFT)) - break; - - mdelay(HW_SYNC_SLEEP_TIME_INTERVAL); - end -= HW_SYNC_SLEEP_TIME_INTERVAL; - } - - if (end <= 0) { - dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n"); - spin_unlock_irqrestore(lock, flags); - return -EBUSY; - } - - bt_cmd_l = cpu_to_le32(bt_ba); - roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M, - ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S, - bt_ba >> BT_BA_SHIFT); - - bt_cmd_val[0] = bt_cmd_l; - bt_cmd_val[1] = bt_cmd_h; - hns_roce_write64_k(bt_cmd_val, - hr_dev->reg_base + ROCEE_BT_CMD_L_REG); - spin_unlock_irqrestore(lock, flags); - } - - return ret; -} - -static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev, - struct hns_roce_hem_table *table, - unsigned long obj) +static int calc_hem_config(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, unsigned long obj, + struct hns_roce_hem_mhop *mhop, + struct hns_roce_hem_index *index) { - struct device *dev = hr_dev->dev; - struct hns_roce_hem_mhop mhop; - struct hns_roce_hem_iter iter; - u32 buf_chunk_size; - u32 bt_chunk_size; + struct ib_device *ibdev = &hr_dev->ib_dev; + unsigned long mhop_obj = obj; + u32 l0_idx, l1_idx, l2_idx; u32 chunk_ba_num; - u32 hop_num; - u32 size; u32 bt_num; - u64 hem_idx; - u64 bt_l1_idx = 0; - u64 bt_l0_idx = 0; - u64 bt_ba; - unsigned long mhop_obj = obj; - int bt_l1_allocated = 0; - int bt_l0_allocated = 0; - int step_idx; int ret; - ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); + ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, mhop); if (ret) return ret; - buf_chunk_size = mhop.buf_chunk_size; - bt_chunk_size = mhop.bt_chunk_size; - hop_num = mhop.hop_num; - chunk_ba_num = bt_chunk_size / BA_BYTE_LEN; - - bt_num = hns_roce_get_bt_num(table->type, hop_num); + l0_idx = mhop->l0_idx; + l1_idx = mhop->l1_idx; + l2_idx = mhop->l2_idx; + chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; + bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); switch (bt_num) { case 3: - hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num + - mhop.l1_idx * chunk_ba_num + mhop.l2_idx; - bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; - bt_l0_idx = mhop.l0_idx; + index->l1 = l0_idx * chunk_ba_num + l1_idx; + index->l0 = l0_idx; + index->buf = l0_idx * chunk_ba_num * chunk_ba_num + + l1_idx * chunk_ba_num + l2_idx; break; case 2: - hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; - bt_l0_idx = mhop.l0_idx; + index->l0 = l0_idx; + index->buf = l0_idx * chunk_ba_num + l1_idx; break; case 1: - hem_idx = mhop.l0_idx; + index->buf = l0_idx; break; default: - dev_err(dev, "Table %d not support hop_num = %d!\n", - table->type, hop_num); + ibdev_err(ibdev, "table %u not support mhop.hop_num = %u!\n", + table->type, mhop->hop_num); return -EINVAL; } - if (unlikely(hem_idx >= table->num_hem)) { - dev_err(dev, "Table %d exceed hem limt idx = %llu,max = %lu!\n", - table->type, hem_idx, table->num_hem); + if (unlikely(index->buf >= table->num_hem)) { + ibdev_err(ibdev, "table %u exceed hem limt idx %llu, max %lu!\n", + table->type, index->buf, table->num_hem); return -EINVAL; } - mutex_lock(&table->mutex); + return 0; +} - if (table->hem[hem_idx]) { - ++table->hem[hem_idx]->refcount; - goto out; +static void free_mhop_hem(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, + struct hns_roce_hem_mhop *mhop, + struct hns_roce_hem_index *index) +{ + u32 bt_size = mhop->bt_chunk_size; + struct device *dev = hr_dev->dev; + + if (index->inited & HEM_INDEX_BUF) { + hns_roce_free_hem(hr_dev, table->hem[index->buf]); + table->hem[index->buf] = NULL; + } + + if (index->inited & HEM_INDEX_L1) { + dma_free_coherent(dev, bt_size, table->bt_l1[index->l1], + table->bt_l1_dma_addr[index->l1]); + table->bt_l1[index->l1] = NULL; } + if (index->inited & HEM_INDEX_L0) { + dma_free_coherent(dev, bt_size, table->bt_l0[index->l0], + table->bt_l0_dma_addr[index->l0]); + table->bt_l0[index->l0] = NULL; + } +} + +static int alloc_mhop_hem(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, + struct hns_roce_hem_mhop *mhop, + struct hns_roce_hem_index *index) +{ + u32 bt_size = mhop->bt_chunk_size; + struct device *dev = hr_dev->dev; + struct hns_roce_hem_iter iter; + gfp_t flag; + u64 bt_ba; + u32 size; + int ret; + /* alloc L1 BA's chunk */ - if ((check_whether_bt_num_3(table->type, hop_num) || - check_whether_bt_num_2(table->type, hop_num)) && - !table->bt_l0[bt_l0_idx]) { - table->bt_l0[bt_l0_idx] = dma_alloc_coherent(dev, bt_chunk_size, - &(table->bt_l0_dma_addr[bt_l0_idx]), + if ((check_whether_bt_num_3(table->type, mhop->hop_num) || + check_whether_bt_num_2(table->type, mhop->hop_num)) && + !table->bt_l0[index->l0]) { + table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size, + &table->bt_l0_dma_addr[index->l0], GFP_KERNEL); - if (!table->bt_l0[bt_l0_idx]) { + if (!table->bt_l0[index->l0]) { ret = -ENOMEM; goto out; } - bt_l0_allocated = 1; - - /* set base address to hardware */ - if (table->type < HEM_TYPE_MTT) { - step_idx = 0; - if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) { - ret = -ENODEV; - dev_err(dev, "set HEM base address to HW failed!\n"); - goto err_dma_alloc_l1; - } - } + index->inited |= HEM_INDEX_L0; } /* alloc L2 BA's chunk */ - if (check_whether_bt_num_3(table->type, hop_num) && - !table->bt_l1[bt_l1_idx]) { - table->bt_l1[bt_l1_idx] = dma_alloc_coherent(dev, bt_chunk_size, - &(table->bt_l1_dma_addr[bt_l1_idx]), + if (check_whether_bt_num_3(table->type, mhop->hop_num) && + !table->bt_l1[index->l1]) { + table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size, + &table->bt_l1_dma_addr[index->l1], GFP_KERNEL); - if (!table->bt_l1[bt_l1_idx]) { + if (!table->bt_l1[index->l1]) { ret = -ENOMEM; - goto err_dma_alloc_l1; - } - bt_l1_allocated = 1; - *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = - table->bt_l1_dma_addr[bt_l1_idx]; - - /* set base address to hardware */ - step_idx = 1; - if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) { - ret = -ENODEV; - dev_err(dev, "set HEM base address to HW failed!\n"); - goto err_alloc_hem_buf; + goto err_alloc_hem; } + index->inited |= HEM_INDEX_L1; + *(table->bt_l0[index->l0] + mhop->l1_idx) = + table->bt_l1_dma_addr[index->l1]; } /* * alloc buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC. * alloc bt space chunk for MTT/CQE. */ - size = table->type < HEM_TYPE_MTT ? buf_chunk_size : bt_chunk_size; - table->hem[hem_idx] = hns_roce_alloc_hem(hr_dev, - size >> PAGE_SHIFT, - size, - (table->lowmem ? GFP_KERNEL : - GFP_HIGHUSER) | __GFP_NOWARN); - if (!table->hem[hem_idx]) { + size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size; + flag = GFP_KERNEL | __GFP_NOWARN; + table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT, + size, flag); + if (!table->hem[index->buf]) { ret = -ENOMEM; - goto err_alloc_hem_buf; + goto err_alloc_hem; } - hns_roce_hem_first(table->hem[hem_idx], &iter); + index->inited |= HEM_INDEX_BUF; + hns_roce_hem_first(table->hem[index->buf], &iter); bt_ba = hns_roce_hem_addr(&iter); - if (table->type < HEM_TYPE_MTT) { - if (hop_num == 2) { - *(table->bt_l1[bt_l1_idx] + mhop.l2_idx) = bt_ba; - step_idx = 2; - } else if (hop_num == 1) { - *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba; - step_idx = 1; - } else if (hop_num == HNS_ROCE_HOP_NUM_0) { - step_idx = 0; - } else { - ret = -EINVAL; - goto err_dma_alloc_l1; + if (mhop->hop_num == 2) + *(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba; + else if (mhop->hop_num == 1) + *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; + } else if (mhop->hop_num == 2) { + *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; + } + + return 0; +err_alloc_hem: + free_mhop_hem(hr_dev, table, mhop, index); +out: + return ret; +} + +static int set_mhop_hem(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, unsigned long obj, + struct hns_roce_hem_mhop *mhop, + struct hns_roce_hem_index *index) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + u32 step_idx; + int ret = 0; + + if (index->inited & HEM_INDEX_L0) { + ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0); + if (ret) { + ibdev_err(ibdev, "set HEM step 0 failed!\n"); + goto out; } + } - /* set HEM base address to hardware */ - if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) { - ret = -ENODEV; - dev_err(dev, "set HEM base address to HW failed!\n"); - goto err_alloc_hem_buf; + if (index->inited & HEM_INDEX_L1) { + ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1); + if (ret) { + ibdev_err(ibdev, "set HEM step 1 failed!\n"); + goto out; } - } else if (hop_num == 2) { - *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba; } - ++table->hem[hem_idx]->refcount; - goto out; + if (index->inited & HEM_INDEX_BUF) { + if (mhop->hop_num == HNS_ROCE_HOP_NUM_0) + step_idx = 0; + else + step_idx = mhop->hop_num; + ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx); + if (ret) + ibdev_err(ibdev, "set HEM step last failed!\n"); + } +out: + return ret; +} -err_alloc_hem_buf: - if (bt_l1_allocated) { - dma_free_coherent(dev, bt_chunk_size, table->bt_l1[bt_l1_idx], - table->bt_l1_dma_addr[bt_l1_idx]); - table->bt_l1[bt_l1_idx] = NULL; +static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, + unsigned long obj) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + struct hns_roce_hem_index index = {}; + struct hns_roce_hem_mhop mhop = {}; + int ret; + + ret = calc_hem_config(hr_dev, table, obj, &mhop, &index); + if (ret) { + ibdev_err(ibdev, "calc hem config failed!\n"); + return ret; } -err_dma_alloc_l1: - if (bt_l0_allocated) { - dma_free_coherent(dev, bt_chunk_size, table->bt_l0[bt_l0_idx], - table->bt_l0_dma_addr[bt_l0_idx]); - table->bt_l0[bt_l0_idx] = NULL; + mutex_lock(&table->mutex); + if (table->hem[index.buf]) { + refcount_inc(&table->hem[index.buf]->refcount); + goto out; + } + + ret = alloc_mhop_hem(hr_dev, table, &mhop, &index); + if (ret) { + ibdev_err(ibdev, "alloc mhop hem failed!\n"); + goto out; + } + + /* set HEM base address to hardware */ + if (table->type < HEM_TYPE_MTT) { + ret = set_mhop_hem(hr_dev, table, obj, &mhop, &index); + if (ret) { + ibdev_err(ibdev, "set HEM address to HW failed!\n"); + goto err_alloc; + } } + refcount_set(&table->hem[index.buf]->refcount, 1); + goto out; + +err_alloc: + free_mhop_hem(hr_dev, table, &mhop, &index); out: mutex_unlock(&table->mutex); return ret; @@ -615,34 +570,32 @@ int hns_roce_table_get(struct hns_roce_dev *hr_dev, struct hns_roce_hem_table *table, unsigned long obj) { struct device *dev = hr_dev->dev; - int ret = 0; unsigned long i; + int ret = 0; if (hns_roce_check_whether_mhop(hr_dev, table->type)) return hns_roce_table_mhop_get(hr_dev, table, obj); - i = (obj & (table->num_obj - 1)) / (table->table_chunk_size / - table->obj_size); + i = obj / (table->table_chunk_size / table->obj_size); mutex_lock(&table->mutex); if (table->hem[i]) { - ++table->hem[i]->refcount; + refcount_inc(&table->hem[i]->refcount); goto out; } table->hem[i] = hns_roce_alloc_hem(hr_dev, table->table_chunk_size >> PAGE_SHIFT, table->table_chunk_size, - (table->lowmem ? GFP_KERNEL : - GFP_HIGHUSER) | __GFP_NOWARN); + GFP_KERNEL | __GFP_NOWARN); if (!table->hem[i]) { ret = -ENOMEM; goto out; } /* Set HEM base address(128K/page, pa) to Hardware */ - if (hns_roce_set_hem(hr_dev, table, obj)) { + if (hr_dev->hw->set_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT)) { hns_roce_free_hem(hr_dev, table->hem[i]); table->hem[i] = NULL; ret = -ENODEV; @@ -650,122 +603,81 @@ int hns_roce_table_get(struct hns_roce_dev *hr_dev, goto out; } - ++table->hem[i]->refcount; + refcount_set(&table->hem[i]->refcount, 1); out: mutex_unlock(&table->mutex); return ret; } -static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev, - struct hns_roce_hem_table *table, - unsigned long obj, - int check_refcount) +static void clear_mhop_hem(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, unsigned long obj, + struct hns_roce_hem_mhop *mhop, + struct hns_roce_hem_index *index) { - struct device *dev = hr_dev->dev; - struct hns_roce_hem_mhop mhop; - unsigned long mhop_obj = obj; - u32 bt_chunk_size; + struct ib_device *ibdev = &hr_dev->ib_dev; + u32 hop_num = mhop->hop_num; u32 chunk_ba_num; - u32 hop_num; - u32 start_idx; - u32 bt_num; - u64 hem_idx; - u64 bt_l1_idx = 0; - int ret; + u32 step_idx; - ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); - if (ret) - return; + index->inited = HEM_INDEX_BUF; + chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; + if (check_whether_bt_num_2(table->type, hop_num)) { + if (hns_roce_check_hem_null(table->hem, index->buf, + chunk_ba_num, table->num_hem)) + index->inited |= HEM_INDEX_L0; + } else if (check_whether_bt_num_3(table->type, hop_num)) { + if (hns_roce_check_hem_null(table->hem, index->buf, + chunk_ba_num, table->num_hem)) { + index->inited |= HEM_INDEX_L1; + if (hns_roce_check_bt_null(table->bt_l1, index->l1, + chunk_ba_num)) + index->inited |= HEM_INDEX_L0; + } + } - bt_chunk_size = mhop.bt_chunk_size; - hop_num = mhop.hop_num; - chunk_ba_num = bt_chunk_size / BA_BYTE_LEN; + if (table->type < HEM_TYPE_MTT) { + if (hop_num == HNS_ROCE_HOP_NUM_0) + step_idx = 0; + else + step_idx = hop_num; - bt_num = hns_roce_get_bt_num(table->type, hop_num); - switch (bt_num) { - case 3: - hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num + - mhop.l1_idx * chunk_ba_num + mhop.l2_idx; - bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; - break; - case 2: - hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; - break; - case 1: - hem_idx = mhop.l0_idx; - break; - default: - dev_err(dev, "Table %d not support hop_num = %d!\n", - table->type, hop_num); - return; + if (hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx)) + ibdev_warn(ibdev, "failed to clear hop%u HEM.\n", hop_num); + + if (index->inited & HEM_INDEX_L1) + if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1)) + ibdev_warn(ibdev, "failed to clear HEM step 1.\n"); + + if (index->inited & HEM_INDEX_L0) + if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) + ibdev_warn(ibdev, "failed to clear HEM step 0.\n"); } +} - mutex_lock(&table->mutex); +static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, + unsigned long obj, + int check_refcount) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + struct hns_roce_hem_index index = {}; + struct hns_roce_hem_mhop mhop = {}; + int ret; - if (check_refcount && (--table->hem[hem_idx]->refcount > 0)) { - mutex_unlock(&table->mutex); + ret = calc_hem_config(hr_dev, table, obj, &mhop, &index); + if (ret) { + ibdev_err(ibdev, "calc hem config failed!\n"); return; } - if (table->type < HEM_TYPE_MTT && hop_num == 1) { - if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1)) - dev_warn(dev, "Clear HEM base address failed.\n"); - } else if (table->type < HEM_TYPE_MTT && hop_num == 2) { - if (hr_dev->hw->clear_hem(hr_dev, table, obj, 2)) - dev_warn(dev, "Clear HEM base address failed.\n"); - } else if (table->type < HEM_TYPE_MTT && - hop_num == HNS_ROCE_HOP_NUM_0) { - if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) - dev_warn(dev, "Clear HEM base address failed.\n"); - } + if (!check_refcount) + mutex_lock(&table->mutex); + else if (!refcount_dec_and_mutex_lock(&table->hem[index.buf]->refcount, + &table->mutex)) + return; - /* - * free buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC. - * free bt space chunk for MTT/CQE. - */ - hns_roce_free_hem(hr_dev, table->hem[hem_idx]); - table->hem[hem_idx] = NULL; - - if (check_whether_bt_num_2(table->type, hop_num)) { - start_idx = mhop.l0_idx * chunk_ba_num; - if (hns_roce_check_hem_null(table->hem, start_idx, - chunk_ba_num, table->num_hem)) { - if (table->type < HEM_TYPE_MTT && - hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) - dev_warn(dev, "Clear HEM base address failed.\n"); - - dma_free_coherent(dev, bt_chunk_size, - table->bt_l0[mhop.l0_idx], - table->bt_l0_dma_addr[mhop.l0_idx]); - table->bt_l0[mhop.l0_idx] = NULL; - } - } else if (check_whether_bt_num_3(table->type, hop_num)) { - start_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num + - mhop.l1_idx * chunk_ba_num; - if (hns_roce_check_hem_null(table->hem, start_idx, - chunk_ba_num, table->num_hem)) { - if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1)) - dev_warn(dev, "Clear HEM base address failed.\n"); - - dma_free_coherent(dev, bt_chunk_size, - table->bt_l1[bt_l1_idx], - table->bt_l1_dma_addr[bt_l1_idx]); - table->bt_l1[bt_l1_idx] = NULL; - - start_idx = mhop.l0_idx * chunk_ba_num; - if (hns_roce_check_bt_null(table->bt_l1, start_idx, - chunk_ba_num)) { - if (hr_dev->hw->clear_hem(hr_dev, table, obj, - 0)) - dev_warn(dev, "Clear HEM base address failed.\n"); - - dma_free_coherent(dev, bt_chunk_size, - table->bt_l0[mhop.l0_idx], - table->bt_l0_dma_addr[mhop.l0_idx]); - table->bt_l0[mhop.l0_idx] = NULL; - } - } - } + clear_mhop_hem(hr_dev, table, obj, &mhop, &index); + free_mhop_hem(hr_dev, table, &mhop, &index); mutex_unlock(&table->mutex); } @@ -781,19 +693,17 @@ void hns_roce_table_put(struct hns_roce_dev *hr_dev, return; } - i = (obj & (table->num_obj - 1)) / - (table->table_chunk_size / table->obj_size); + i = obj / (table->table_chunk_size / table->obj_size); - mutex_lock(&table->mutex); + if (!refcount_dec_and_mutex_lock(&table->hem[i]->refcount, + &table->mutex)) + return; - if (--table->hem[i]->refcount == 0) { - /* Clear HEM base address */ - if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) - dev_warn(dev, "Clear HEM base address failed.\n"); + if (hr_dev->hw->clear_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT)) + dev_warn(dev, "failed to clear HEM base address.\n"); - hns_roce_free_hem(hr_dev, table->hem[i]); - table->hem[i] = NULL; - } + hns_roce_free_hem(hr_dev, table->hem[i]); + table->hem[i] = NULL; mutex_unlock(&table->mutex); } @@ -805,24 +715,21 @@ void *hns_roce_table_find(struct hns_roce_dev *hr_dev, struct hns_roce_hem_chunk *chunk; struct hns_roce_hem_mhop mhop; struct hns_roce_hem *hem; - void *addr = NULL; unsigned long mhop_obj = obj; unsigned long obj_per_chunk; unsigned long idx_offset; int offset, dma_offset; + void *addr = NULL; + u32 hem_idx = 0; int length; int i, j; - u32 hem_idx = 0; - - if (!table->lowmem) - return NULL; mutex_lock(&table->mutex); if (!hns_roce_check_whether_mhop(hr_dev, table->type)) { obj_per_chunk = table->table_chunk_size / table->obj_size; - hem = table->hem[(obj & (table->num_obj - 1)) / obj_per_chunk]; - idx_offset = (obj & (table->num_obj - 1)) % obj_per_chunk; + hem = table->hem[obj / obj_per_chunk]; + idx_offset = obj % obj_per_chunk; dma_offset = offset = idx_offset * table->obj_size; } else { u32 seg_size = 64; /* 8 bytes per BA and 8 BA per segment */ @@ -839,8 +746,7 @@ void *hns_roce_table_find(struct hns_roce_dev *hr_dev, hem_idx = i; hem = table->hem[hem_idx]; - dma_offset = offset = (obj & (table->num_obj - 1)) * seg_size % - mhop.bt_chunk_size; + dma_offset = offset = obj * seg_size % mhop.bt_chunk_size; if (mhop.hop_num == 2) dma_offset = offset = 0; } @@ -871,61 +777,9 @@ out: return addr; } -int hns_roce_table_get_range(struct hns_roce_dev *hr_dev, - struct hns_roce_hem_table *table, - unsigned long start, unsigned long end) -{ - struct hns_roce_hem_mhop mhop; - unsigned long inc = table->table_chunk_size / table->obj_size; - unsigned long i = 0; - int ret; - - if (hns_roce_check_whether_mhop(hr_dev, table->type)) { - ret = hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop); - if (ret) - goto fail; - inc = mhop.bt_chunk_size / table->obj_size; - } - - /* Allocate MTT entry memory according to chunk(128K) */ - for (i = start; i <= end; i += inc) { - ret = hns_roce_table_get(hr_dev, table, i); - if (ret) - goto fail; - } - - return 0; - -fail: - while (i > start) { - i -= inc; - hns_roce_table_put(hr_dev, table, i); - } - return ret; -} - -void hns_roce_table_put_range(struct hns_roce_dev *hr_dev, - struct hns_roce_hem_table *table, - unsigned long start, unsigned long end) -{ - struct hns_roce_hem_mhop mhop; - unsigned long inc = table->table_chunk_size / table->obj_size; - unsigned long i; - - if (hns_roce_check_whether_mhop(hr_dev, table->type)) { - if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop)) - return; - inc = mhop.bt_chunk_size / table->obj_size; - } - - for (i = start; i <= end; i += inc) - hns_roce_table_put(hr_dev, table, i); -} - int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, struct hns_roce_hem_table *table, u32 type, - unsigned long obj_size, unsigned long nobj, - int use_lowmem) + unsigned long obj_size, unsigned long nobj) { unsigned long obj_per_chunk; unsigned long num_hem; @@ -933,7 +787,7 @@ int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, if (!hns_roce_check_whether_mhop(hr_dev, type)) { table->table_chunk_size = hr_dev->caps.chunk_sz; obj_per_chunk = table->table_chunk_size / obj_size; - num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk; + num_hem = DIV_ROUND_UP(nobj, obj_per_chunk); table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL); if (!table->hem) @@ -943,7 +797,7 @@ int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, unsigned long buf_chunk_size; unsigned long bt_chunk_size; unsigned long bt_chunk_num; - unsigned long num_bt_l0 = 0; + unsigned long num_bt_l0; u32 hop_num; if (get_hem_table_config(hr_dev, &mhop, type)) @@ -955,8 +809,9 @@ int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, hop_num = mhop.hop_num; obj_per_chunk = buf_chunk_size / obj_size; - num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk; + num_hem = DIV_ROUND_UP(nobj, obj_per_chunk); bt_chunk_num = bt_chunk_size / BA_BYTE_LEN; + if (type >= HEM_TYPE_MTT) num_bt_l0 = bt_chunk_num; @@ -968,8 +823,7 @@ int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, if (check_whether_bt_num_3(type, hop_num)) { unsigned long num_bt_l1; - num_bt_l1 = (num_hem + bt_chunk_num - 1) / - bt_chunk_num; + num_bt_l1 = DIV_ROUND_UP(num_hem, bt_chunk_num); table->bt_l1 = kcalloc(num_bt_l1, sizeof(*table->bt_l1), GFP_KERNEL); @@ -1001,9 +855,7 @@ int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, table->type = type; table->num_hem = num_hem; - table->num_obj = nobj; table->obj_size = obj_size; - table->lowmem = use_lowmem; mutex_init(&table->mutex); return 0; @@ -1033,8 +885,8 @@ static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev, { struct hns_roce_hem_mhop mhop; u32 buf_chunk_size; - int i; u64 obj; + int i; if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop)) return; @@ -1074,7 +926,7 @@ void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev, if (table->hem[i]) { if (hr_dev->hw->clear_hem(hr_dev, table, i * table->table_chunk_size / table->obj_size, 0)) - dev_err(dev, "Clear HEM base address failed.\n"); + dev_err(dev, "clear HEM base address failed.\n"); hns_roce_free_hem(hr_dev, table->hem[i]); } @@ -1084,13 +936,7 @@ void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev, void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev) { - if ((hr_dev->caps.num_idx_segs)) - hns_roce_cleanup_hem_table(hr_dev, - &hr_dev->mr_table.mtt_idx_table); - if (hr_dev->caps.num_srqwqe_segs) - hns_roce_cleanup_hem_table(hr_dev, - &hr_dev->mr_table.mtt_srqwqe_table); - if (hr_dev->caps.srqc_entry_sz) + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) hns_roce_cleanup_hem_table(hr_dev, &hr_dev->srq_table.table); hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table); @@ -1100,22 +946,22 @@ void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev) if (hr_dev->caps.cqc_timer_entry_sz) hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cqc_timer_table); - if (hr_dev->caps.sccc_entry_sz) + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.sccc_table); if (hr_dev->caps.trrl_entry_sz) hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.trrl_table); + + if (hr_dev->caps.gmv_entry_sz) + hns_roce_cleanup_hem_table(hr_dev, &hr_dev->gmv_table); + hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table); hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table); hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table); - if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) - hns_roce_cleanup_hem_table(hr_dev, - &hr_dev->mr_table.mtt_cqe_table); - hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table); } -struct roce_hem_item { +struct hns_roce_hem_item { struct list_head list; /* link all hems in the same bt level */ struct list_head sibling; /* link all hems in last hop for mtt */ void *addr; @@ -1125,21 +971,26 @@ struct roce_hem_item { int end; /* end buf offset in this hem */ }; -static struct roce_hem_item *hem_list_alloc_item(struct hns_roce_dev *hr_dev, - int start, int end, - int count, bool exist_bt, - int bt_level) +/* All HEM items are linked in a tree structure */ +struct hns_roce_hem_head { + struct list_head branch[HNS_ROCE_MAX_BT_REGION]; + struct list_head root; + struct list_head leaf; +}; + +static struct hns_roce_hem_item * +hem_list_alloc_item(struct hns_roce_dev *hr_dev, int start, int end, int count, + bool exist_bt) { - struct roce_hem_item *hem; + struct hns_roce_hem_item *hem; hem = kzalloc(sizeof(*hem), GFP_KERNEL); if (!hem) return NULL; if (exist_bt) { - hem->addr = dma_alloc_coherent(hr_dev->dev, - count * BA_BYTE_LEN, - &hem->dma_addr, GFP_KERNEL); + hem->addr = dma_alloc_coherent(hr_dev->dev, count * BA_BYTE_LEN, + &hem->dma_addr, GFP_KERNEL); if (!hem->addr) { kfree(hem); return NULL; @@ -1156,7 +1007,7 @@ static struct roce_hem_item *hem_list_alloc_item(struct hns_roce_dev *hr_dev, } static void hem_list_free_item(struct hns_roce_dev *hr_dev, - struct roce_hem_item *hem, bool exist_bt) + struct hns_roce_hem_item *hem, bool exist_bt) { if (exist_bt) dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN, @@ -1167,7 +1018,7 @@ static void hem_list_free_item(struct hns_roce_dev *hr_dev, static void hem_list_free_all(struct hns_roce_dev *hr_dev, struct list_head *head, bool exist_bt) { - struct roce_hem_item *hem, *temp_hem; + struct hns_roce_hem_item *hem, *temp_hem; list_for_each_entry_safe(hem, temp_hem, head, list) { list_del(&hem->list); @@ -1183,24 +1034,24 @@ static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr, /* assign L0 table address to hem from root bt */ static void hem_list_assign_bt(struct hns_roce_dev *hr_dev, - struct roce_hem_item *hem, void *cpu_addr, + struct hns_roce_hem_item *hem, void *cpu_addr, u64 phy_addr) { hem->addr = cpu_addr; hem->dma_addr = (dma_addr_t)phy_addr; } -static inline bool hem_list_page_is_in_range(struct roce_hem_item *hem, +static inline bool hem_list_page_is_in_range(struct hns_roce_hem_item *hem, int offset) { return (hem->start <= offset && offset <= hem->end); } -static struct roce_hem_item *hem_list_search_item(struct list_head *ba_list, - int page_offset) +static struct hns_roce_hem_item *hem_list_search_item(struct list_head *ba_list, + int page_offset) { - struct roce_hem_item *hem, *temp_hem; - struct roce_hem_item *found = NULL; + struct hns_roce_hem_item *hem, *temp_hem; + struct hns_roce_hem_item *found = NULL; list_for_each_entry_safe(hem, temp_hem, ba_list, list) { if (hem_list_page_is_in_range(hem, page_offset)) { @@ -1224,7 +1075,7 @@ static bool hem_list_is_bottom_bt(int hopnum, int bt_level) return bt_level >= (hopnum ? hopnum - 1 : hopnum); } -/** +/* * calc base address entries num * @hopnum: num of mutihop addressing * @bt_level: base address table level @@ -1257,7 +1108,7 @@ static u32 hem_list_calc_ba_range(int hopnum, int bt_level, int unit) return step; } -/** +/* * calc the root ba entries which could cover all regions * @regions: buf region array * @region_cnt: array size of @regions @@ -1290,9 +1141,9 @@ static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev, int offset, struct list_head *mid_bt, struct list_head *btm_bt) { - struct roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL }; + struct hns_roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL }; struct list_head temp_list[HNS_ROCE_MAX_BT_LEVEL]; - struct roce_hem_item *cur, *pre; + struct hns_roce_hem_item *cur, *pre; const int hopnum = r->hopnum; int start_aligned; int distance; @@ -1311,7 +1162,7 @@ static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev, } if (offset < r->offset) { - dev_err(hr_dev->dev, "invalid offset %d,min %d!\n", + dev_err(hr_dev->dev, "invalid offset %d, min %u!\n", offset, r->offset); return -EINVAL; } @@ -1338,7 +1189,7 @@ static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev, start_aligned = (distance / step) * step + r->offset; end = min_t(int, start_aligned + step - 1, max_ofs); cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit, - true, level); + true); if (!cur) { ret = -ENOMEM; goto err_exit; @@ -1370,47 +1221,96 @@ err_exit: return ret; } -static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev, - struct hns_roce_hem_list *hem_list, int unit, - const struct hns_roce_buf_region *regions, - int region_cnt) +static struct hns_roce_hem_item * +alloc_root_hem(struct hns_roce_dev *hr_dev, int unit, int *max_ba_num, + const struct hns_roce_buf_region *regions, int region_cnt) { - struct roce_hem_item *hem, *temp_hem, *root_hem; - struct list_head temp_list[HNS_ROCE_MAX_BT_REGION]; const struct hns_roce_buf_region *r; - struct list_head temp_root; - struct list_head temp_btm; - void *cpu_base; - u64 phy_base; - int ret = 0; + struct hns_roce_hem_item *hem; + int ba_num; int offset; - int total; - int step; - int i; - r = ®ions[0]; - root_hem = hem_list_search_item(&hem_list->root_bt, r->offset); - if (root_hem) - return 0; + ba_num = hns_roce_hem_list_calc_root_ba(regions, region_cnt, unit); + if (ba_num < 1) + return ERR_PTR(-ENOMEM); - INIT_LIST_HEAD(&temp_root); - total = r->offset; + if (ba_num > unit) + return ERR_PTR(-ENOBUFS); + + offset = regions[0].offset; /* indicate to last region */ r = ®ions[region_cnt - 1]; - root_hem = hem_list_alloc_item(hr_dev, total, r->offset + r->count - 1, - unit, true, 0); - if (!root_hem) + hem = hem_list_alloc_item(hr_dev, offset, r->offset + r->count - 1, + ba_num, true); + if (!hem) + return ERR_PTR(-ENOMEM); + + *max_ba_num = ba_num; + + return hem; +} + +static int alloc_fake_root_bt(struct hns_roce_dev *hr_dev, void *cpu_base, + u64 phy_base, const struct hns_roce_buf_region *r, + struct list_head *branch_head, + struct list_head *leaf_head) +{ + struct hns_roce_hem_item *hem; + + hem = hem_list_alloc_item(hr_dev, r->offset, r->offset + r->count - 1, + r->count, false); + if (!hem) return -ENOMEM; - list_add(&root_hem->list, &temp_root); - hem_list->root_ba = root_hem->dma_addr; + hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base); + list_add(&hem->list, branch_head); + list_add(&hem->sibling, leaf_head); - INIT_LIST_HEAD(&temp_btm); - for (i = 0; i < region_cnt; i++) - INIT_LIST_HEAD(&temp_list[i]); + return r->count; +} + +static int setup_middle_bt(struct hns_roce_dev *hr_dev, void *cpu_base, + int unit, const struct hns_roce_buf_region *r, + const struct list_head *branch_head) +{ + struct hns_roce_hem_item *hem, *temp_hem; + int total = 0; + int offset; + int step; + + step = hem_list_calc_ba_range(r->hopnum, 1, unit); + if (step < 1) + return -EINVAL; + + /* if exist mid bt, link L1 to L0 */ + list_for_each_entry_safe(hem, temp_hem, branch_head, list) { + offset = (hem->start - r->offset) / step * BA_BYTE_LEN; + hem_list_link_bt(hr_dev, cpu_base + offset, hem->dma_addr); + total++; + } + + return total; +} + +static int +setup_root_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem_list *hem_list, + int unit, int max_ba_num, struct hns_roce_hem_head *head, + const struct hns_roce_buf_region *regions, int region_cnt) +{ + const struct hns_roce_buf_region *r; + struct hns_roce_hem_item *root_hem; + void *cpu_base; + u64 phy_base; + int i, total; + int ret; + + root_hem = list_first_entry(&head->root, + struct hns_roce_hem_item, list); + if (!root_hem) + return -ENOMEM; total = 0; - for (i = 0; i < region_cnt && total < unit; i++) { + for (i = 0; i < region_cnt && total < max_ba_num; i++) { r = ®ions[i]; if (!r->count) continue; @@ -1422,47 +1322,64 @@ static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev, /* if hopnum is 0 or 1, cut a new fake hem from the root bt * which's address share to all regions. */ - if (hem_list_is_bottom_bt(r->hopnum, 0)) { - hem = hem_list_alloc_item(hr_dev, r->offset, - r->offset + r->count - 1, - r->count, false, 0); - if (!hem) { - ret = -ENOMEM; - goto err_exit; - } - hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base); - list_add(&hem->list, &temp_list[i]); - list_add(&hem->sibling, &temp_btm); - total += r->count; - } else { - step = hem_list_calc_ba_range(r->hopnum, 1, unit); - if (step < 1) { - ret = -EINVAL; - goto err_exit; - } - /* if exist mid bt, link L1 to L0 */ - list_for_each_entry_safe(hem, temp_hem, - &hem_list->mid_bt[i][1], list) { - offset = hem->start / step * BA_BYTE_LEN; - hem_list_link_bt(hr_dev, cpu_base + offset, - hem->dma_addr); - total++; - } - } + if (hem_list_is_bottom_bt(r->hopnum, 0)) + ret = alloc_fake_root_bt(hr_dev, cpu_base, phy_base, r, + &head->branch[i], &head->leaf); + else + ret = setup_middle_bt(hr_dev, cpu_base, unit, r, + &hem_list->mid_bt[i][1]); + + if (ret < 0) + return ret; + + total += ret; } - list_splice(&temp_btm, &hem_list->btm_bt); - list_splice(&temp_root, &hem_list->root_bt); + list_splice(&head->leaf, &hem_list->btm_bt); + list_splice(&head->root, &hem_list->root_bt); for (i = 0; i < region_cnt; i++) - list_splice(&temp_list[i], &hem_list->mid_bt[i][0]); + list_splice(&head->branch[i], &hem_list->mid_bt[i][0]); return 0; +} -err_exit: +static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_list *hem_list, int unit, + const struct hns_roce_buf_region *regions, + int region_cnt) +{ + struct hns_roce_hem_item *root_hem; + struct hns_roce_hem_head head; + int max_ba_num; + int ret; + int i; + + root_hem = hem_list_search_item(&hem_list->root_bt, regions[0].offset); + if (root_hem) + return 0; + + max_ba_num = 0; + root_hem = alloc_root_hem(hr_dev, unit, &max_ba_num, regions, + region_cnt); + if (IS_ERR(root_hem)) + return PTR_ERR(root_hem); + + /* List head for storing all allocated HEM items */ + INIT_LIST_HEAD(&head.root); + INIT_LIST_HEAD(&head.leaf); for (i = 0; i < region_cnt; i++) - hem_list_free_all(hr_dev, &temp_list[i], false); + INIT_LIST_HEAD(&head.branch[i]); - hem_list_free_all(hr_dev, &temp_root, true); + hem_list->root_ba = root_hem->dma_addr; + list_add(&root_hem->list, &head.root); + ret = setup_root_hem(hr_dev, hem_list, unit, max_ba_num, &head, regions, + region_cnt); + if (ret) { + for (i = 0; i < region_cnt; i++) + hem_list_free_all(hr_dev, &head.branch[i], false); + + hem_list_free_all(hr_dev, &head.root, true); + } return ret; } @@ -1471,12 +1388,12 @@ err_exit: int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev, struct hns_roce_hem_list *hem_list, const struct hns_roce_buf_region *regions, - int region_cnt) + int region_cnt, unsigned int bt_pg_shift) { const struct hns_roce_buf_region *r; int ofs, end; - int ret = 0; int unit; + int ret; int i; if (region_cnt > HNS_ROCE_MAX_BT_REGION) { @@ -1485,7 +1402,7 @@ int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev, return -EINVAL; } - unit = (1 << hem_list->bt_pg_shift) / BA_BYTE_LEN; + unit = (1 << bt_pg_shift) / BA_BYTE_LEN; for (i = 0; i < region_cnt; i++) { r = ®ions[i]; if (!r->count) @@ -1498,7 +1415,7 @@ int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev, &hem_list->btm_bt); if (ret) { dev_err(hr_dev->dev, - "alloc hem trunk fail ret=%d!\n", ret); + "alloc hem trunk fail ret = %d!\n", ret); goto err_alloc; } } @@ -1507,7 +1424,7 @@ int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev, ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions, region_cnt); if (ret) - dev_err(hr_dev->dev, "alloc hem root fail ret=%d!\n", ret); + dev_err(hr_dev->dev, "alloc hem root fail ret = %d!\n", ret); else return 0; @@ -1532,8 +1449,7 @@ void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev, hem_list->root_ba = 0; } -void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list, - int bt_page_order) +void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list) { int i, j; @@ -1542,25 +1458,21 @@ void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list, for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++) for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++) INIT_LIST_HEAD(&hem_list->mid_bt[i][j]); - - hem_list->bt_pg_shift = bt_page_order; } void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_hem_list *hem_list, - int offset, int *mtt_cnt, u64 *phy_addr) + int offset, int *mtt_cnt) { struct list_head *head = &hem_list->btm_bt; - struct roce_hem_item *hem, *temp_hem; + struct hns_roce_hem_item *hem, *temp_hem; void *cpu_base = NULL; - u64 phy_base = 0; int nr = 0; list_for_each_entry_safe(hem, temp_hem, head, sibling) { if (hem_list_page_is_in_range(hem, offset)) { nr = offset - hem->start; cpu_base = hem->addr + nr * BA_BYTE_LEN; - phy_base = hem->dma_addr + nr * BA_BYTE_LEN; nr = hem->end + 1 - offset; break; } @@ -1569,8 +1481,5 @@ void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev, if (mtt_cnt) *mtt_cnt = nr; - if (phy_addr) - *phy_addr = phy_base; - return cpu_base; } |