diff options
author | Tvrtko Ursulin <tvrtko.ursulin@intel.com> | 2022-02-25 13:44:44 +0000 |
---|---|---|
committer | Tvrtko Ursulin <tvrtko.ursulin@intel.com> | 2022-02-25 13:44:44 +0000 |
commit | db927686e43ffebfc5d1693c1cb4fd74f462d99b (patch) | |
tree | 0cbd82aea0825efc1cb10bad4af664ed1bed9b83 /drivers/gpu/drm/amd/amdkfd/kfd_svm.c | |
parent | drm/i915: Check stolen memory size before calling drm_mm_init (diff) | |
parent | Merge tag 'drm-misc-next-2022-02-23' of git://anongit.freedesktop.org/drm/drm-misc into drm-next (diff) | |
download | linux-dev-db927686e43ffebfc5d1693c1cb4fd74f462d99b.tar.xz linux-dev-db927686e43ffebfc5d1693c1cb4fd74f462d99b.zip |
Merge drm/drm-next into drm-intel-gt-next
Matt needed some buddy allocator changes for landing DG2 small BAR
support patches.
Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Diffstat (limited to 'drivers/gpu/drm/amd/amdkfd/kfd_svm.c')
-rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 437 |
1 files changed, 380 insertions, 57 deletions
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index f2805ba74c80..b71d47afd243 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -45,6 +45,11 @@ */ #define AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING 2000 +struct criu_svm_metadata { + struct list_head list; + struct kfd_criu_svm_range_priv_data data; +}; + static void svm_range_evict_svm_bo_worker(struct work_struct *work); static bool svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni, @@ -1224,19 +1229,20 @@ svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start, if (r) break; } - amdgpu_amdkfd_flush_gpu_tlb_pasid(pdd->dev->adev, - p->pasid, TLB_FLUSH_HEAVYWEIGHT); + kfd_flush_tlb(pdd, TLB_FLUSH_HEAVYWEIGHT); } return r; } static int -svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm, - struct svm_range *prange, unsigned long offset, - unsigned long npages, bool readonly, dma_addr_t *dma_addr, - struct amdgpu_device *bo_adev, struct dma_fence **fence) +svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange, + unsigned long offset, unsigned long npages, bool readonly, + dma_addr_t *dma_addr, struct amdgpu_device *bo_adev, + struct dma_fence **fence) { + struct amdgpu_device *adev = pdd->dev->adev; + struct amdgpu_vm *vm = drm_priv_to_vm(pdd->drm_priv); bool table_freed = false; uint64_t pte_flags; unsigned long last_start; @@ -1300,12 +1306,8 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm, if (fence) *fence = dma_fence_get(vm->last_update); - if (table_freed) { - struct kfd_process *p; - - p = container_of(prange->svms, struct kfd_process, svms); - amdgpu_amdkfd_flush_gpu_tlb_pasid(adev, p->pasid, TLB_FLUSH_LEGACY); - } + if (table_freed) + kfd_flush_tlb(pdd, TLB_FLUSH_LEGACY); out: return r; } @@ -1346,8 +1348,7 @@ svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset, continue; } - r = svm_range_map_to_gpu(pdd->dev->adev, drm_priv_to_vm(pdd->drm_priv), - prange, offset, npages, readonly, + r = svm_range_map_to_gpu(pdd, prange, offset, npages, readonly, prange->dma_addr[gpuidx], bo_adev, wait ? &fence : NULL); if (r) @@ -1643,13 +1644,14 @@ static void svm_range_restore_work(struct work_struct *work) pr_debug("restore svm ranges\n"); - /* kfd_process_notifier_release destroys this worker thread. So during - * the lifetime of this thread, kfd_process and mm will be valid. - */ p = container_of(svms, struct kfd_process, svms); - mm = p->mm; - if (!mm) + + /* Keep mm reference when svm_range_validate_and_map ranges */ + mm = get_task_mm(p->lead_thread); + if (!mm) { + pr_debug("svms 0x%p process mm gone\n", svms); return; + } svm_range_list_lock_and_flush_work(svms, mm); mutex_lock(&svms->lock); @@ -1703,6 +1705,7 @@ static void svm_range_restore_work(struct work_struct *work) out_reschedule: mutex_unlock(&svms->lock); mmap_write_unlock(mm); + mmput(mm); /* If validation failed, reschedule another attempt */ if (evicted_ranges) { @@ -1985,10 +1988,9 @@ svm_range_update_notifier_and_interval_tree(struct mm_struct *mm, } static void -svm_range_handle_list_op(struct svm_range_list *svms, struct svm_range *prange) +svm_range_handle_list_op(struct svm_range_list *svms, struct svm_range *prange, + struct mm_struct *mm) { - struct mm_struct *mm = prange->work_item.mm; - switch (prange->work_item.op) { case SVM_OP_NULL: pr_debug("NULL OP 0x%p prange 0x%p [0x%lx 0x%lx]\n", @@ -2065,40 +2067,44 @@ static void svm_range_deferred_list_work(struct work_struct *work) struct svm_range_list *svms; struct svm_range *prange; struct mm_struct *mm; - struct kfd_process *p; svms = container_of(work, struct svm_range_list, deferred_list_work); pr_debug("enter svms 0x%p\n", svms); - p = container_of(svms, struct kfd_process, svms); - /* Avoid mm is gone when inserting mmu notifier */ - mm = get_task_mm(p->lead_thread); - if (!mm) { - pr_debug("svms 0x%p process mm gone\n", svms); - return; - } -retry: - mmap_write_lock(mm); - - /* Checking for the need to drain retry faults must be inside - * mmap write lock to serialize with munmap notifiers. - */ - if (unlikely(atomic_read(&svms->drain_pagefaults))) { - mmap_write_unlock(mm); - svm_range_drain_retry_fault(svms); - goto retry; - } - spin_lock(&svms->deferred_list_lock); while (!list_empty(&svms->deferred_range_list)) { prange = list_first_entry(&svms->deferred_range_list, struct svm_range, deferred_list); - list_del_init(&prange->deferred_list); spin_unlock(&svms->deferred_list_lock); pr_debug("prange 0x%p [0x%lx 0x%lx] op %d\n", prange, prange->start, prange->last, prange->work_item.op); + mm = prange->work_item.mm; +retry: + mmap_write_lock(mm); + + /* Checking for the need to drain retry faults must be inside + * mmap write lock to serialize with munmap notifiers. + */ + if (unlikely(atomic_read(&svms->drain_pagefaults))) { + mmap_write_unlock(mm); + svm_range_drain_retry_fault(svms); + goto retry; + } + + /* Remove from deferred_list must be inside mmap write lock, for + * two race cases: + * 1. unmap_from_cpu may change work_item.op and add the range + * to deferred_list again, cause use after free bug. + * 2. svm_range_list_lock_and_flush_work may hold mmap write + * lock and continue because deferred_list is empty, but + * deferred_list work is actually waiting for mmap lock. + */ + spin_lock(&svms->deferred_list_lock); + list_del_init(&prange->deferred_list); + spin_unlock(&svms->deferred_list_lock); + mutex_lock(&svms->lock); mutex_lock(&prange->migrate_mutex); while (!list_empty(&prange->child_list)) { @@ -2109,19 +2115,20 @@ retry: pr_debug("child prange 0x%p op %d\n", pchild, pchild->work_item.op); list_del_init(&pchild->child_list); - svm_range_handle_list_op(svms, pchild); + svm_range_handle_list_op(svms, pchild, mm); } mutex_unlock(&prange->migrate_mutex); - svm_range_handle_list_op(svms, prange); + svm_range_handle_list_op(svms, prange, mm); mutex_unlock(&svms->lock); + mmap_write_unlock(mm); + + /* Pairs with mmget in svm_range_add_list_work */ + mmput(mm); spin_lock(&svms->deferred_list_lock); } spin_unlock(&svms->deferred_list_lock); - - mmap_write_unlock(mm); - mmput(mm); pr_debug("exit svms 0x%p\n", svms); } @@ -2139,6 +2146,9 @@ svm_range_add_list_work(struct svm_range_list *svms, struct svm_range *prange, prange->work_item.op = op; } else { prange->work_item.op = op; + + /* Pairs with mmput in deferred_list_work */ + mmget(mm); prange->work_item.mm = mm; list_add_tail(&prange->deferred_list, &prange->svms->deferred_range_list); @@ -2830,6 +2840,8 @@ void svm_range_list_fini(struct kfd_process *p) pr_debug("pasid 0x%x svms 0x%p\n", p->pasid, &p->svms); + cancel_delayed_work_sync(&p->svms.restore_work); + /* Ensure list work is finished before process is destroyed */ flush_work(&p->svms.deferred_list_work); @@ -2840,7 +2852,6 @@ void svm_range_list_fini(struct kfd_process *p) atomic_inc(&p->svms.drain_pagefaults); svm_range_drain_retry_fault(&p->svms); - list_for_each_entry_safe(prange, next, &p->svms.list, list) { svm_range_unlink(prange); svm_range_remove_notifier(prange); @@ -2865,6 +2876,7 @@ int svm_range_list_init(struct kfd_process *p) INIT_DELAYED_WORK(&svms->restore_work, svm_range_restore_work); INIT_WORK(&svms->deferred_list_work, svm_range_deferred_list_work); INIT_LIST_HEAD(&svms->deferred_range_list); + INIT_LIST_HEAD(&svms->criu_svm_metadata_list); spin_lock_init(&svms->deferred_list_lock); for (i = 0; i < p->n_pdds; i++) @@ -3193,10 +3205,10 @@ static void svm_range_evict_svm_bo_worker(struct work_struct *work) } static int -svm_range_set_attr(struct kfd_process *p, uint64_t start, uint64_t size, - uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs) +svm_range_set_attr(struct kfd_process *p, struct mm_struct *mm, + uint64_t start, uint64_t size, uint32_t nattr, + struct kfd_ioctl_svm_attribute *attrs) { - struct mm_struct *mm = current->mm; struct list_head update_list; struct list_head insert_list; struct list_head remove_list; @@ -3295,8 +3307,9 @@ out: } static int -svm_range_get_attr(struct kfd_process *p, uint64_t start, uint64_t size, - uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs) +svm_range_get_attr(struct kfd_process *p, struct mm_struct *mm, + uint64_t start, uint64_t size, uint32_t nattr, + struct kfd_ioctl_svm_attribute *attrs) { DECLARE_BITMAP(bitmap_access, MAX_GPU_INSTANCE); DECLARE_BITMAP(bitmap_aip, MAX_GPU_INSTANCE); @@ -3306,7 +3319,6 @@ svm_range_get_attr(struct kfd_process *p, uint64_t start, uint64_t size, bool get_accessible = false; bool get_flags = false; uint64_t last = start + size - 1UL; - struct mm_struct *mm = current->mm; uint8_t granularity = 0xff; struct interval_tree_node *node; struct svm_range_list *svms; @@ -3471,10 +3483,321 @@ fill_values: return 0; } +int kfd_criu_resume_svm(struct kfd_process *p) +{ + struct kfd_ioctl_svm_attribute *set_attr_new, *set_attr = NULL; + int nattr_common = 4, nattr_accessibility = 1; + struct criu_svm_metadata *criu_svm_md = NULL; + struct svm_range_list *svms = &p->svms; + struct criu_svm_metadata *next = NULL; + uint32_t set_flags = 0xffffffff; + int i, j, num_attrs, ret = 0; + uint64_t set_attr_size; + struct mm_struct *mm; + + if (list_empty(&svms->criu_svm_metadata_list)) { + pr_debug("No SVM data from CRIU restore stage 2\n"); + return ret; + } + + mm = get_task_mm(p->lead_thread); + if (!mm) { + pr_err("failed to get mm for the target process\n"); + return -ESRCH; + } + + num_attrs = nattr_common + (nattr_accessibility * p->n_pdds); + + i = j = 0; + list_for_each_entry(criu_svm_md, &svms->criu_svm_metadata_list, list) { + pr_debug("criu_svm_md[%d]\n\tstart: 0x%llx size: 0x%llx (npages)\n", + i, criu_svm_md->data.start_addr, criu_svm_md->data.size); + + for (j = 0; j < num_attrs; j++) { + pr_debug("\ncriu_svm_md[%d]->attrs[%d].type : 0x%x\ncriu_svm_md[%d]->attrs[%d].value : 0x%x\n", + i, j, criu_svm_md->data.attrs[j].type, + i, j, criu_svm_md->data.attrs[j].value); + switch (criu_svm_md->data.attrs[j].type) { + /* During Checkpoint operation, the query for + * KFD_IOCTL_SVM_ATTR_PREFETCH_LOC attribute might + * return KFD_IOCTL_SVM_LOCATION_UNDEFINED if they were + * not used by the range which was checkpointed. Care + * must be taken to not restore with an invalid value + * otherwise the gpuidx value will be invalid and + * set_attr would eventually fail so just replace those + * with another dummy attribute such as + * KFD_IOCTL_SVM_ATTR_SET_FLAGS. + */ + case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC: + if (criu_svm_md->data.attrs[j].value == + KFD_IOCTL_SVM_LOCATION_UNDEFINED) { + criu_svm_md->data.attrs[j].type = + KFD_IOCTL_SVM_ATTR_SET_FLAGS; + criu_svm_md->data.attrs[j].value = 0; + } + break; + case KFD_IOCTL_SVM_ATTR_SET_FLAGS: + set_flags = criu_svm_md->data.attrs[j].value; + break; + default: + break; + } + } + + /* CLR_FLAGS is not available via get_attr during checkpoint but + * it needs to be inserted before restoring the ranges so + * allocate extra space for it before calling set_attr + */ + set_attr_size = sizeof(struct kfd_ioctl_svm_attribute) * + (num_attrs + 1); + set_attr_new = krealloc(set_attr, set_attr_size, + GFP_KERNEL); + if (!set_attr_new) { + ret = -ENOMEM; + goto exit; + } + set_attr = set_attr_new; + + memcpy(set_attr, criu_svm_md->data.attrs, num_attrs * + sizeof(struct kfd_ioctl_svm_attribute)); + set_attr[num_attrs].type = KFD_IOCTL_SVM_ATTR_CLR_FLAGS; + set_attr[num_attrs].value = ~set_flags; + + ret = svm_range_set_attr(p, mm, criu_svm_md->data.start_addr, + criu_svm_md->data.size, num_attrs + 1, + set_attr); + if (ret) { + pr_err("CRIU: failed to set range attributes\n"); + goto exit; + } + + i++; + } +exit: + kfree(set_attr); + list_for_each_entry_safe(criu_svm_md, next, &svms->criu_svm_metadata_list, list) { + pr_debug("freeing criu_svm_md[]\n\tstart: 0x%llx\n", + criu_svm_md->data.start_addr); + kfree(criu_svm_md); + } + + mmput(mm); + return ret; + +} + +int kfd_criu_restore_svm(struct kfd_process *p, + uint8_t __user *user_priv_ptr, + uint64_t *priv_data_offset, + uint64_t max_priv_data_size) +{ + uint64_t svm_priv_data_size, svm_object_md_size, svm_attrs_size; + int nattr_common = 4, nattr_accessibility = 1; + struct criu_svm_metadata *criu_svm_md = NULL; + struct svm_range_list *svms = &p->svms; + uint32_t num_devices; + int ret = 0; + + num_devices = p->n_pdds; + /* Handle one SVM range object at a time, also the number of gpus are + * assumed to be same on the restore node, checking must be done while + * evaluating the topology earlier + */ + + svm_attrs_size = sizeof(struct kfd_ioctl_svm_attribute) * + (nattr_common + nattr_accessibility * num_devices); + svm_object_md_size = sizeof(struct criu_svm_metadata) + svm_attrs_size; + + svm_priv_data_size = sizeof(struct kfd_criu_svm_range_priv_data) + + svm_attrs_size; + + criu_svm_md = kzalloc(svm_object_md_size, GFP_KERNEL); + if (!criu_svm_md) { + pr_err("failed to allocate memory to store svm metadata\n"); + return -ENOMEM; + } + if (*priv_data_offset + svm_priv_data_size > max_priv_data_size) { + ret = -EINVAL; + goto exit; + } + + ret = copy_from_user(&criu_svm_md->data, user_priv_ptr + *priv_data_offset, + svm_priv_data_size); + if (ret) { + ret = -EFAULT; + goto exit; + } + *priv_data_offset += svm_priv_data_size; + + list_add_tail(&criu_svm_md->list, &svms->criu_svm_metadata_list); + + return 0; + + +exit: + kfree(criu_svm_md); + return ret; +} + +int svm_range_get_info(struct kfd_process *p, uint32_t *num_svm_ranges, + uint64_t *svm_priv_data_size) +{ + uint64_t total_size, accessibility_size, common_attr_size; + int nattr_common = 4, nattr_accessibility = 1; + int num_devices = p->n_pdds; + struct svm_range_list *svms; + struct svm_range *prange; + uint32_t count = 0; + + *svm_priv_data_size = 0; + + svms = &p->svms; + if (!svms) + return -EINVAL; + + mutex_lock(&svms->lock); + list_for_each_entry(prange, &svms->list, list) { + pr_debug("prange: 0x%p start: 0x%lx\t npages: 0x%llx\t end: 0x%llx\n", + prange, prange->start, prange->npages, + prange->start + prange->npages - 1); + count++; + } + mutex_unlock(&svms->lock); + + *num_svm_ranges = count; + /* Only the accessbility attributes need to be queried for all the gpus + * individually, remaining ones are spanned across the entire process + * regardless of the various gpu nodes. Of the remaining attributes, + * KFD_IOCTL_SVM_ATTR_CLR_FLAGS need not be saved. + * + * KFD_IOCTL_SVM_ATTR_PREFERRED_LOC + * KFD_IOCTL_SVM_ATTR_PREFETCH_LOC + * KFD_IOCTL_SVM_ATTR_SET_FLAGS + * KFD_IOCTL_SVM_ATTR_GRANULARITY + * + * ** ACCESSBILITY ATTRIBUTES ** + * (Considered as one, type is altered during query, value is gpuid) + * KFD_IOCTL_SVM_ATTR_ACCESS + * KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE + * KFD_IOCTL_SVM_ATTR_NO_ACCESS + */ + if (*num_svm_ranges > 0) { + common_attr_size = sizeof(struct kfd_ioctl_svm_attribute) * + nattr_common; + accessibility_size = sizeof(struct kfd_ioctl_svm_attribute) * + nattr_accessibility * num_devices; + + total_size = sizeof(struct kfd_criu_svm_range_priv_data) + + common_attr_size + accessibility_size; + + *svm_priv_data_size = *num_svm_ranges * total_size; + } + + pr_debug("num_svm_ranges %u total_priv_size %llu\n", *num_svm_ranges, + *svm_priv_data_size); + return 0; +} + +int kfd_criu_checkpoint_svm(struct kfd_process *p, + uint8_t __user *user_priv_data, + uint64_t *priv_data_offset) +{ + struct kfd_criu_svm_range_priv_data *svm_priv = NULL; + struct kfd_ioctl_svm_attribute *query_attr = NULL; + uint64_t svm_priv_data_size, query_attr_size = 0; + int index, nattr_common = 4, ret = 0; + struct svm_range_list *svms; + int num_devices = p->n_pdds; + struct svm_range *prange; + struct mm_struct *mm; + + svms = &p->svms; + if (!svms) + return -EINVAL; + + mm = get_task_mm(p->lead_thread); + if (!mm) { + pr_err("failed to get mm for the target process\n"); + return -ESRCH; + } + + query_attr_size = sizeof(struct kfd_ioctl_svm_attribute) * + (nattr_common + num_devices); + + query_attr = kzalloc(query_attr_size, GFP_KERNEL); + if (!query_attr) { + ret = -ENOMEM; + goto exit; + } + + query_attr[0].type = KFD_IOCTL_SVM_ATTR_PREFERRED_LOC; + query_attr[1].type = KFD_IOCTL_SVM_ATTR_PREFETCH_LOC; + query_attr[2].type = KFD_IOCTL_SVM_ATTR_SET_FLAGS; + query_attr[3].type = KFD_IOCTL_SVM_ATTR_GRANULARITY; + + for (index = 0; index < num_devices; index++) { + struct kfd_process_device *pdd = p->pdds[index]; + + query_attr[index + nattr_common].type = + KFD_IOCTL_SVM_ATTR_ACCESS; + query_attr[index + nattr_common].value = pdd->user_gpu_id; + } + + svm_priv_data_size = sizeof(*svm_priv) + query_attr_size; + + svm_priv = kzalloc(svm_priv_data_size, GFP_KERNEL); + if (!svm_priv) { + ret = -ENOMEM; + goto exit_query; + } + + index = 0; + list_for_each_entry(prange, &svms->list, list) { + + svm_priv->object_type = KFD_CRIU_OBJECT_TYPE_SVM_RANGE; + svm_priv->start_addr = prange->start; + svm_priv->size = prange->npages; + memcpy(&svm_priv->attrs, query_attr, query_attr_size); + pr_debug("CRIU: prange: 0x%p start: 0x%lx\t npages: 0x%llx end: 0x%llx\t size: 0x%llx\n", + prange, prange->start, prange->npages, + prange->start + prange->npages - 1, + prange->npages * PAGE_SIZE); + + ret = svm_range_get_attr(p, mm, svm_priv->start_addr, + svm_priv->size, + (nattr_common + num_devices), + svm_priv->attrs); + if (ret) { + pr_err("CRIU: failed to obtain range attributes\n"); + goto exit_priv; + } + + if (copy_to_user(user_priv_data + *priv_data_offset, svm_priv, + svm_priv_data_size)) { + pr_err("Failed to copy svm priv to user\n"); + ret = -EFAULT; + goto exit_priv; + } + + *priv_data_offset += svm_priv_data_size; + + } + + +exit_priv: + kfree(svm_priv); +exit_query: + kfree(query_attr); +exit: + mmput(mm); + return ret; +} + int svm_ioctl(struct kfd_process *p, enum kfd_ioctl_svm_op op, uint64_t start, uint64_t size, uint32_t nattrs, struct kfd_ioctl_svm_attribute *attrs) { + struct mm_struct *mm = current->mm; int r; start >>= PAGE_SHIFT; @@ -3482,10 +3805,10 @@ svm_ioctl(struct kfd_process *p, enum kfd_ioctl_svm_op op, uint64_t start, switch (op) { case KFD_IOCTL_SVM_OP_SET_ATTR: - r = svm_range_set_attr(p, start, size, nattrs, attrs); + r = svm_range_set_attr(p, mm, start, size, nattrs, attrs); break; case KFD_IOCTL_SVM_OP_GET_ATTR: - r = svm_range_get_attr(p, start, size, nattrs, attrs); + r = svm_range_get_attr(p, mm, start, size, nattrs, attrs); break; default: r = EINVAL; |