diff options
Diffstat (limited to 'drivers/gpu/drm/ttm/ttm_bo.c')
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo.c | 158 |
1 files changed, 85 insertions, 73 deletions
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 58c403eda04e..20ff56f27aa4 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -41,7 +41,7 @@ #include <linux/file.h> #include <linux/module.h> #include <linux/atomic.h> -#include <linux/reservation.h> +#include <linux/dma-resv.h> static void ttm_bo_global_kobj_release(struct kobject *kobj); @@ -160,7 +160,8 @@ static void ttm_bo_release_list(struct kref *list_kref) ttm_tt_destroy(bo->ttm); atomic_dec(&bo->bdev->glob->bo_count); dma_fence_put(bo->moving); - reservation_object_fini(&bo->ttm_resv); + if (!ttm_bo_uses_embedded_gem_object(bo)) + dma_resv_fini(&bo->base._resv); mutex_destroy(&bo->wu_mutex); bo->destroy(bo); ttm_mem_global_free(bdev->glob->mem_glob, acc_size); @@ -172,7 +173,7 @@ static void ttm_bo_add_mem_to_lru(struct ttm_buffer_object *bo, struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_type_manager *man; - reservation_object_assert_held(bo->resv); + dma_resv_assert_held(bo->base.resv); if (!list_empty(&bo->lru)) return; @@ -243,7 +244,7 @@ static void ttm_bo_bulk_move_set_pos(struct ttm_lru_bulk_move_pos *pos, void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo, struct ttm_lru_bulk_move *bulk) { - reservation_object_assert_held(bo->resv); + dma_resv_assert_held(bo->base.resv); ttm_bo_del_from_lru(bo); ttm_bo_add_to_lru(bo); @@ -276,8 +277,8 @@ void ttm_bo_bulk_move_lru_tail(struct ttm_lru_bulk_move *bulk) if (!pos->first) continue; - reservation_object_assert_held(pos->first->resv); - reservation_object_assert_held(pos->last->resv); + dma_resv_assert_held(pos->first->base.resv); + dma_resv_assert_held(pos->last->base.resv); man = &pos->first->bdev->man[TTM_PL_TT]; list_bulk_move_tail(&man->lru[i], &pos->first->lru, @@ -291,8 +292,8 @@ void ttm_bo_bulk_move_lru_tail(struct ttm_lru_bulk_move *bulk) if (!pos->first) continue; - reservation_object_assert_held(pos->first->resv); - reservation_object_assert_held(pos->last->resv); + dma_resv_assert_held(pos->first->base.resv); + dma_resv_assert_held(pos->last->base.resv); man = &pos->first->bdev->man[TTM_PL_VRAM]; list_bulk_move_tail(&man->lru[i], &pos->first->lru, @@ -306,8 +307,8 @@ void ttm_bo_bulk_move_lru_tail(struct ttm_lru_bulk_move *bulk) if (!pos->first) continue; - reservation_object_assert_held(pos->first->resv); - reservation_object_assert_held(pos->last->resv); + dma_resv_assert_held(pos->first->base.resv); + dma_resv_assert_held(pos->last->base.resv); lru = &pos->first->bdev->glob->swap_lru[i]; list_bulk_move_tail(lru, &pos->first->swap, &pos->last->swap); @@ -438,32 +439,32 @@ static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) { int r; - if (bo->resv == &bo->ttm_resv) + if (bo->base.resv == &bo->base._resv) return 0; - BUG_ON(!reservation_object_trylock(&bo->ttm_resv)); + BUG_ON(!dma_resv_trylock(&bo->base._resv)); - r = reservation_object_copy_fences(&bo->ttm_resv, bo->resv); + r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv); if (r) - reservation_object_unlock(&bo->ttm_resv); + dma_resv_unlock(&bo->base._resv); return r; } static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) { - struct reservation_object_list *fobj; + struct dma_resv_list *fobj; struct dma_fence *fence; int i; - fobj = reservation_object_get_list(&bo->ttm_resv); - fence = reservation_object_get_excl(&bo->ttm_resv); + fobj = dma_resv_get_list(&bo->base._resv); + fence = dma_resv_get_excl(&bo->base._resv); if (fence && !fence->ops->signaled) dma_fence_enable_sw_signaling(fence); for (i = 0; fobj && i < fobj->shared_count; ++i) { fence = rcu_dereference_protected(fobj->shared[i], - reservation_object_held(bo->resv)); + dma_resv_held(bo->base.resv)); if (!fence->ops->signaled) dma_fence_enable_sw_signaling(fence); @@ -481,23 +482,23 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) /* Last resort, if we fail to allocate memory for the * fences block for the BO to become idle */ - reservation_object_wait_timeout_rcu(bo->resv, true, false, + dma_resv_wait_timeout_rcu(bo->base.resv, true, false, 30 * HZ); spin_lock(&glob->lru_lock); goto error; } spin_lock(&glob->lru_lock); - ret = reservation_object_trylock(bo->resv) ? 0 : -EBUSY; + ret = dma_resv_trylock(bo->base.resv) ? 0 : -EBUSY; if (!ret) { - if (reservation_object_test_signaled_rcu(&bo->ttm_resv, true)) { + if (dma_resv_test_signaled_rcu(&bo->base._resv, true)) { ttm_bo_del_from_lru(bo); spin_unlock(&glob->lru_lock); - if (bo->resv != &bo->ttm_resv) - reservation_object_unlock(&bo->ttm_resv); + if (bo->base.resv != &bo->base._resv) + dma_resv_unlock(&bo->base._resv); ttm_bo_cleanup_memtype_use(bo); - reservation_object_unlock(bo->resv); + dma_resv_unlock(bo->base.resv); return; } @@ -513,10 +514,10 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) ttm_bo_add_to_lru(bo); } - reservation_object_unlock(bo->resv); + dma_resv_unlock(bo->base.resv); } - if (bo->resv != &bo->ttm_resv) - reservation_object_unlock(&bo->ttm_resv); + if (bo->base.resv != &bo->base._resv) + dma_resv_unlock(&bo->base._resv); error: kref_get(&bo->list_kref); @@ -545,15 +546,15 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, bool unlock_resv) { struct ttm_bo_global *glob = bo->bdev->glob; - struct reservation_object *resv; + struct dma_resv *resv; int ret; if (unlikely(list_empty(&bo->ddestroy))) - resv = bo->resv; + resv = bo->base.resv; else - resv = &bo->ttm_resv; + resv = &bo->base._resv; - if (reservation_object_test_signaled_rcu(resv, true)) + if (dma_resv_test_signaled_rcu(resv, true)) ret = 0; else ret = -EBUSY; @@ -562,10 +563,10 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, long lret; if (unlock_resv) - reservation_object_unlock(bo->resv); + dma_resv_unlock(bo->base.resv); spin_unlock(&glob->lru_lock); - lret = reservation_object_wait_timeout_rcu(resv, true, + lret = dma_resv_wait_timeout_rcu(resv, true, interruptible, 30 * HZ); @@ -575,7 +576,7 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, return -EBUSY; spin_lock(&glob->lru_lock); - if (unlock_resv && !reservation_object_trylock(bo->resv)) { + if (unlock_resv && !dma_resv_trylock(bo->base.resv)) { /* * We raced, and lost, someone else holds the reservation now, * and is probably busy in ttm_bo_cleanup_memtype_use. @@ -592,7 +593,7 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, if (ret || unlikely(list_empty(&bo->ddestroy))) { if (unlock_resv) - reservation_object_unlock(bo->resv); + dma_resv_unlock(bo->base.resv); spin_unlock(&glob->lru_lock); return ret; } @@ -605,7 +606,7 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, ttm_bo_cleanup_memtype_use(bo); if (unlock_resv) - reservation_object_unlock(bo->resv); + dma_resv_unlock(bo->base.resv); return 0; } @@ -631,14 +632,14 @@ static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) kref_get(&bo->list_kref); list_move_tail(&bo->ddestroy, &removed); - if (remove_all || bo->resv != &bo->ttm_resv) { + if (remove_all || bo->base.resv != &bo->base._resv) { spin_unlock(&glob->lru_lock); - reservation_object_lock(bo->resv, NULL); + dma_resv_lock(bo->base.resv, NULL); spin_lock(&glob->lru_lock); ttm_bo_cleanup_refs(bo, false, !remove_all, true); - } else if (reservation_object_trylock(bo->resv)) { + } else if (dma_resv_trylock(bo->base.resv)) { ttm_bo_cleanup_refs(bo, false, !remove_all, true); } else { spin_unlock(&glob->lru_lock); @@ -671,7 +672,10 @@ static void ttm_bo_release(struct kref *kref) struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; - drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node); + if (bo->bdev->driver->release_notify) + bo->bdev->driver->release_notify(bo); + + drm_vma_offset_remove(&bdev->vma_manager, &bo->base.vma_node); ttm_mem_io_lock(man, false); ttm_mem_io_free_vm(bo); ttm_mem_io_unlock(man); @@ -707,7 +711,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, struct ttm_placement placement; int ret = 0; - reservation_object_assert_held(bo->resv); + dma_resv_assert_held(bo->base.resv); placement.num_placement = 0; placement.num_busy_placement = 0; @@ -777,8 +781,8 @@ static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, { bool ret = false; - if (bo->resv == ctx->resv) { - reservation_object_assert_held(bo->resv); + if (bo->base.resv == ctx->resv) { + dma_resv_assert_held(bo->base.resv); if (ctx->flags & TTM_OPT_FLAG_ALLOW_RES_EVICT || !list_empty(&bo->ddestroy)) ret = true; @@ -786,7 +790,7 @@ static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, if (busy) *busy = false; } else { - ret = reservation_object_trylock(bo->resv); + ret = dma_resv_trylock(bo->base.resv); *locked = ret; if (busy) *busy = !ret; @@ -814,10 +818,10 @@ static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo, return -EBUSY; if (ctx->interruptible) - r = reservation_object_lock_interruptible(busy_bo->resv, + r = dma_resv_lock_interruptible(busy_bo->base.resv, ticket); else - r = reservation_object_lock(busy_bo->resv, ticket); + r = dma_resv_lock(busy_bo->base.resv, ticket); /* * TODO: It would be better to keep the BO locked until allocation is at @@ -825,7 +829,7 @@ static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo, * of TTM. */ if (!r) - reservation_object_unlock(busy_bo->resv); + dma_resv_unlock(busy_bo->base.resv); return r == -EDEADLK ? -EBUSY : r; } @@ -850,8 +854,8 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev, if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked, &busy)) { - if (busy && !busy_bo && - bo->resv->lock.ctx != ticket) + if (busy && !busy_bo && ticket != + dma_resv_locking_ctx(bo->base.resv)) busy_bo = bo; continue; } @@ -859,7 +863,7 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev, if (place && !bdev->driver->eviction_valuable(bo, place)) { if (locked) - reservation_object_unlock(bo->resv); + dma_resv_unlock(bo->base.resv); continue; } break; @@ -931,9 +935,9 @@ static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo, spin_unlock(&man->move_lock); if (fence) { - reservation_object_add_shared_fence(bo->resv, fence); + dma_resv_add_shared_fence(bo->base.resv, fence); - ret = reservation_object_reserve_shared(bo->resv, 1); + ret = dma_resv_reserve_shared(bo->base.resv, 1); if (unlikely(ret)) { dma_fence_put(fence); return ret; @@ -957,8 +961,10 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, { struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; + struct ww_acquire_ctx *ticket; int ret; + ticket = dma_resv_locking_ctx(bo->base.resv); do { ret = (*man->func->get_node)(man, bo, place, mem); if (unlikely(ret != 0)) @@ -966,7 +972,7 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, if (mem->mm_node) break; ret = ttm_mem_evict_first(bdev, mem->mem_type, place, ctx, - bo->resv->lock.ctx); + ticket); if (unlikely(ret != 0)) return ret; } while (1); @@ -1088,7 +1094,7 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, bool type_found = false; int i, ret; - ret = reservation_object_reserve_shared(bo->resv, 1); + ret = dma_resv_reserve_shared(bo->base.resv, 1); if (unlikely(ret)) return ret; @@ -1169,7 +1175,7 @@ static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, int ret = 0; struct ttm_mem_reg mem; - reservation_object_assert_held(bo->resv); + dma_resv_assert_held(bo->base.resv); mem.num_pages = bo->num_pages; mem.size = mem.num_pages << PAGE_SHIFT; @@ -1239,7 +1245,7 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, int ret; uint32_t new_flags; - reservation_object_assert_held(bo->resv); + dma_resv_assert_held(bo->base.resv); /* * Check whether we need to move buffer. */ @@ -1276,7 +1282,7 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, struct ttm_operation_ctx *ctx, size_t acc_size, struct sg_table *sg, - struct reservation_object *resv, + struct dma_resv *resv, void (*destroy) (struct ttm_buffer_object *)) { int ret = 0; @@ -1329,14 +1335,20 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, bo->acc_size = acc_size; bo->sg = sg; if (resv) { - bo->resv = resv; - reservation_object_assert_held(bo->resv); + bo->base.resv = resv; + dma_resv_assert_held(bo->base.resv); } else { - bo->resv = &bo->ttm_resv; + bo->base.resv = &bo->base._resv; + } + if (!ttm_bo_uses_embedded_gem_object(bo)) { + /* + * bo.gem is not initialized, so we have to setup the + * struct elements we want use regardless. + */ + dma_resv_init(&bo->base._resv); + drm_vma_node_reset(&bo->base.vma_node); } - reservation_object_init(&bo->ttm_resv); atomic_inc(&bo->bdev->glob->bo_count); - drm_vma_node_reset(&bo->vma_node); /* * For ttm_bo_type_device buffers, allocate @@ -1344,14 +1356,14 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, */ if (bo->type == ttm_bo_type_device || bo->type == ttm_bo_type_sg) - ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node, + ret = drm_vma_offset_add(&bdev->vma_manager, &bo->base.vma_node, bo->mem.num_pages); /* passed reservation objects should already be locked, * since otherwise lockdep will be angered in radeon. */ if (!resv) { - locked = reservation_object_trylock(bo->resv); + locked = dma_resv_trylock(bo->base.resv); WARN_ON(!locked); } @@ -1385,7 +1397,7 @@ int ttm_bo_init(struct ttm_bo_device *bdev, bool interruptible, size_t acc_size, struct sg_table *sg, - struct reservation_object *resv, + struct dma_resv *resv, void (*destroy) (struct ttm_buffer_object *)) { struct ttm_operation_ctx ctx = { interruptible, false }; @@ -1772,7 +1784,7 @@ void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) { struct ttm_bo_device *bdev = bo->bdev; - drm_vma_node_unmap(&bo->vma_node, bdev->dev_mapping); + drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping); ttm_mem_io_free_vm(bo); } @@ -1795,13 +1807,13 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, long timeout = 15 * HZ; if (no_wait) { - if (reservation_object_test_signaled_rcu(bo->resv, true)) + if (dma_resv_test_signaled_rcu(bo->base.resv, true)) return 0; else return -EBUSY; } - timeout = reservation_object_wait_timeout_rcu(bo->resv, true, + timeout = dma_resv_wait_timeout_rcu(bo->base.resv, true, interruptible, timeout); if (timeout < 0) return timeout; @@ -1809,7 +1821,7 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, if (timeout == 0) return -EBUSY; - reservation_object_add_excl_fence(bo->resv, NULL); + dma_resv_add_excl_fence(bo->base.resv, NULL); return 0; } EXPORT_SYMBOL(ttm_bo_wait); @@ -1925,7 +1937,7 @@ out: * already swapped buffer. */ if (locked) - reservation_object_unlock(bo->resv); + dma_resv_unlock(bo->base.resv); kref_put(&bo->list_kref, ttm_bo_release_list); return ret; } @@ -1963,14 +1975,14 @@ int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo) ret = mutex_lock_interruptible(&bo->wu_mutex); if (unlikely(ret != 0)) return -ERESTARTSYS; - if (!ww_mutex_is_locked(&bo->resv->lock)) + if (!dma_resv_is_locked(bo->base.resv)) goto out_unlock; - ret = reservation_object_lock_interruptible(bo->resv, NULL); + ret = dma_resv_lock_interruptible(bo->base.resv, NULL); if (ret == -EINTR) ret = -ERESTARTSYS; if (unlikely(ret != 0)) goto out_unlock; - reservation_object_unlock(bo->resv); + dma_resv_unlock(bo->base.resv); out_unlock: mutex_unlock(&bo->wu_mutex); |