diff options
Diffstat (limited to 'include/drm/ttm')
| -rw-r--r-- | include/drm/ttm/ttm_bo_api.h | 37 | ||||
| -rw-r--r-- | include/drm/ttm/ttm_bo_driver.h | 169 | ||||
| -rw-r--r-- | include/drm/ttm/ttm_execbuf_util.h | 12 |
3 files changed, 123 insertions, 95 deletions
diff --git a/include/drm/ttm/ttm_bo_api.h b/include/drm/ttm/ttm_bo_api.h index 3cb5d848fb66..8a6aa56ece52 100644 --- a/include/drm/ttm/ttm_bo_api.h +++ b/include/drm/ttm/ttm_bo_api.h @@ -39,6 +39,7 @@ #include <linux/mm.h> #include <linux/rbtree.h> #include <linux/bitmap.h> +#include <linux/reservation.h> struct ttm_bo_device; @@ -153,7 +154,6 @@ struct ttm_tt; * Lru lists may keep one refcount, the delayed delete list, and kref != 0 * keeps one refcount. When this refcount reaches zero, * the object is destroyed. - * @event_queue: Queue for processes waiting on buffer object status change. * @mem: structure describing current placement. * @persistent_swap_storage: Usually the swap storage is deleted for buffers * pinned in physical memory. If this behaviour is not desired, this member @@ -164,12 +164,6 @@ struct ttm_tt; * @lru: List head for the lru list. * @ddestroy: List head for the delayed destroy list. * @swap: List head for swap LRU list. - * @val_seq: Sequence of the validation holding the @reserved lock. - * Used to avoid starvation when many processes compete to validate the - * buffer. This member is protected by the bo_device::lru_lock. - * @seq_valid: The value of @val_seq is valid. This value is protected by - * the bo_device::lru_lock. - * @reserved: Deadlock-free lock used for synchronization state transitions. * @sync_obj: Pointer to a synchronization object. * @priv_flags: Flags describing buffer object internal state. * @vm_rb: Rb node for the vm rb tree. @@ -209,10 +203,9 @@ struct ttm_buffer_object { struct kref kref; struct kref list_kref; - wait_queue_head_t event_queue; /** - * Members protected by the bo::reserved lock. + * Members protected by the bo::resv::reserved lock. */ struct ttm_mem_reg mem; @@ -234,15 +227,6 @@ struct ttm_buffer_object { struct list_head ddestroy; struct list_head swap; struct list_head io_reserve_lru; - uint32_t val_seq; - bool seq_valid; - - /** - * Members protected by the bdev::lru_lock - * only when written to. - */ - - atomic_t reserved; /** * Members protected by struct buffer_object_device::fence_lock @@ -272,6 +256,9 @@ struct ttm_buffer_object { uint32_t cur_placement; struct sg_table *sg; + + struct reservation_object *resv; + struct reservation_object ttm_resv; }; /** @@ -725,18 +712,4 @@ extern ssize_t ttm_bo_io(struct ttm_bo_device *bdev, struct file *filp, extern void ttm_bo_swapout_all(struct ttm_bo_device *bdev); -/** - * ttm_bo_is_reserved - return an indication if a ttm buffer object is reserved - * - * @bo: The buffer object to check. - * - * This function returns an indication if a bo is reserved or not, and should - * only be used to print an error when it is not from incorrect api usage, since - * there's no guarantee that it is the caller that is holding the reservation. - */ -static inline bool ttm_bo_is_reserved(struct ttm_buffer_object *bo) -{ - return atomic_read(&bo->reserved); -} - #endif diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h index 9c8dca79808e..984fc2d571a1 100644 --- a/include/drm/ttm/ttm_bo_driver.h +++ b/include/drm/ttm/ttm_bo_driver.h @@ -33,11 +33,13 @@ #include <ttm/ttm_bo_api.h> #include <ttm/ttm_memory.h> #include <ttm/ttm_module.h> +#include <ttm/ttm_placement.h> #include <drm/drm_mm.h> #include <drm/drm_global.h> #include <linux/workqueue.h> #include <linux/fs.h> #include <linux/spinlock.h> +#include <linux/reservation.h> struct ttm_backend_func { /** @@ -771,6 +773,55 @@ extern int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible); extern void ttm_mem_io_unlock(struct ttm_mem_type_manager *man); +extern void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo); +extern void ttm_bo_add_to_lru(struct ttm_buffer_object *bo); + +/** + * ttm_bo_reserve_nolru: + * + * @bo: A pointer to a struct ttm_buffer_object. + * @interruptible: Sleep interruptible if waiting. + * @no_wait: Don't sleep while trying to reserve, rather return -EBUSY. + * @use_ticket: If @bo is already reserved, Only sleep waiting for + * it to become unreserved if @ticket->stamp is older. + * + * Will not remove reserved buffers from the lru lists. + * Otherwise identical to ttm_bo_reserve. + * + * Returns: + * -EDEADLK: The reservation may cause a deadlock. + * Release all buffer reservations, wait for @bo to become unreserved and + * try again. (only if use_sequence == 1). + * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by + * a signal. Release all buffer reservations and return to user-space. + * -EBUSY: The function needed to sleep, but @no_wait was true + * -EALREADY: Bo already reserved using @ticket. This error code will only + * be returned if @use_ticket is set to true. + */ +static inline int ttm_bo_reserve_nolru(struct ttm_buffer_object *bo, + bool interruptible, + bool no_wait, bool use_ticket, + struct ww_acquire_ctx *ticket) +{ + int ret = 0; + + if (no_wait) { + bool success; + if (WARN_ON(ticket)) + return -EBUSY; + + success = ww_mutex_trylock(&bo->resv->lock); + return success ? 0 : -EBUSY; + } + + if (interruptible) + ret = ww_mutex_lock_interruptible(&bo->resv->lock, ticket); + else + ret = ww_mutex_lock(&bo->resv->lock, ticket); + if (ret == -EINTR) + return -ERESTARTSYS; + return ret; +} /** * ttm_bo_reserve: @@ -778,8 +829,8 @@ extern void ttm_mem_io_unlock(struct ttm_mem_type_manager *man); * @bo: A pointer to a struct ttm_buffer_object. * @interruptible: Sleep interruptible if waiting. * @no_wait: Don't sleep while trying to reserve, rather return -EBUSY. - * @use_sequence: If @bo is already reserved, Only sleep waiting for - * it to become unreserved if @sequence < (@bo)->sequence. + * @use_ticket: If @bo is already reserved, Only sleep waiting for + * it to become unreserved if @ticket->stamp is older. * * Locks a buffer object for validation. (Or prevents other processes from * locking it for validation) and removes it from lru lists, while taking @@ -793,7 +844,7 @@ extern void ttm_mem_io_unlock(struct ttm_mem_type_manager *man); * Processes attempting to reserve multiple buffers other than for eviction, * (typically execbuf), should first obtain a unique 32-bit * validation sequence number, - * and call this function with @use_sequence == 1 and @sequence == the unique + * and call this function with @use_ticket == 1 and @ticket->stamp == the unique * sequence number. If upon call of this function, the buffer object is already * reserved, the validation sequence is checked against the validation * sequence of the process currently reserving the buffer, @@ -808,36 +859,31 @@ extern void ttm_mem_io_unlock(struct ttm_mem_type_manager *man); * will eventually succeed, preventing both deadlocks and starvation. * * Returns: - * -EAGAIN: The reservation may cause a deadlock. + * -EDEADLK: The reservation may cause a deadlock. * Release all buffer reservations, wait for @bo to become unreserved and * try again. (only if use_sequence == 1). * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by * a signal. Release all buffer reservations and return to user-space. * -EBUSY: The function needed to sleep, but @no_wait was true - * -EDEADLK: Bo already reserved using @sequence. This error code will only - * be returned if @use_sequence is set to true. + * -EALREADY: Bo already reserved using @ticket. This error code will only + * be returned if @use_ticket is set to true. */ -extern int ttm_bo_reserve(struct ttm_buffer_object *bo, - bool interruptible, - bool no_wait, bool use_sequence, uint32_t sequence); +static inline int ttm_bo_reserve(struct ttm_buffer_object *bo, + bool interruptible, + bool no_wait, bool use_ticket, + struct ww_acquire_ctx *ticket) +{ + int ret; -/** - * ttm_bo_reserve_slowpath_nolru: - * @bo: A pointer to a struct ttm_buffer_object. - * @interruptible: Sleep interruptible if waiting. - * @sequence: Set (@bo)->sequence to this value after lock - * - * This is called after ttm_bo_reserve returns -EAGAIN and we backed off - * from all our other reservations. Because there are no other reservations - * held by us, this function cannot deadlock any more. - * - * Will not remove reserved buffers from the lru lists. - * Otherwise identical to ttm_bo_reserve_slowpath. - */ -extern int ttm_bo_reserve_slowpath_nolru(struct ttm_buffer_object *bo, - bool interruptible, - uint32_t sequence); + WARN_ON(!atomic_read(&bo->kref.refcount)); + ret = ttm_bo_reserve_nolru(bo, interruptible, no_wait, use_ticket, + ticket); + if (likely(ret == 0)) + ttm_bo_del_sub_from_lru(bo); + + return ret; +} /** * ttm_bo_reserve_slowpath: @@ -849,54 +895,57 @@ extern int ttm_bo_reserve_slowpath_nolru(struct ttm_buffer_object *bo, * from all our other reservations. Because there are no other reservations * held by us, this function cannot deadlock any more. */ -extern int ttm_bo_reserve_slowpath(struct ttm_buffer_object *bo, - bool interruptible, uint32_t sequence); +static inline int ttm_bo_reserve_slowpath(struct ttm_buffer_object *bo, + bool interruptible, + struct ww_acquire_ctx *ticket) +{ + int ret = 0; -/** - * ttm_bo_reserve_nolru: - * - * @bo: A pointer to a struct ttm_buffer_object. - * @interruptible: Sleep interruptible if waiting. - * @no_wait: Don't sleep while trying to reserve, rather return -EBUSY. - * @use_sequence: If @bo is already reserved, Only sleep waiting for - * it to become unreserved if @sequence < (@bo)->sequence. - * - * Will not remove reserved buffers from the lru lists. - * Otherwise identical to ttm_bo_reserve. - * - * Returns: - * -EAGAIN: The reservation may cause a deadlock. - * Release all buffer reservations, wait for @bo to become unreserved and - * try again. (only if use_sequence == 1). - * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by - * a signal. Release all buffer reservations and return to user-space. - * -EBUSY: The function needed to sleep, but @no_wait was true - * -EDEADLK: Bo already reserved using @sequence. This error code will only - * be returned if @use_sequence is set to true. - */ -extern int ttm_bo_reserve_nolru(struct ttm_buffer_object *bo, - bool interruptible, - bool no_wait, bool use_sequence, - uint32_t sequence); + WARN_ON(!atomic_read(&bo->kref.refcount)); + + if (interruptible) + ret = ww_mutex_lock_slow_interruptible(&bo->resv->lock, + ticket); + else + ww_mutex_lock_slow(&bo->resv->lock, ticket); + + if (likely(ret == 0)) + ttm_bo_del_sub_from_lru(bo); + else if (ret == -EINTR) + ret = -ERESTARTSYS; + + return ret; +} /** - * ttm_bo_unreserve - * + * ttm_bo_unreserve_ticket * @bo: A pointer to a struct ttm_buffer_object. + * @ticket: ww_acquire_ctx used for reserving * - * Unreserve a previous reservation of @bo. + * Unreserve a previous reservation of @bo made with @ticket. */ -extern void ttm_bo_unreserve(struct ttm_buffer_object *bo); +static inline void ttm_bo_unreserve_ticket(struct ttm_buffer_object *bo, + struct ww_acquire_ctx *t) +{ + if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { + spin_lock(&bo->glob->lru_lock); + ttm_bo_add_to_lru(bo); + spin_unlock(&bo->glob->lru_lock); + } + ww_mutex_unlock(&bo->resv->lock); +} /** - * ttm_bo_unreserve_locked + * ttm_bo_unreserve * * @bo: A pointer to a struct ttm_buffer_object. * * Unreserve a previous reservation of @bo. - * Needs to be called with struct ttm_bo_global::lru_lock held. */ -extern void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo); +static inline void ttm_bo_unreserve(struct ttm_buffer_object *bo) +{ + ttm_bo_unreserve_ticket(bo, NULL); +} /* * ttm_bo_util.c diff --git a/include/drm/ttm/ttm_execbuf_util.h b/include/drm/ttm/ttm_execbuf_util.h index 547e19f06e57..ec8a1d306510 100644 --- a/include/drm/ttm/ttm_execbuf_util.h +++ b/include/drm/ttm/ttm_execbuf_util.h @@ -57,17 +57,20 @@ struct ttm_validate_buffer { /** * function ttm_eu_backoff_reservation * + * @ticket: ww_acquire_ctx from reserve call * @list: thread private list of ttm_validate_buffer structs. * * Undoes all buffer validation reservations for bos pointed to by * the list entries. */ -extern void ttm_eu_backoff_reservation(struct list_head *list); +extern void ttm_eu_backoff_reservation(struct ww_acquire_ctx *ticket, + struct list_head *list); /** * function ttm_eu_reserve_buffers * + * @ticket: [out] ww_acquire_ctx returned by call. * @list: thread private list of ttm_validate_buffer structs. * * Tries to reserve bos pointed to by the list entries for validation. @@ -90,11 +93,13 @@ extern void ttm_eu_backoff_reservation(struct list_head *list); * has failed. */ -extern int ttm_eu_reserve_buffers(struct list_head *list); +extern int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket, + struct list_head *list); /** * function ttm_eu_fence_buffer_objects. * + * @ticket: ww_acquire_ctx from reserve call * @list: thread private list of ttm_validate_buffer structs. * @sync_obj: The new sync object for the buffers. * @@ -104,6 +109,7 @@ extern int ttm_eu_reserve_buffers(struct list_head *list); * */ -extern void ttm_eu_fence_buffer_objects(struct list_head *list, void *sync_obj); +extern void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket, + struct list_head *list, void *sync_obj); #endif |