diff options
Diffstat (limited to 'drivers/gpu/drm/i915/i915_active.c')
| -rw-r--r-- | drivers/gpu/drm/i915/i915_active.c | 139 |
1 files changed, 116 insertions, 23 deletions
diff --git a/drivers/gpu/drm/i915/i915_active.c b/drivers/gpu/drm/i915/i915_active.c index b0a499753526..c4048628188a 100644 --- a/drivers/gpu/drm/i915/i915_active.c +++ b/drivers/gpu/drm/i915/i915_active.c @@ -7,6 +7,7 @@ #include <linux/debugobjects.h> #include "gt/intel_context.h" +#include "gt/intel_engine_heartbeat.h" #include "gt/intel_engine_pm.h" #include "gt/intel_ring.h" @@ -390,13 +391,23 @@ out: return err; } -void i915_active_set_exclusive(struct i915_active *ref, struct dma_fence *f) +struct dma_fence * +i915_active_set_exclusive(struct i915_active *ref, struct dma_fence *f) { + struct dma_fence *prev; + /* We expect the caller to manage the exclusive timeline ordering */ GEM_BUG_ON(i915_active_is_idle(ref)); - if (!__i915_active_fence_set(&ref->excl, f)) + rcu_read_lock(); + prev = __i915_active_fence_set(&ref->excl, f); + if (prev) + prev = dma_fence_get_rcu(prev); + else atomic_inc(&ref->count); + rcu_read_unlock(); + + return prev; } bool i915_active_acquire_if_busy(struct i915_active *ref) @@ -442,6 +453,9 @@ static void enable_signaling(struct i915_active_fence *active) { struct dma_fence *fence; + if (unlikely(is_barrier(active))) + return; + fence = i915_active_fence_get(active); if (!fence) return; @@ -450,26 +464,49 @@ static void enable_signaling(struct i915_active_fence *active) dma_fence_put(fence); } -int i915_active_wait(struct i915_active *ref) +static int flush_barrier(struct active_node *it) { - struct active_node *it, *n; - int err = 0; + struct intel_engine_cs *engine; - might_sleep(); + if (likely(!is_barrier(&it->base))) + return 0; - if (!i915_active_acquire_if_busy(ref)) + engine = __barrier_to_engine(it); + smp_rmb(); /* serialise with add_active_barriers */ + if (!is_barrier(&it->base)) return 0; - /* Flush lazy signals */ + return intel_engine_flush_barriers(engine); +} + +static int flush_lazy_signals(struct i915_active *ref) +{ + struct active_node *it, *n; + int err = 0; + enable_signaling(&ref->excl); rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) { - if (is_barrier(&it->base)) /* unconnected idle barrier */ - continue; + err = flush_barrier(it); /* unconnected idle barrier? */ + if (err) + break; enable_signaling(&it->base); } - /* Any fence added after the wait begins will not be auto-signaled */ + return err; +} + +int i915_active_wait(struct i915_active *ref) +{ + int err; + + might_sleep(); + + if (!i915_active_acquire_if_busy(ref)) + return 0; + + /* Any fence added after the wait begins will not be auto-signaled */ + err = flush_lazy_signals(ref); i915_active_release(ref); if (err) return err; @@ -481,25 +518,81 @@ int i915_active_wait(struct i915_active *ref) return 0; } -int i915_request_await_active(struct i915_request *rq, struct i915_active *ref) +static int __await_active(struct i915_active_fence *active, + int (*fn)(void *arg, struct dma_fence *fence), + void *arg) +{ + struct dma_fence *fence; + + if (is_barrier(active)) /* XXX flush the barrier? */ + return 0; + + fence = i915_active_fence_get(active); + if (fence) { + int err; + + err = fn(arg, fence); + dma_fence_put(fence); + if (err < 0) + return err; + } + + return 0; +} + +static int await_active(struct i915_active *ref, + unsigned int flags, + int (*fn)(void *arg, struct dma_fence *fence), + void *arg) { int err = 0; + /* We must always wait for the exclusive fence! */ if (rcu_access_pointer(ref->excl.fence)) { - struct dma_fence *fence; - - rcu_read_lock(); - fence = dma_fence_get_rcu_safe(&ref->excl.fence); - rcu_read_unlock(); - if (fence) { - err = i915_request_await_dma_fence(rq, fence); - dma_fence_put(fence); + err = __await_active(&ref->excl, fn, arg); + if (err) + return err; + } + + if (flags & I915_ACTIVE_AWAIT_ALL && i915_active_acquire_if_busy(ref)) { + struct active_node *it, *n; + + rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) { + err = __await_active(&it->base, fn, arg); + if (err) + break; } + i915_active_release(ref); + if (err) + return err; } - /* In the future we may choose to await on all fences */ + return 0; +} - return err; +static int rq_await_fence(void *arg, struct dma_fence *fence) +{ + return i915_request_await_dma_fence(arg, fence); +} + +int i915_request_await_active(struct i915_request *rq, + struct i915_active *ref, + unsigned int flags) +{ + return await_active(ref, flags, rq_await_fence, rq); +} + +static int sw_await_fence(void *arg, struct dma_fence *fence) +{ + return i915_sw_fence_await_dma_fence(arg, fence, 0, + GFP_NOWAIT | __GFP_NOWARN); +} + +int i915_sw_fence_await_active(struct i915_sw_fence *fence, + struct i915_active *ref, + unsigned int flags) +{ + return await_active(ref, flags, sw_await_fence, fence); } #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) @@ -623,6 +716,7 @@ int i915_active_acquire_preallocate_barrier(struct i915_active *ref, * We can then use the preallocated nodes in * i915_active_acquire_barrier() */ + GEM_BUG_ON(!mask); for_each_engine_masked(engine, gt, mask, tmp) { u64 idx = engine->kernel_context->timeline->fence_context; struct llist_node *prev = first; @@ -812,7 +906,6 @@ __i915_active_fence_set(struct i915_active_fence *active, __list_del_entry(&active->cb.node); spin_unlock(prev->lock); /* serialise with prev->cb_list */ } - GEM_BUG_ON(rcu_access_pointer(active->fence) != fence); list_add_tail(&active->cb.node, &fence->cb_list); spin_unlock_irqrestore(fence->lock, flags); |