diff options
Diffstat (limited to 'drivers/gpu/drm/i915/intel_engine_cs.c')
-rw-r--r-- | drivers/gpu/drm/i915/intel_engine_cs.c | 491 |
1 files changed, 277 insertions, 214 deletions
diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c index 49fa43ff02ba..eea9bec04f1b 100644 --- a/drivers/gpu/drm/i915/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/intel_engine_cs.c @@ -84,7 +84,6 @@ static const struct engine_class_info intel_engine_classes[] = { #define MAX_MMIO_BASES 3 struct engine_info { unsigned int hw_id; - unsigned int uabi_id; u8 class; u8 instance; /* mmio bases table *must* be sorted in reverse gen order */ @@ -95,27 +94,24 @@ struct engine_info { }; static const struct engine_info intel_engines[] = { - [RCS] = { - .hw_id = RCS_HW, - .uabi_id = I915_EXEC_RENDER, + [RCS0] = { + .hw_id = RCS0_HW, .class = RENDER_CLASS, .instance = 0, .mmio_bases = { { .gen = 1, .base = RENDER_RING_BASE } }, }, - [BCS] = { - .hw_id = BCS_HW, - .uabi_id = I915_EXEC_BLT, + [BCS0] = { + .hw_id = BCS0_HW, .class = COPY_ENGINE_CLASS, .instance = 0, .mmio_bases = { { .gen = 6, .base = BLT_RING_BASE } }, }, - [VCS] = { - .hw_id = VCS_HW, - .uabi_id = I915_EXEC_BSD, + [VCS0] = { + .hw_id = VCS0_HW, .class = VIDEO_DECODE_CLASS, .instance = 0, .mmio_bases = { @@ -124,9 +120,8 @@ static const struct engine_info intel_engines[] = { { .gen = 4, .base = BSD_RING_BASE } }, }, - [VCS2] = { - .hw_id = VCS2_HW, - .uabi_id = I915_EXEC_BSD, + [VCS1] = { + .hw_id = VCS1_HW, .class = VIDEO_DECODE_CLASS, .instance = 1, .mmio_bases = { @@ -134,27 +129,24 @@ static const struct engine_info intel_engines[] = { { .gen = 8, .base = GEN8_BSD2_RING_BASE } }, }, - [VCS3] = { - .hw_id = VCS3_HW, - .uabi_id = I915_EXEC_BSD, + [VCS2] = { + .hw_id = VCS2_HW, .class = VIDEO_DECODE_CLASS, .instance = 2, .mmio_bases = { { .gen = 11, .base = GEN11_BSD3_RING_BASE } }, }, - [VCS4] = { - .hw_id = VCS4_HW, - .uabi_id = I915_EXEC_BSD, + [VCS3] = { + .hw_id = VCS3_HW, .class = VIDEO_DECODE_CLASS, .instance = 3, .mmio_bases = { { .gen = 11, .base = GEN11_BSD4_RING_BASE } }, }, - [VECS] = { - .hw_id = VECS_HW, - .uabi_id = I915_EXEC_VEBOX, + [VECS0] = { + .hw_id = VECS0_HW, .class = VIDEO_ENHANCEMENT_CLASS, .instance = 0, .mmio_bases = { @@ -162,9 +154,8 @@ static const struct engine_info intel_engines[] = { { .gen = 7, .base = VEBOX_RING_BASE } }, }, - [VECS2] = { - .hw_id = VECS2_HW, - .uabi_id = I915_EXEC_VEBOX, + [VECS1] = { + .hw_id = VECS1_HW, .class = VIDEO_ENHANCEMENT_CLASS, .instance = 1, .mmio_bases = { @@ -264,21 +255,17 @@ static void __sprint_engine_name(char *name, const struct engine_info *info) void intel_engine_set_hwsp_writemask(struct intel_engine_cs *engine, u32 mask) { - struct drm_i915_private *dev_priv = engine->i915; - i915_reg_t hwstam; - /* * Though they added more rings on g4x/ilk, they did not add * per-engine HWSTAM until gen6. */ - if (INTEL_GEN(dev_priv) < 6 && engine->class != RENDER_CLASS) + if (INTEL_GEN(engine->i915) < 6 && engine->class != RENDER_CLASS) return; - hwstam = RING_HWSTAM(engine->mmio_base); - if (INTEL_GEN(dev_priv) >= 3) - I915_WRITE(hwstam, mask); + if (INTEL_GEN(engine->i915) >= 3) + ENGINE_WRITE(engine, RING_HWSTAM, mask); else - I915_WRITE16(hwstam, mask); + ENGINE_WRITE16(engine, RING_HWSTAM, mask); } static void intel_engine_sanitize_mmio(struct intel_engine_cs *engine) @@ -313,15 +300,18 @@ intel_engine_setup(struct drm_i915_private *dev_priv, if (!engine) return -ENOMEM; + BUILD_BUG_ON(BITS_PER_TYPE(engine->mask) < I915_NUM_ENGINES); + engine->id = id; + engine->mask = BIT(id); engine->i915 = dev_priv; + engine->uncore = &dev_priv->uncore; __sprint_engine_name(engine->name, info); engine->hw_id = engine->guc_id = info->hw_id; engine->mmio_base = __engine_mmio_base(dev_priv, info->mmio_bases); engine->class = info->class; engine->instance = info->instance; - engine->uabi_id = info->uabi_id; engine->uabi_class = intel_engine_classes[info->class].uabi_class; engine->context_size = __intel_engine_context_size(dev_priv, @@ -355,15 +345,15 @@ intel_engine_setup(struct drm_i915_private *dev_priv, int intel_engines_init_mmio(struct drm_i915_private *dev_priv) { struct intel_device_info *device_info = mkwrite_device_info(dev_priv); - const unsigned int ring_mask = INTEL_INFO(dev_priv)->ring_mask; + const unsigned int engine_mask = INTEL_INFO(dev_priv)->engine_mask; struct intel_engine_cs *engine; enum intel_engine_id id; unsigned int mask = 0; unsigned int i; int err; - WARN_ON(ring_mask == 0); - WARN_ON(ring_mask & + WARN_ON(engine_mask == 0); + WARN_ON(engine_mask & GENMASK(BITS_PER_TYPE(mask) - 1, I915_NUM_ENGINES)); if (i915_inject_load_failure()) @@ -377,7 +367,7 @@ int intel_engines_init_mmio(struct drm_i915_private *dev_priv) if (err) goto cleanup; - mask |= ENGINE_MASK(i); + mask |= BIT(i); } /* @@ -385,16 +375,16 @@ int intel_engines_init_mmio(struct drm_i915_private *dev_priv) * are added to the driver by a warning and disabling the forgotten * engines. */ - if (WARN_ON(mask != ring_mask)) - device_info->ring_mask = mask; + if (WARN_ON(mask != engine_mask)) + device_info->engine_mask = mask; /* We always presume we have at least RCS available for later probing */ - if (WARN_ON(!HAS_ENGINE(dev_priv, RCS))) { + if (WARN_ON(!HAS_ENGINE(dev_priv, RCS0))) { err = -ENODEV; goto cleanup; } - RUNTIME_INFO(dev_priv)->num_rings = hweight32(mask); + RUNTIME_INFO(dev_priv)->num_engines = hweight32(mask); i915_check_and_clear_faults(dev_priv); @@ -455,12 +445,6 @@ cleanup: return err; } -void intel_engine_write_global_seqno(struct intel_engine_cs *engine, u32 seqno) -{ - intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno); - GEM_BUG_ON(intel_engine_get_seqno(engine) != seqno); -} - static void intel_engine_init_batch_pool(struct intel_engine_cs *engine) { i915_gem_batch_pool_init(&engine->batch_pool, engine); @@ -541,9 +525,7 @@ static int init_status_page(struct intel_engine_cs *engine) return PTR_ERR(obj); } - ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC); - if (ret) - goto err; + i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC); vma = i915_vma_instance(obj, &engine->i915->ggtt.vm, NULL); if (IS_ERR(vma)) { @@ -594,7 +576,6 @@ int intel_engine_setup_common(struct intel_engine_cs *engine) err = i915_timeline_init(engine->i915, &engine->timeline, - engine->name, engine->status_page.vma); if (err) goto err_hwsp; @@ -614,10 +595,44 @@ err_hwsp: return err; } -static void __intel_context_unpin(struct i915_gem_context *ctx, - struct intel_engine_cs *engine) +void intel_engines_set_scheduler_caps(struct drm_i915_private *i915) { - intel_context_unpin(to_intel_context(ctx, engine)); + static const struct { + u8 engine; + u8 sched; + } map[] = { +#define MAP(x, y) { ilog2(I915_ENGINE_HAS_##x), ilog2(I915_SCHEDULER_CAP_##y) } + MAP(PREEMPTION, PREEMPTION), + MAP(SEMAPHORES, SEMAPHORES), +#undef MAP + }; + struct intel_engine_cs *engine; + enum intel_engine_id id; + u32 enabled, disabled; + + enabled = 0; + disabled = 0; + for_each_engine(engine, i915, id) { /* all engines must agree! */ + int i; + + if (engine->schedule) + enabled |= (I915_SCHEDULER_CAP_ENABLED | + I915_SCHEDULER_CAP_PRIORITY); + else + disabled |= (I915_SCHEDULER_CAP_ENABLED | + I915_SCHEDULER_CAP_PRIORITY); + + for (i = 0; i < ARRAY_SIZE(map); i++) { + if (engine->flags & BIT(map[i].engine)) + enabled |= BIT(map[i].sched); + else + disabled |= BIT(map[i].sched); + } + } + + i915->caps.scheduler = enabled & ~disabled; + if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_ENABLED)) + i915->caps.scheduler = 0; } struct measure_breadcrumb { @@ -639,7 +654,7 @@ static int measure_breadcrumb_dw(struct intel_engine_cs *engine) return -ENOMEM; if (i915_timeline_init(engine->i915, - &frame->timeline, "measure", + &frame->timeline, engine->status_page.vma)) goto out_frame; @@ -670,6 +685,20 @@ out_frame: return dw; } +static int pin_context(struct i915_gem_context *ctx, + struct intel_engine_cs *engine, + struct intel_context **out) +{ + struct intel_context *ce; + + ce = intel_context_pin(ctx, engine); + if (IS_ERR(ce)) + return PTR_ERR(ce); + + *out = ce; + return 0; +} + /** * intel_engines_init_common - initialize cengine state which might require hw access * @engine: Engine to initialize. @@ -684,11 +713,8 @@ out_frame: int intel_engine_init_common(struct intel_engine_cs *engine) { struct drm_i915_private *i915 = engine->i915; - struct intel_context *ce; int ret; - engine->set_default_submission(engine); - /* We may need to do things with the shrinker which * require us to immediately switch back to the default * context. This can cause a problem as pinning the @@ -696,39 +722,61 @@ int intel_engine_init_common(struct intel_engine_cs *engine) * be available. To avoid this we always pin the default * context. */ - ce = intel_context_pin(i915->kernel_context, engine); - if (IS_ERR(ce)) - return PTR_ERR(ce); + ret = pin_context(i915->kernel_context, engine, + &engine->kernel_context); + if (ret) + return ret; /* * Similarly the preempt context must always be available so that - * we can interrupt the engine at any time. + * we can interrupt the engine at any time. However, as preemption + * is optional, we allow it to fail. */ - if (i915->preempt_context) { - ce = intel_context_pin(i915->preempt_context, engine); - if (IS_ERR(ce)) { - ret = PTR_ERR(ce); - goto err_unpin_kernel; - } - } + if (i915->preempt_context) + pin_context(i915->preempt_context, engine, + &engine->preempt_context); ret = measure_breadcrumb_dw(engine); if (ret < 0) - goto err_unpin_preempt; + goto err_unpin; engine->emit_fini_breadcrumb_dw = ret; - return 0; + engine->set_default_submission(engine); -err_unpin_preempt: - if (i915->preempt_context) - __intel_context_unpin(i915->preempt_context, engine); + return 0; -err_unpin_kernel: - __intel_context_unpin(i915->kernel_context, engine); +err_unpin: + if (engine->preempt_context) + intel_context_unpin(engine->preempt_context); + intel_context_unpin(engine->kernel_context); return ret; } +void intel_gt_resume(struct drm_i915_private *i915) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + + /* + * After resume, we may need to poke into the pinned kernel + * contexts to paper over any damage caused by the sudden suspend. + * Only the kernel contexts should remain pinned over suspend, + * allowing us to fixup the user contexts on their first pin. + */ + for_each_engine(engine, i915, id) { + struct intel_context *ce; + + ce = engine->kernel_context; + if (ce) + ce->ops->reset(ce); + + ce = engine->preempt_context; + if (ce) + ce->ops->reset(ce); + } +} + /** * intel_engines_cleanup_common - cleans up the engine state created by * the common initiailizers. @@ -738,8 +786,6 @@ err_unpin_kernel: */ void intel_engine_cleanup_common(struct intel_engine_cs *engine) { - struct drm_i915_private *i915 = engine->i915; - cleanup_status_page(engine); intel_engine_fini_breadcrumbs(engine); @@ -749,9 +795,9 @@ void intel_engine_cleanup_common(struct intel_engine_cs *engine) if (engine->default_state) i915_gem_object_put(engine->default_state); - if (i915->preempt_context) - __intel_context_unpin(i915->preempt_context, engine); - __intel_context_unpin(i915->kernel_context, engine); + if (engine->preempt_context) + intel_context_unpin(engine->preempt_context); + intel_context_unpin(engine->kernel_context); i915_timeline_fini(&engine->timeline); @@ -762,50 +808,48 @@ void intel_engine_cleanup_common(struct intel_engine_cs *engine) u64 intel_engine_get_active_head(const struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = engine->i915; + struct drm_i915_private *i915 = engine->i915; + u64 acthd; - if (INTEL_GEN(dev_priv) >= 8) - acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base), - RING_ACTHD_UDW(engine->mmio_base)); - else if (INTEL_GEN(dev_priv) >= 4) - acthd = I915_READ(RING_ACTHD(engine->mmio_base)); + if (INTEL_GEN(i915) >= 8) + acthd = ENGINE_READ64(engine, RING_ACTHD, RING_ACTHD_UDW); + else if (INTEL_GEN(i915) >= 4) + acthd = ENGINE_READ(engine, RING_ACTHD); else - acthd = I915_READ(ACTHD); + acthd = ENGINE_READ(engine, ACTHD); return acthd; } u64 intel_engine_get_last_batch_head(const struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = engine->i915; u64 bbaddr; - if (INTEL_GEN(dev_priv) >= 8) - bbaddr = I915_READ64_2x32(RING_BBADDR(engine->mmio_base), - RING_BBADDR_UDW(engine->mmio_base)); + if (INTEL_GEN(engine->i915) >= 8) + bbaddr = ENGINE_READ64(engine, RING_BBADDR, RING_BBADDR_UDW); else - bbaddr = I915_READ(RING_BBADDR(engine->mmio_base)); + bbaddr = ENGINE_READ(engine, RING_BBADDR); return bbaddr; } int intel_engine_stop_cs(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = engine->i915; + struct intel_uncore *uncore = engine->uncore; const u32 base = engine->mmio_base; const i915_reg_t mode = RING_MI_MODE(base); int err; - if (INTEL_GEN(dev_priv) < 3) + if (INTEL_GEN(engine->i915) < 3) return -ENODEV; GEM_TRACE("%s\n", engine->name); - I915_WRITE_FW(mode, _MASKED_BIT_ENABLE(STOP_RING)); + intel_uncore_write_fw(uncore, mode, _MASKED_BIT_ENABLE(STOP_RING)); err = 0; - if (__intel_wait_for_register_fw(dev_priv, + if (__intel_wait_for_register_fw(uncore, mode, MODE_IDLE, MODE_IDLE, 1000, 0, NULL)) { @@ -814,19 +858,16 @@ int intel_engine_stop_cs(struct intel_engine_cs *engine) } /* A final mmio read to let GPU writes be hopefully flushed to memory */ - POSTING_READ_FW(mode); + intel_uncore_posting_read_fw(uncore, mode); return err; } void intel_engine_cancel_stop_cs(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = engine->i915; - GEM_TRACE("%s\n", engine->name); - I915_WRITE_FW(RING_MI_MODE(engine->mmio_base), - _MASKED_BIT_DISABLE(STOP_RING)); + ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING)); } const char *i915_cache_level_str(struct drm_i915_private *i915, int type) @@ -863,6 +904,7 @@ static inline u32 read_subslice_reg(struct drm_i915_private *dev_priv, int slice, int subslice, i915_reg_t reg) { + struct intel_uncore *uncore = &dev_priv->uncore; u32 mcr_slice_subslice_mask; u32 mcr_slice_subslice_select; u32 default_mcr_s_ss_select; @@ -884,33 +926,33 @@ read_subslice_reg(struct drm_i915_private *dev_priv, int slice, default_mcr_s_ss_select = intel_calculate_mcr_s_ss_select(dev_priv); - fw_domains = intel_uncore_forcewake_for_reg(dev_priv, reg, + fw_domains = intel_uncore_forcewake_for_reg(uncore, reg, FW_REG_READ); - fw_domains |= intel_uncore_forcewake_for_reg(dev_priv, + fw_domains |= intel_uncore_forcewake_for_reg(uncore, GEN8_MCR_SELECTOR, FW_REG_READ | FW_REG_WRITE); - spin_lock_irq(&dev_priv->uncore.lock); - intel_uncore_forcewake_get__locked(dev_priv, fw_domains); + spin_lock_irq(&uncore->lock); + intel_uncore_forcewake_get__locked(uncore, fw_domains); - mcr = I915_READ_FW(GEN8_MCR_SELECTOR); + mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR); WARN_ON_ONCE((mcr & mcr_slice_subslice_mask) != default_mcr_s_ss_select); mcr &= ~mcr_slice_subslice_mask; mcr |= mcr_slice_subslice_select; - I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr); + intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr); - ret = I915_READ_FW(reg); + ret = intel_uncore_read_fw(uncore, reg); mcr &= ~mcr_slice_subslice_mask; mcr |= default_mcr_s_ss_select; - I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr); + intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr); - intel_uncore_forcewake_put__locked(dev_priv, fw_domains); - spin_unlock_irq(&dev_priv->uncore.lock); + intel_uncore_forcewake_put__locked(uncore, fw_domains); + spin_unlock_irq(&uncore->lock); return ret; } @@ -920,6 +962,7 @@ void intel_engine_get_instdone(struct intel_engine_cs *engine, struct intel_instdone *instdone) { struct drm_i915_private *dev_priv = engine->i915; + struct intel_uncore *uncore = engine->uncore; u32 mmio_base = engine->mmio_base; int slice; int subslice; @@ -928,12 +971,14 @@ void intel_engine_get_instdone(struct intel_engine_cs *engine, switch (INTEL_GEN(dev_priv)) { default: - instdone->instdone = I915_READ(RING_INSTDONE(mmio_base)); + instdone->instdone = + intel_uncore_read(uncore, RING_INSTDONE(mmio_base)); - if (engine->id != RCS) + if (engine->id != RCS0) break; - instdone->slice_common = I915_READ(GEN7_SC_INSTDONE); + instdone->slice_common = + intel_uncore_read(uncore, GEN7_SC_INSTDONE); for_each_instdone_slice_subslice(dev_priv, slice, subslice) { instdone->sampler[slice][subslice] = read_subslice_reg(dev_priv, slice, subslice, @@ -944,28 +989,33 @@ void intel_engine_get_instdone(struct intel_engine_cs *engine, } break; case 7: - instdone->instdone = I915_READ(RING_INSTDONE(mmio_base)); + instdone->instdone = + intel_uncore_read(uncore, RING_INSTDONE(mmio_base)); - if (engine->id != RCS) + if (engine->id != RCS0) break; - instdone->slice_common = I915_READ(GEN7_SC_INSTDONE); - instdone->sampler[0][0] = I915_READ(GEN7_SAMPLER_INSTDONE); - instdone->row[0][0] = I915_READ(GEN7_ROW_INSTDONE); + instdone->slice_common = + intel_uncore_read(uncore, GEN7_SC_INSTDONE); + instdone->sampler[0][0] = + intel_uncore_read(uncore, GEN7_SAMPLER_INSTDONE); + instdone->row[0][0] = + intel_uncore_read(uncore, GEN7_ROW_INSTDONE); break; case 6: case 5: case 4: - instdone->instdone = I915_READ(RING_INSTDONE(mmio_base)); - - if (engine->id == RCS) + instdone->instdone = + intel_uncore_read(uncore, RING_INSTDONE(mmio_base)); + if (engine->id == RCS0) /* HACK: Using the wrong struct member */ - instdone->slice_common = I915_READ(GEN4_INSTDONE1); + instdone->slice_common = + intel_uncore_read(uncore, GEN4_INSTDONE1); break; case 3: case 2: - instdone->instdone = I915_READ(GEN2_INSTDONE); + instdone->instdone = intel_uncore_read(uncore, GEN2_INSTDONE); break; } } @@ -985,12 +1035,13 @@ static bool ring_is_idle(struct intel_engine_cs *engine) return true; /* First check that no commands are left in the ring */ - if ((I915_READ_HEAD(engine) & HEAD_ADDR) != - (I915_READ_TAIL(engine) & TAIL_ADDR)) + if ((ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR) != + (ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR)) idle = false; /* No bit for gen2, so assume the CS parser is idle */ - if (INTEL_GEN(dev_priv) > 2 && !(I915_READ_MODE(engine) & MODE_IDLE)) + if (INTEL_GEN(dev_priv) > 2 && + !(ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE)) idle = false; intel_runtime_pm_put(dev_priv, wakeref); @@ -1007,16 +1058,10 @@ static bool ring_is_idle(struct intel_engine_cs *engine) */ bool intel_engine_is_idle(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = engine->i915; - /* More white lies, if wedged, hw state is inconsistent */ - if (i915_terminally_wedged(&dev_priv->gpu_error)) + if (i915_reset_failed(engine->i915)) return true; - /* Any inflight/incomplete requests? */ - if (!intel_engine_signaled(engine, intel_engine_last_submit(engine))) - return false; - /* Waiting to drain ELSP? */ if (READ_ONCE(engine->execlists.active)) { struct tasklet_struct *t = &engine->execlists.tasklet; @@ -1045,7 +1090,7 @@ bool intel_engine_is_idle(struct intel_engine_cs *engine) return ring_is_idle(engine); } -bool intel_engines_are_idle(struct drm_i915_private *dev_priv) +bool intel_engines_are_idle(struct drm_i915_private *i915) { struct intel_engine_cs *engine; enum intel_engine_id id; @@ -1054,10 +1099,14 @@ bool intel_engines_are_idle(struct drm_i915_private *dev_priv) * If the driver is wedged, HW state may be very inconsistent and * report that it is still busy, even though we have stopped using it. */ - if (i915_terminally_wedged(&dev_priv->gpu_error)) + if (i915_reset_failed(i915)) return true; - for_each_engine(engine, dev_priv, id) { + /* Already parked (and passed an idleness test); must still be idle */ + if (!READ_ONCE(i915->gt.awake)) + return true; + + for_each_engine(engine, i915, id) { if (!intel_engine_is_idle(engine)) return false; } @@ -1065,34 +1114,6 @@ bool intel_engines_are_idle(struct drm_i915_private *dev_priv) return true; } -/** - * intel_engine_has_kernel_context: - * @engine: the engine - * - * Returns true if the last context to be executed on this engine, or has been - * executed if the engine is already idle, is the kernel context - * (#i915.kernel_context). - */ -bool intel_engine_has_kernel_context(const struct intel_engine_cs *engine) -{ - const struct intel_context *kernel_context = - to_intel_context(engine->i915->kernel_context, engine); - struct i915_request *rq; - - lockdep_assert_held(&engine->i915->drm.struct_mutex); - - /* - * Check the last context seen by the engine. If active, it will be - * the last request that remains in the timeline. When idle, it is - * the last executed context as tracked by retirement. - */ - rq = __i915_active_request_peek(&engine->timeline.last_request); - if (rq) - return rq->hw_context == kernel_context; - else - return engine->last_retired_context == kernel_context; -} - void intel_engines_reset_default_submission(struct drm_i915_private *i915) { struct intel_engine_cs *engine; @@ -1180,6 +1201,8 @@ void intel_engines_park(struct drm_i915_private *i915) i915_gem_batch_pool_fini(&engine->batch_pool); engine->execlists.no_priolist = false; } + + i915->gt.active_engines = 0; } /** @@ -1283,15 +1306,14 @@ static void print_request(struct drm_printer *m, x = print_sched_attr(rq->i915, &rq->sched.attr, buf, x, sizeof(buf)); - drm_printf(m, "%s%x%s%s [%llx:%llx]%s @ %dms: %s\n", + drm_printf(m, "%s %llx:%llx%s%s %s @ %dms: %s\n", prefix, - rq->global_seqno, + rq->fence.context, rq->fence.seqno, i915_request_completed(rq) ? "!" : i915_request_started(rq) ? "*" : "", test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &rq->fence.flags) ? "+" : "", - rq->fence.context, rq->fence.seqno, buf, jiffies_to_msecs(jiffies - rq->emitted_jiffies), name); @@ -1334,25 +1356,26 @@ static void intel_engine_print_registers(const struct intel_engine_cs *engine, &engine->execlists; u64 addr; - if (engine->id == RCS && IS_GEN_RANGE(dev_priv, 4, 7)) - drm_printf(m, "\tCCID: 0x%08x\n", I915_READ(CCID)); + if (engine->id == RCS0 && IS_GEN_RANGE(dev_priv, 4, 7)) + drm_printf(m, "\tCCID: 0x%08x\n", ENGINE_READ(engine, CCID)); drm_printf(m, "\tRING_START: 0x%08x\n", - I915_READ(RING_START(engine->mmio_base))); + ENGINE_READ(engine, RING_START)); drm_printf(m, "\tRING_HEAD: 0x%08x\n", - I915_READ(RING_HEAD(engine->mmio_base)) & HEAD_ADDR); + ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR); drm_printf(m, "\tRING_TAIL: 0x%08x\n", - I915_READ(RING_TAIL(engine->mmio_base)) & TAIL_ADDR); + ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR); drm_printf(m, "\tRING_CTL: 0x%08x%s\n", - I915_READ(RING_CTL(engine->mmio_base)), - I915_READ(RING_CTL(engine->mmio_base)) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? " [waiting]" : ""); + ENGINE_READ(engine, RING_CTL), + ENGINE_READ(engine, RING_CTL) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? " [waiting]" : ""); if (INTEL_GEN(engine->i915) > 2) { drm_printf(m, "\tRING_MODE: 0x%08x%s\n", - I915_READ(RING_MI_MODE(engine->mmio_base)), - I915_READ(RING_MI_MODE(engine->mmio_base)) & (MODE_IDLE) ? " [idle]" : ""); + ENGINE_READ(engine, RING_MI_MODE), + ENGINE_READ(engine, RING_MI_MODE) & (MODE_IDLE) ? " [idle]" : ""); } if (INTEL_GEN(dev_priv) >= 6) { - drm_printf(m, "\tRING_IMR: %08x\n", I915_READ_IMR(engine)); + drm_printf(m, "\tRING_IMR: %08x\n", + ENGINE_READ(engine, RING_IMR)); } addr = intel_engine_get_active_head(engine); @@ -1362,57 +1385,53 @@ static void intel_engine_print_registers(const struct intel_engine_cs *engine, drm_printf(m, "\tBBADDR: 0x%08x_%08x\n", upper_32_bits(addr), lower_32_bits(addr)); if (INTEL_GEN(dev_priv) >= 8) - addr = I915_READ64_2x32(RING_DMA_FADD(engine->mmio_base), - RING_DMA_FADD_UDW(engine->mmio_base)); + addr = ENGINE_READ64(engine, RING_DMA_FADD, RING_DMA_FADD_UDW); else if (INTEL_GEN(dev_priv) >= 4) - addr = I915_READ(RING_DMA_FADD(engine->mmio_base)); + addr = ENGINE_READ(engine, RING_DMA_FADD); else - addr = I915_READ(DMA_FADD_I8XX); + addr = ENGINE_READ(engine, DMA_FADD_I8XX); drm_printf(m, "\tDMA_FADDR: 0x%08x_%08x\n", upper_32_bits(addr), lower_32_bits(addr)); if (INTEL_GEN(dev_priv) >= 4) { drm_printf(m, "\tIPEIR: 0x%08x\n", - I915_READ(RING_IPEIR(engine->mmio_base))); + ENGINE_READ(engine, RING_IPEIR)); drm_printf(m, "\tIPEHR: 0x%08x\n", - I915_READ(RING_IPEHR(engine->mmio_base))); + ENGINE_READ(engine, RING_IPEHR)); } else { - drm_printf(m, "\tIPEIR: 0x%08x\n", I915_READ(IPEIR)); - drm_printf(m, "\tIPEHR: 0x%08x\n", I915_READ(IPEHR)); + drm_printf(m, "\tIPEIR: 0x%08x\n", ENGINE_READ(engine, IPEIR)); + drm_printf(m, "\tIPEHR: 0x%08x\n", ENGINE_READ(engine, IPEHR)); } if (HAS_EXECLISTS(dev_priv)) { const u32 *hws = &engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX]; + const u8 num_entries = execlists->csb_size; unsigned int idx; u8 read, write; - drm_printf(m, "\tExeclist status: 0x%08x %08x\n", - I915_READ(RING_EXECLIST_STATUS_LO(engine)), - I915_READ(RING_EXECLIST_STATUS_HI(engine))); + drm_printf(m, "\tExeclist status: 0x%08x %08x, entries %u\n", + ENGINE_READ(engine, RING_EXECLIST_STATUS_LO), + ENGINE_READ(engine, RING_EXECLIST_STATUS_HI), + num_entries); read = execlists->csb_head; write = READ_ONCE(*execlists->csb_write); - drm_printf(m, "\tExeclist CSB read %d, write %d [mmio:%d], tasklet queued? %s (%s)\n", + drm_printf(m, "\tExeclist CSB read %d, write %d, tasklet queued? %s (%s)\n", read, write, - GEN8_CSB_WRITE_PTR(I915_READ(RING_CONTEXT_STATUS_PTR(engine))), yesno(test_bit(TASKLET_STATE_SCHED, &engine->execlists.tasklet.state)), enableddisabled(!atomic_read(&engine->execlists.tasklet.count))); - if (read >= GEN8_CSB_ENTRIES) + if (read >= num_entries) read = 0; - if (write >= GEN8_CSB_ENTRIES) + if (write >= num_entries) write = 0; if (read > write) - write += GEN8_CSB_ENTRIES; + write += num_entries; while (read < write) { - idx = ++read % GEN8_CSB_ENTRIES; - drm_printf(m, "\tExeclist CSB[%d]: 0x%08x [mmio:0x%08x], context: %d [mmio:%d]\n", - idx, - hws[idx * 2], - I915_READ(RING_CONTEXT_STATUS_BUF_LO(engine, idx)), - hws[idx * 2 + 1], - I915_READ(RING_CONTEXT_STATUS_BUF_HI(engine, idx))); + idx = ++read % num_entries; + drm_printf(m, "\tExeclist CSB[%d]: 0x%08x, context: %d\n", + idx, hws[idx * 2], hws[idx * 2 + 1]); } rcu_read_lock(); @@ -1425,10 +1444,11 @@ static void intel_engine_print_registers(const struct intel_engine_cs *engine, char hdr[80]; snprintf(hdr, sizeof(hdr), - "\t\tELSP[%d] count=%d, ring:{start:%08x, hwsp:%08x}, rq: ", + "\t\tELSP[%d] count=%d, ring:{start:%08x, hwsp:%08x, seqno:%08x}, rq: ", idx, count, i915_ggtt_offset(rq->ring->vma), - rq->timeline->hwsp_offset); + rq->timeline->hwsp_offset, + hwsp_seqno(rq)); print_request(m, rq, hdr); } else { drm_printf(m, "\t\tELSP[%d] idle\n", idx); @@ -1438,11 +1458,11 @@ static void intel_engine_print_registers(const struct intel_engine_cs *engine, rcu_read_unlock(); } else if (INTEL_GEN(dev_priv) > 6) { drm_printf(m, "\tPP_DIR_BASE: 0x%08x\n", - I915_READ(RING_PP_DIR_BASE(engine))); + ENGINE_READ(engine, RING_PP_DIR_BASE)); drm_printf(m, "\tPP_DIR_BASE_READ: 0x%08x\n", - I915_READ(RING_PP_DIR_BASE_READ(engine))); + ENGINE_READ(engine, RING_PP_DIR_BASE_READ)); drm_printf(m, "\tPP_DIR_DCLV: 0x%08x\n", - I915_READ(RING_PP_DIR_DCLV(engine))); + ENGINE_READ(engine, RING_PP_DIR_DCLV)); } } @@ -1495,13 +1515,12 @@ void intel_engine_dump(struct intel_engine_cs *engine, va_end(ap); } - if (i915_terminally_wedged(&engine->i915->gpu_error)) + if (i915_reset_failed(engine->i915)) drm_printf(m, "*** WEDGED ***\n"); - drm_printf(m, "\tcurrent seqno %x, last %x, hangcheck %x [%d ms]\n", - intel_engine_get_seqno(engine), - intel_engine_last_submit(engine), - engine->hangcheck.seqno, + drm_printf(m, "\tHangcheck %x:%x [%d ms]\n", + engine->hangcheck.last_seqno, + engine->hangcheck.next_seqno, jiffies_to_msecs(jiffies - engine->hangcheck.action_timestamp)); drm_printf(m, "\tReset count: %d (global %d)\n", i915_reset_engine_count(error, engine), @@ -1521,7 +1540,7 @@ void intel_engine_dump(struct intel_engine_cs *engine, if (&rq->link != &engine->timeline.requests) print_request(m, rq, "\t\tlast "); - rq = i915_gem_find_active_request(engine); + rq = intel_engine_find_active_request(engine); if (rq) { print_request(m, rq, "\t\tactive "); @@ -1688,6 +1707,50 @@ void intel_disable_engine_stats(struct intel_engine_cs *engine) write_sequnlock_irqrestore(&engine->stats.lock, flags); } +static bool match_ring(struct i915_request *rq) +{ + u32 ring = ENGINE_READ(rq->engine, RING_START); + + return ring == i915_ggtt_offset(rq->ring->vma); +} + +struct i915_request * +intel_engine_find_active_request(struct intel_engine_cs *engine) +{ + struct i915_request *request, *active = NULL; + unsigned long flags; + + /* + * We are called by the error capture, reset and to dump engine + * state at random points in time. In particular, note that neither is + * crucially ordered with an interrupt. After a hang, the GPU is dead + * and we assume that no more writes can happen (we waited long enough + * for all writes that were in transaction to be flushed) - adding an + * extra delay for a recent interrupt is pointless. Hence, we do + * not need an engine->irq_seqno_barrier() before the seqno reads. + * At all other times, we must assume the GPU is still running, but + * we only care about the snapshot of this moment. + */ + spin_lock_irqsave(&engine->timeline.lock, flags); + list_for_each_entry(request, &engine->timeline.requests, link) { + if (i915_request_completed(request)) + continue; + + if (!i915_request_started(request)) + break; + + /* More than one preemptible request may match! */ + if (!match_ring(request)) + break; + + active = request; + break; + } + spin_unlock_irqrestore(&engine->timeline.lock, flags); + + return active; +} + #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) #include "selftests/mock_engine.c" #include "selftests/intel_engine_cs.c" |