diff options
Diffstat (limited to 'drivers/gpu/drm/i915')
| -rw-r--r-- | drivers/gpu/drm/i915/Kconfig | 1 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/Kconfig.debug | 1 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/gvt/cmd_parser.c | 16 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/gvt/gtt.c | 42 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/gvt/handlers.c | 64 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/gvt/mmio_context.c | 23 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/gvt/reg.h | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/gvt/scheduler.c | 48 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/gvt/scheduler.h | 1 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/i915_priolist_types.h | 5 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/i915_reg.h | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/i915_request.c | 42 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/i915_scheduler.c | 255 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/i915_scheduler_types.h | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/intel_lrc.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/intel_workarounds.c | 6 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/selftests/intel_lrc.c | 12 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/selftests/lib_sw_fence.h | 4 |
18 files changed, 313 insertions, 217 deletions
diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig index 3d5f1cb6a76c..255f224db64b 100644 --- a/drivers/gpu/drm/i915/Kconfig +++ b/drivers/gpu/drm/i915/Kconfig @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only config DRM_I915 tristate "Intel 8xx/9xx/G3x/G4x/HD Graphics" depends on DRM diff --git a/drivers/gpu/drm/i915/Kconfig.debug b/drivers/gpu/drm/i915/Kconfig.debug index ad4d71161dda..04b686d2c2d0 100644 --- a/drivers/gpu/drm/i915/Kconfig.debug +++ b/drivers/gpu/drm/i915/Kconfig.debug @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only config DRM_I915_WERROR bool "Force GCC to throw an error instead of a warning when compiling" # As this may inadvertently break the build, only allow the user diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c index ab002cfd3cab..de5347725564 100644 --- a/drivers/gpu/drm/i915/gvt/cmd_parser.c +++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c @@ -896,12 +896,16 @@ static int cmd_reg_handler(struct parser_exec_state *s, } /* TODO - * Right now only scan LRI command on KBL and in inhibit context. - * It's good enough to support initializing mmio by lri command in - * vgpu inhibit context on KBL. + * In order to let workload with inhibit context to generate + * correct image data into memory, vregs values will be loaded to + * hw via LRIs in the workload with inhibit context. But as + * indirect context is loaded prior to LRIs in workload, we don't + * want reg values specified in indirect context overwritten by + * LRIs in workloads. So, when scanning an indirect context, we + * update reg values in it into vregs, so LRIs in workload with + * inhibit context will restore with correct values */ - if ((IS_KABYLAKE(s->vgpu->gvt->dev_priv) - || IS_COFFEELAKE(s->vgpu->gvt->dev_priv)) && + if (IS_GEN(gvt->dev_priv, 9) && intel_gvt_mmio_is_in_ctx(gvt, offset) && !strncmp(cmd, "lri", 3)) { intel_gvt_hypervisor_read_gpa(s->vgpu, @@ -2526,7 +2530,7 @@ static const struct cmd_info cmd_info[] = { 0, 12, NULL}, {"VEB_DI_IECP", OP_VEB_DNDI_IECP_STATE, F_LEN_VAR, R_VECS, D_BDW_PLUS, - 0, 20, NULL}, + 0, 12, NULL}, }; static void add_cmd_entry(struct intel_gvt *gvt, struct cmd_entry *e) diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c index 08c74e65836b..53115bdae12b 100644 --- a/drivers/gpu/drm/i915/gvt/gtt.c +++ b/drivers/gpu/drm/i915/gvt/gtt.c @@ -53,13 +53,19 @@ static int preallocated_oos_pages = 8192; */ bool intel_gvt_ggtt_validate_range(struct intel_vgpu *vgpu, u64 addr, u32 size) { - if ((!vgpu_gmadr_is_valid(vgpu, addr)) || (size - && !vgpu_gmadr_is_valid(vgpu, addr + size - 1))) { - gvt_vgpu_err("invalid range gmadr 0x%llx size 0x%x\n", - addr, size); - return false; - } - return true; + if (size == 0) + return vgpu_gmadr_is_valid(vgpu, addr); + + if (vgpu_gmadr_is_aperture(vgpu, addr) && + vgpu_gmadr_is_aperture(vgpu, addr + size - 1)) + return true; + else if (vgpu_gmadr_is_hidden(vgpu, addr) && + vgpu_gmadr_is_hidden(vgpu, addr + size - 1)) + return true; + + gvt_dbg_mm("Invalid ggtt range at 0x%llx, size: 0x%x\n", + addr, size); + return false; } /* translate a guest gmadr to host gmadr */ @@ -942,7 +948,16 @@ static int ppgtt_invalidate_spt_by_shadow_entry(struct intel_vgpu *vgpu, if (e->type != GTT_TYPE_PPGTT_ROOT_L3_ENTRY && e->type != GTT_TYPE_PPGTT_ROOT_L4_ENTRY) { - cur_pt_type = get_next_pt_type(e->type) + 1; + cur_pt_type = get_next_pt_type(e->type); + + if (!gtt_type_is_pt(cur_pt_type) || + !gtt_type_is_pt(cur_pt_type + 1)) { + WARN(1, "Invalid page table type, cur_pt_type is: %d\n", cur_pt_type); + return -EINVAL; + } + + cur_pt_type += 1; + if (ops->get_pfn(e) == vgpu->gtt.scratch_pt[cur_pt_type].page_mfn) return 0; @@ -1076,8 +1091,10 @@ static struct intel_vgpu_ppgtt_spt *ppgtt_populate_spt_by_guest_entry( } else { int type = get_next_pt_type(we->type); - if (!gtt_type_is_pt(type)) + if (!gtt_type_is_pt(type)) { + ret = -EINVAL; goto err; + } spt = ppgtt_alloc_spt_gfn(vgpu, type, ops->get_pfn(we), ips); if (IS_ERR(spt)) { @@ -1100,6 +1117,7 @@ static struct intel_vgpu_ppgtt_spt *ppgtt_populate_spt_by_guest_entry( err_free_spt: ppgtt_free_spt(spt); + spt = NULL; err: gvt_vgpu_err("fail: shadow page %p guest entry 0x%llx type %d\n", spt, we->val64, we->type); @@ -2181,7 +2199,8 @@ static int emulate_ggtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off, struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops; unsigned long g_gtt_index = off >> info->gtt_entry_size_shift; unsigned long gma, gfn; - struct intel_gvt_gtt_entry e, m; + struct intel_gvt_gtt_entry e = {.val64 = 0, .type = GTT_TYPE_GGTT_PTE}; + struct intel_gvt_gtt_entry m = {.val64 = 0, .type = GTT_TYPE_GGTT_PTE}; dma_addr_t dma_addr; int ret; struct intel_gvt_partial_pte *partial_pte, *pos, *n; @@ -2248,7 +2267,8 @@ static int emulate_ggtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off, if (!partial_update && (ops->test_present(&e))) { gfn = ops->get_pfn(&e); - m = e; + m.val64 = e.val64; + m.type = e.type; /* one PTE update may be issued in multiple writes and the * first write may not construct a valid gfn diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c index 90673fca792f..a6ade66349bd 100644 --- a/drivers/gpu/drm/i915/gvt/handlers.c +++ b/drivers/gpu/drm/i915/gvt/handlers.c @@ -464,6 +464,8 @@ static i915_reg_t force_nonpriv_white_list[] = { _MMIO(0x2690), _MMIO(0x2694), _MMIO(0x2698), + _MMIO(0x2754), + _MMIO(0x28a0), _MMIO(0x4de0), _MMIO(0x4de4), _MMIO(0x4dfc), @@ -1364,7 +1366,6 @@ static int dma_ctrl_write(struct intel_vgpu *vgpu, unsigned int offset, static int gen9_trtte_write(struct intel_vgpu *vgpu, unsigned int offset, void *p_data, unsigned int bytes) { - struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; u32 trtte = *(u32 *)p_data; if ((trtte & 1) && (trtte & (1 << 1)) == 0) { @@ -1373,11 +1374,6 @@ static int gen9_trtte_write(struct intel_vgpu *vgpu, unsigned int offset, return -EINVAL; } write_vreg(vgpu, offset, p_data, bytes); - /* TRTTE is not per-context */ - - mmio_hw_access_pre(dev_priv); - I915_WRITE(_MMIO(offset), vgpu_vreg(vgpu, offset)); - mmio_hw_access_post(dev_priv); return 0; } @@ -1385,15 +1381,6 @@ static int gen9_trtte_write(struct intel_vgpu *vgpu, unsigned int offset, static int gen9_trtt_chicken_write(struct intel_vgpu *vgpu, unsigned int offset, void *p_data, unsigned int bytes) { - struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; - u32 val = *(u32 *)p_data; - - if (val & 1) { - /* unblock hw logic */ - mmio_hw_access_pre(dev_priv); - I915_WRITE(_MMIO(offset), val); - mmio_hw_access_post(dev_priv); - } write_vreg(vgpu, offset, p_data, bytes); return 0; } @@ -1705,8 +1692,22 @@ static int ring_mode_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, bool enable_execlist; int ret; + (*(u32 *)p_data) &= ~_MASKED_BIT_ENABLE(1); + if (IS_COFFEELAKE(vgpu->gvt->dev_priv)) + (*(u32 *)p_data) &= ~_MASKED_BIT_ENABLE(2); write_vreg(vgpu, offset, p_data, bytes); + if (data & _MASKED_BIT_ENABLE(1)) { + enter_failsafe_mode(vgpu, GVT_FAILSAFE_UNSUPPORTED_GUEST); + return 0; + } + + if (IS_COFFEELAKE(vgpu->gvt->dev_priv) && + data & _MASKED_BIT_ENABLE(2)) { + enter_failsafe_mode(vgpu, GVT_FAILSAFE_UNSUPPORTED_GUEST); + return 0; + } + /* when PPGTT mode enabled, we will check if guest has called * pvinfo, if not, we will treat this guest as non-gvtg-aware * guest, and stop emulating its cfg space, mmio, gtt, etc. @@ -1788,6 +1789,21 @@ static int ring_reset_ctl_write(struct intel_vgpu *vgpu, return 0; } +static int csfe_chicken1_mmio_write(struct intel_vgpu *vgpu, + unsigned int offset, void *p_data, + unsigned int bytes) +{ + u32 data = *(u32 *)p_data; + + (*(u32 *)p_data) &= ~_MASKED_BIT_ENABLE(0x18); + write_vreg(vgpu, offset, p_data, bytes); + + if (data & _MASKED_BIT_ENABLE(0x10) || data & _MASKED_BIT_ENABLE(0x8)) + enter_failsafe_mode(vgpu, GVT_FAILSAFE_UNSUPPORTED_GUEST); + + return 0; +} + #define MMIO_F(reg, s, f, am, rm, d, r, w) do { \ ret = new_mmio_info(gvt, i915_mmio_reg_offset(reg), \ f, s, am, rm, d, r, w); \ @@ -1908,7 +1924,8 @@ static int init_generic_mmio_info(struct intel_gvt *gvt) MMIO_DFH(_MMIO(0x20dc), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); MMIO_DFH(_3D_CHICKEN3, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); MMIO_DFH(_MMIO(0x2088), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(_MMIO(0x20e4), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(FF_SLICE_CS_CHICKEN2, D_ALL, + F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); MMIO_DFH(_MMIO(0x2470), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); MMIO_DFH(GAM_ECOCHK, D_ALL, F_CMD_ACCESS, NULL, NULL); MMIO_DFH(GEN7_COMMON_SLICE_CHICKEN1, D_ALL, F_MODE_MASK | F_CMD_ACCESS, @@ -3012,7 +3029,7 @@ static int init_skl_mmio_info(struct intel_gvt *gvt) MMIO_D(CSR_HTP_SKL, D_SKL_PLUS); MMIO_D(CSR_LAST_WRITE, D_SKL_PLUS); - MMIO_D(BDW_SCRATCH1, D_SKL_PLUS); + MMIO_DFH(BDW_SCRATCH1, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); MMIO_D(SKL_DFSM, D_SKL_PLUS); MMIO_D(DISPIO_CR_TX_BMU_CR0, D_SKL_PLUS); @@ -3025,8 +3042,8 @@ static int init_skl_mmio_info(struct intel_gvt *gvt) MMIO_D(RPM_CONFIG0, D_SKL_PLUS); MMIO_D(_MMIO(0xd08), D_SKL_PLUS); MMIO_D(RC6_LOCATION, D_SKL_PLUS); - MMIO_DFH(GEN7_FF_SLICE_CS_CHICKEN1, D_SKL_PLUS, F_MODE_MASK, - NULL, NULL); + MMIO_DFH(GEN7_FF_SLICE_CS_CHICKEN1, D_SKL_PLUS, + F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); MMIO_DFH(GEN9_CS_DEBUG_MODE1, D_SKL_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); @@ -3045,7 +3062,7 @@ static int init_skl_mmio_info(struct intel_gvt *gvt) MMIO_D(_MMIO(0x46520), D_SKL_PLUS); MMIO_D(_MMIO(0xc403c), D_SKL_PLUS); - MMIO_D(_MMIO(0xb004), D_SKL_PLUS); + MMIO_DFH(GEN8_GARBCNTL, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); MMIO_DH(DMA_CTRL, D_SKL_PLUS, NULL, dma_ctrl_write); MMIO_D(_MMIO(0x65900), D_SKL_PLUS); @@ -3074,7 +3091,10 @@ static int init_skl_mmio_info(struct intel_gvt *gvt) MMIO_D(_MMIO(_PLANE_KEYMSK_1(PIPE_C)), D_SKL_PLUS); MMIO_D(_MMIO(0x44500), D_SKL_PLUS); - MMIO_DFH(GEN9_CSFE_CHICKEN1_RCS, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); +#define CSFE_CHICKEN1_REG(base) _MMIO((base) + 0xD4) + MMIO_RING_DFH(CSFE_CHICKEN1_REG, D_SKL_PLUS, F_MODE_MASK | F_CMD_ACCESS, + NULL, csfe_chicken1_mmio_write); +#undef CSFE_CHICKEN1_REG MMIO_DFH(GEN8_HDC_CHICKEN1, D_SKL_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); MMIO_DFH(GEN9_WM_CHICKEN3, D_SKL_PLUS, F_MODE_MASK | F_CMD_ACCESS, @@ -3254,7 +3274,7 @@ static int init_bxt_mmio_info(struct intel_gvt *gvt) MMIO_D(GEN8_PUSHBUS_ENABLE, D_BXT); MMIO_D(GEN8_PUSHBUS_SHIFT, D_BXT); MMIO_D(GEN6_GFXPAUSE, D_BXT); - MMIO_D(GEN8_L3SQCREG1, D_BXT); + MMIO_DFH(GEN8_L3SQCREG1, D_BXT, F_CMD_ACCESS, NULL, NULL); MMIO_DFH(GEN9_CTX_PREEMPT_REG, D_BXT, F_CMD_ACCESS, NULL, NULL); diff --git a/drivers/gpu/drm/i915/gvt/mmio_context.c b/drivers/gpu/drm/i915/gvt/mmio_context.c index edf6d646eb25..90bb3df0db50 100644 --- a/drivers/gpu/drm/i915/gvt/mmio_context.c +++ b/drivers/gpu/drm/i915/gvt/mmio_context.c @@ -108,12 +108,13 @@ static struct engine_mmio gen9_engine_mmio_list[] __cacheline_aligned = { {RCS0, GEN9_HALF_SLICE_CHICKEN5, 0xffff, true}, /* 0xe188 */ {RCS0, GEN9_HALF_SLICE_CHICKEN7, 0xffff, true}, /* 0xe194 */ {RCS0, GEN8_ROW_CHICKEN, 0xffff, true}, /* 0xe4f0 */ - {RCS0, TRVATTL3PTRDW(0), 0, false}, /* 0x4de0 */ - {RCS0, TRVATTL3PTRDW(1), 0, false}, /* 0x4de4 */ - {RCS0, TRNULLDETCT, 0, false}, /* 0x4de8 */ - {RCS0, TRINVTILEDETCT, 0, false}, /* 0x4dec */ - {RCS0, TRVADR, 0, false}, /* 0x4df0 */ - {RCS0, TRTTE, 0, false}, /* 0x4df4 */ + {RCS0, TRVATTL3PTRDW(0), 0, true}, /* 0x4de0 */ + {RCS0, TRVATTL3PTRDW(1), 0, true}, /* 0x4de4 */ + {RCS0, TRNULLDETCT, 0, true}, /* 0x4de8 */ + {RCS0, TRINVTILEDETCT, 0, true}, /* 0x4dec */ + {RCS0, TRVADR, 0, true}, /* 0x4df0 */ + {RCS0, TRTTE, 0, true}, /* 0x4df4 */ + {RCS0, _MMIO(0x4dfc), 0, true}, {BCS0, RING_GFX_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2229c */ {BCS0, RING_MI_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2209c */ @@ -392,10 +393,7 @@ static void switch_mocs(struct intel_vgpu *pre, struct intel_vgpu *next, if (WARN_ON(ring_id >= ARRAY_SIZE(regs))) return; - if (ring_id == RCS0 && - (IS_KABYLAKE(dev_priv) || - IS_BROXTON(dev_priv) || - IS_COFFEELAKE(dev_priv))) + if (ring_id == RCS0 && IS_GEN(dev_priv, 9)) return; if (!pre && !gen9_render_mocs.initialized) @@ -470,11 +468,10 @@ static void switch_mmio(struct intel_vgpu *pre, continue; /* * No need to do save or restore of the mmio which is in context - * state image on kabylake, it's initialized by lri command and + * state image on gen9, it's initialized by lri command and * save or restore with context together. */ - if ((IS_KABYLAKE(dev_priv) || IS_BROXTON(dev_priv) - || IS_COFFEELAKE(dev_priv)) && mmio->in_context) + if (IS_GEN(dev_priv, 9) && mmio->in_context) continue; // save diff --git a/drivers/gpu/drm/i915/gvt/reg.h b/drivers/gpu/drm/i915/gvt/reg.h index 33aaa14bfdde..5b66e14c5b7b 100644 --- a/drivers/gpu/drm/i915/gvt/reg.h +++ b/drivers/gpu/drm/i915/gvt/reg.h @@ -102,6 +102,8 @@ #define FORCEWAKE_ACK_MEDIA_GEN9_REG 0x0D88 #define FORCEWAKE_ACK_HSW_REG 0x130044 +#define RB_HEAD_WRAP_CNT_MAX ((1 << 11) - 1) +#define RB_HEAD_WRAP_CNT_OFF 21 #define RB_HEAD_OFF_MASK ((1U << 21) - (1U << 2)) #define RB_TAIL_OFF_MASK ((1U << 21) - (1U << 3)) #define RB_TAIL_SIZE_MASK ((1U << 21) - (1U << 12)) diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c index 7c99bbc3e2b8..0f919f0a43d4 100644 --- a/drivers/gpu/drm/i915/gvt/scheduler.c +++ b/drivers/gpu/drm/i915/gvt/scheduler.c @@ -298,12 +298,29 @@ static int copy_workload_to_ring_buffer(struct intel_vgpu_workload *workload) struct i915_request *req = workload->req; void *shadow_ring_buffer_va; u32 *cs; + int err; - if ((IS_KABYLAKE(req->i915) || IS_BROXTON(req->i915) - || IS_COFFEELAKE(req->i915)) - && is_inhibit_context(req->hw_context)) + if (IS_GEN(req->i915, 9) && is_inhibit_context(req->hw_context)) intel_vgpu_restore_inhibit_context(vgpu, req); + /* + * To track whether a request has started on HW, we can emit a + * breadcrumb at the beginning of the request and check its + * timeline's HWSP to see if the breadcrumb has advanced past the + * start of this request. Actually, the request must have the + * init_breadcrumb if its timeline set has_init_bread_crumb, or the + * scheduler might get a wrong state of it during reset. Since the + * requests from gvt always set the has_init_breadcrumb flag, here + * need to do the emit_init_breadcrumb for all the requests. + */ + if (req->engine->emit_init_breadcrumb) { + err = req->engine->emit_init_breadcrumb(req); + if (err) { + gvt_vgpu_err("fail to emit init breadcrumb\n"); + return err; + } + } + /* allocate shadow ring buffer */ cs = intel_ring_begin(workload->req, workload->rb_len / sizeof(u32)); if (IS_ERR(cs)) { @@ -795,10 +812,31 @@ static void update_guest_context(struct intel_vgpu_workload *workload) void *src; unsigned long context_gpa, context_page_num; int i; + struct drm_i915_private *dev_priv = gvt->dev_priv; + u32 ring_base; + u32 head, tail; + u16 wrap_count; gvt_dbg_sched("ring id %d workload lrca %x\n", rq->engine->id, workload->ctx_desc.lrca); + head = workload->rb_head; + tail = workload->rb_tail; + wrap_count = workload->guest_rb_head >> RB_HEAD_WRAP_CNT_OFF; + + if (tail < head) { + if (wrap_count == RB_HEAD_WRAP_CNT_MAX) + wrap_count = 0; + else + wrap_count += 1; + } + + head = (wrap_count << RB_HEAD_WRAP_CNT_OFF) | tail; + + ring_base = dev_priv->engine[workload->ring_id]->mmio_base; + vgpu_vreg_t(vgpu, RING_TAIL(ring_base)) = tail; + vgpu_vreg_t(vgpu, RING_HEAD(ring_base)) = head; + context_page_num = rq->engine->context_size; context_page_num = context_page_num >> PAGE_SHIFT; @@ -1398,6 +1436,7 @@ intel_vgpu_create_workload(struct intel_vgpu *vgpu, int ring_id, struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; u64 ring_context_gpa; u32 head, tail, start, ctl, ctx_ctl, per_ctx, indirect_ctx; + u32 guest_head; int ret; ring_context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, @@ -1413,6 +1452,8 @@ intel_vgpu_create_workload(struct intel_vgpu *vgpu, int ring_id, intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + RING_CTX_OFF(ring_tail.val), &tail, 4); + guest_head = head; + head &= RB_HEAD_OFF_MASK; tail &= RB_TAIL_OFF_MASK; @@ -1445,6 +1486,7 @@ intel_vgpu_create_workload(struct intel_vgpu *vgpu, int ring_id, workload->ctx_desc = *desc; workload->ring_context_gpa = ring_context_gpa; workload->rb_head = head; + workload->guest_rb_head = guest_head; workload->rb_tail = tail; workload->rb_start = start; workload->rb_ctl = ctl; diff --git a/drivers/gpu/drm/i915/gvt/scheduler.h b/drivers/gpu/drm/i915/gvt/scheduler.h index 90c6756f5453..c50d14a9ce85 100644 --- a/drivers/gpu/drm/i915/gvt/scheduler.h +++ b/drivers/gpu/drm/i915/gvt/scheduler.h @@ -100,6 +100,7 @@ struct intel_vgpu_workload { struct execlist_ctx_descriptor_format ctx_desc; struct execlist_ring_context *ring_context; unsigned long rb_head, rb_tail, rb_ctl, rb_start, rb_len; + unsigned long guest_rb_head; bool restore_inhibit; struct intel_vgpu_elsp_dwords elsp_dwords; bool emulate_schedule_in; diff --git a/drivers/gpu/drm/i915/i915_priolist_types.h b/drivers/gpu/drm/i915/i915_priolist_types.h index cc44ebd3b553..49709de69875 100644 --- a/drivers/gpu/drm/i915/i915_priolist_types.h +++ b/drivers/gpu/drm/i915/i915_priolist_types.h @@ -20,15 +20,14 @@ enum { I915_PRIORITY_INVALID = INT_MIN }; -#define I915_USER_PRIORITY_SHIFT 3 +#define I915_USER_PRIORITY_SHIFT 2 #define I915_USER_PRIORITY(x) ((x) << I915_USER_PRIORITY_SHIFT) #define I915_PRIORITY_COUNT BIT(I915_USER_PRIORITY_SHIFT) #define I915_PRIORITY_MASK (I915_PRIORITY_COUNT - 1) #define I915_PRIORITY_WAIT ((u8)BIT(0)) -#define I915_PRIORITY_NEWCLIENT ((u8)BIT(1)) -#define I915_PRIORITY_NOSEMAPHORE ((u8)BIT(2)) +#define I915_PRIORITY_NOSEMAPHORE ((u8)BIT(1)) #define __NO_PREEMPTION (I915_PRIORITY_WAIT) diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index 249d35c12a75..2aa69d347ec4 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -7620,6 +7620,9 @@ enum { #define GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION (1 << 8) #define GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE (1 << 0) +#define GEN8_L3CNTLREG _MMIO(0x7034) + #define GEN8_ERRDETBCTRL (1 << 9) + #define GEN11_COMMON_SLICE_CHICKEN3 _MMIO(0x7304) #define GEN11_BLEND_EMB_FIX_DISABLE_IN_RCC (1 << 11) diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c index f6c78c0fa74b..c88e538b2ef4 100644 --- a/drivers/gpu/drm/i915/i915_request.c +++ b/drivers/gpu/drm/i915/i915_request.c @@ -502,15 +502,6 @@ void __i915_request_unsubmit(struct i915_request *request) /* We may be recursing from the signal callback of another i915 fence */ spin_lock_nested(&request->lock, SINGLE_DEPTH_NESTING); - /* - * As we do not allow WAIT to preempt inflight requests, - * once we have executed a request, along with triggering - * any execution callbacks, we must preserve its ordering - * within the non-preemptible FIFO. - */ - BUILD_BUG_ON(__NO_PREEMPTION & ~I915_PRIORITY_MASK); /* only internal */ - request->sched.attr.priority |= __NO_PREEMPTION; - if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags)) i915_request_cancel_breadcrumb(request); @@ -582,18 +573,7 @@ semaphore_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) switch (state) { case FENCE_COMPLETE: - /* - * We only check a small portion of our dependencies - * and so cannot guarantee that there remains no - * semaphore chain across all. Instead of opting - * for the full NOSEMAPHORE boost, we go for the - * smaller (but still preempting) boost of - * NEWCLIENT. This will be enough to boost over - * a busywaiting request (as that cannot be - * NEWCLIENT) without accidentally boosting - * a busywait over real work elsewhere. - */ - i915_schedule_bump_priority(request, I915_PRIORITY_NEWCLIENT); + i915_schedule_bump_priority(request, I915_PRIORITY_NOSEMAPHORE); break; case FENCE_FREE: @@ -874,12 +854,6 @@ emit_semaphore_wait(struct i915_request *to, if (err < 0) return err; - err = i915_sw_fence_await_dma_fence(&to->semaphore, - &from->fence, 0, - I915_FENCE_GFP); - if (err < 0) - return err; - /* We need to pin the signaler's HWSP until we are finished reading. */ err = i915_timeline_read_hwsp(from, to, &hwsp_offset); if (err) @@ -945,8 +919,18 @@ i915_request_await_request(struct i915_request *to, struct i915_request *from) &from->fence, 0, I915_FENCE_GFP); } + if (ret < 0) + return ret; - return ret < 0 ? ret : 0; + if (to->sched.flags & I915_SCHED_HAS_SEMAPHORE_CHAIN) { + ret = i915_sw_fence_await_dma_fence(&to->semaphore, + &from->fence, 0, + I915_FENCE_GFP); + if (ret < 0) + return ret; + } + + return 0; } int @@ -1237,7 +1221,7 @@ void i915_request_add(struct i915_request *request) * the bulk clients. (FQ_CODEL) */ if (list_empty(&request->sched.signalers_list)) - attr.priority |= I915_PRIORITY_NEWCLIENT; + attr.priority |= I915_PRIORITY_WAIT; engine->schedule(request, &attr); } diff --git a/drivers/gpu/drm/i915/i915_scheduler.c b/drivers/gpu/drm/i915/i915_scheduler.c index 39bc4f54e272..108f52e1bf35 100644 --- a/drivers/gpu/drm/i915/i915_scheduler.c +++ b/drivers/gpu/drm/i915/i915_scheduler.c @@ -35,109 +35,6 @@ static inline bool node_signaled(const struct i915_sched_node *node) return i915_request_completed(node_to_request(node)); } -void i915_sched_node_init(struct i915_sched_node *node) -{ - INIT_LIST_HEAD(&node->signalers_list); - INIT_LIST_HEAD(&node->waiters_list); - INIT_LIST_HEAD(&node->link); - node->attr.priority = I915_PRIORITY_INVALID; - node->semaphores = 0; - node->flags = 0; -} - -static struct i915_dependency * -i915_dependency_alloc(void) -{ - return kmem_cache_alloc(global.slab_dependencies, GFP_KERNEL); -} - -static void -i915_dependency_free(struct i915_dependency *dep) -{ - kmem_cache_free(global.slab_dependencies, dep); -} - -bool __i915_sched_node_add_dependency(struct i915_sched_node *node, - struct i915_sched_node *signal, - struct i915_dependency *dep, - unsigned long flags) -{ - bool ret = false; - - spin_lock_irq(&schedule_lock); - - if (!node_signaled(signal)) { - INIT_LIST_HEAD(&dep->dfs_link); - list_add(&dep->wait_link, &signal->waiters_list); - list_add(&dep->signal_link, &node->signalers_list); - dep->signaler = signal; - dep->flags = flags; - - /* Keep track of whether anyone on this chain has a semaphore */ - if (signal->flags & I915_SCHED_HAS_SEMAPHORE_CHAIN && - !node_started(signal)) - node->flags |= I915_SCHED_HAS_SEMAPHORE_CHAIN; - - ret = true; - } - - spin_unlock_irq(&schedule_lock); - - return ret; -} - -int i915_sched_node_add_dependency(struct i915_sched_node *node, - struct i915_sched_node *signal) -{ - struct i915_dependency *dep; - - dep = i915_dependency_alloc(); - if (!dep) - return -ENOMEM; - - if (!__i915_sched_node_add_dependency(node, signal, dep, - I915_DEPENDENCY_ALLOC)) - i915_dependency_free(dep); - - return 0; -} - -void i915_sched_node_fini(struct i915_sched_node *node) -{ - struct i915_dependency *dep, *tmp; - - GEM_BUG_ON(!list_empty(&node->link)); - - spin_lock_irq(&schedule_lock); - - /* - * Everyone we depended upon (the fences we wait to be signaled) - * should retire before us and remove themselves from our list. - * However, retirement is run independently on each timeline and - * so we may be called out-of-order. - */ - list_for_each_entry_safe(dep, tmp, &node->signalers_list, signal_link) { - GEM_BUG_ON(!node_signaled(dep->signaler)); - GEM_BUG_ON(!list_empty(&dep->dfs_link)); - - list_del(&dep->wait_link); - if (dep->flags & I915_DEPENDENCY_ALLOC) - i915_dependency_free(dep); - } - - /* Remove ourselves from everyone who depends upon us */ - list_for_each_entry_safe(dep, tmp, &node->waiters_list, wait_link) { - GEM_BUG_ON(dep->signaler != node); - GEM_BUG_ON(!list_empty(&dep->dfs_link)); - - list_del(&dep->signal_link); - if (dep->flags & I915_DEPENDENCY_ALLOC) - i915_dependency_free(dep); - } - - spin_unlock_irq(&schedule_lock); -} - static inline struct i915_priolist *to_priolist(struct rb_node *rb) { return rb_entry(rb, struct i915_priolist, node); @@ -239,6 +136,11 @@ out: return &p->requests[idx]; } +void __i915_priolist_free(struct i915_priolist *p) +{ + kmem_cache_free(global.slab_priorities, p); +} + struct sched_cache { struct list_head *priolist; }; @@ -273,7 +175,7 @@ static bool inflight(const struct i915_request *rq, return active->hw_context == rq->hw_context; } -static void __i915_schedule(struct i915_request *rq, +static void __i915_schedule(struct i915_sched_node *node, const struct i915_sched_attr *attr) { struct intel_engine_cs *engine; @@ -287,13 +189,13 @@ static void __i915_schedule(struct i915_request *rq, lockdep_assert_held(&schedule_lock); GEM_BUG_ON(prio == I915_PRIORITY_INVALID); - if (i915_request_completed(rq)) + if (node_signaled(node)) return; - if (prio <= READ_ONCE(rq->sched.attr.priority)) + if (prio <= READ_ONCE(node->attr.priority)) return; - stack.signaler = &rq->sched; + stack.signaler = node; list_add(&stack.dfs_link, &dfs); /* @@ -344,9 +246,9 @@ static void __i915_schedule(struct i915_request *rq, * execlists_submit_request()), we can set our own priority and skip * acquiring the engine locks. */ - if (rq->sched.attr.priority == I915_PRIORITY_INVALID) { - GEM_BUG_ON(!list_empty(&rq->sched.link)); - rq->sched.attr = *attr; + if (node->attr.priority == I915_PRIORITY_INVALID) { + GEM_BUG_ON(!list_empty(&node->link)); + node->attr = *attr; if (stack.dfs_link.next == stack.dfs_link.prev) return; @@ -355,15 +257,14 @@ static void __i915_schedule(struct i915_request *rq, } memset(&cache, 0, sizeof(cache)); - engine = rq->engine; + engine = node_to_request(node)->engine; spin_lock(&engine->timeline.lock); /* Fifo and depth-first replacement ensure our deps execute before us */ list_for_each_entry_safe_reverse(dep, p, &dfs, dfs_link) { - struct i915_sched_node *node = dep->signaler; - INIT_LIST_HEAD(&dep->dfs_link); + node = dep->signaler; engine = sched_lock_engine(node, engine, &cache); lockdep_assert_held(&engine->timeline.lock); @@ -413,13 +314,20 @@ static void __i915_schedule(struct i915_request *rq, void i915_schedule(struct i915_request *rq, const struct i915_sched_attr *attr) { spin_lock_irq(&schedule_lock); - __i915_schedule(rq, attr); + __i915_schedule(&rq->sched, attr); spin_unlock_irq(&schedule_lock); } +static void __bump_priority(struct i915_sched_node *node, unsigned int bump) +{ + struct i915_sched_attr attr = node->attr; + + attr.priority |= bump; + __i915_schedule(node, &attr); +} + void i915_schedule_bump_priority(struct i915_request *rq, unsigned int bump) { - struct i915_sched_attr attr; unsigned long flags; GEM_BUG_ON(bump & ~I915_PRIORITY_MASK); @@ -428,17 +336,122 @@ void i915_schedule_bump_priority(struct i915_request *rq, unsigned int bump) return; spin_lock_irqsave(&schedule_lock, flags); + __bump_priority(&rq->sched, bump); + spin_unlock_irqrestore(&schedule_lock, flags); +} - attr = rq->sched.attr; - attr.priority |= bump; - __i915_schedule(rq, &attr); +void i915_sched_node_init(struct i915_sched_node *node) +{ + INIT_LIST_HEAD(&node->signalers_list); + INIT_LIST_HEAD(&node->waiters_list); + INIT_LIST_HEAD(&node->link); + node->attr.priority = I915_PRIORITY_INVALID; + node->semaphores = 0; + node->flags = 0; +} - spin_unlock_irqrestore(&schedule_lock, flags); +static struct i915_dependency * +i915_dependency_alloc(void) +{ + return kmem_cache_alloc(global.slab_dependencies, GFP_KERNEL); } -void __i915_priolist_free(struct i915_priolist *p) +static void +i915_dependency_free(struct i915_dependency *dep) { - kmem_cache_free(global.slab_priorities, p); + kmem_cache_free(global.slab_dependencies, dep); +} + +bool __i915_sched_node_add_dependency(struct i915_sched_node *node, + struct i915_sched_node *signal, + struct i915_dependency *dep, + unsigned long flags) +{ + bool ret = false; + + spin_lock_irq(&schedule_lock); + + if (!node_signaled(signal)) { + INIT_LIST_HEAD(&dep->dfs_link); + list_add(&dep->wait_link, &signal->waiters_list); + list_add(&dep->signal_link, &node->signalers_list); + dep->signaler = signal; + dep->flags = flags; + + /* Keep track of whether anyone on this chain has a semaphore */ + if (signal->flags & I915_SCHED_HAS_SEMAPHORE_CHAIN && + !node_started(signal)) + node->flags |= I915_SCHED_HAS_SEMAPHORE_CHAIN; + + /* + * As we do not allow WAIT to preempt inflight requests, + * once we have executed a request, along with triggering + * any execution callbacks, we must preserve its ordering + * within the non-preemptible FIFO. + */ + BUILD_BUG_ON(__NO_PREEMPTION & ~I915_PRIORITY_MASK); + if (flags & I915_DEPENDENCY_EXTERNAL) + __bump_priority(signal, __NO_PREEMPTION); + + ret = true; + } + + spin_unlock_irq(&schedule_lock); + + return ret; +} + +int i915_sched_node_add_dependency(struct i915_sched_node *node, + struct i915_sched_node *signal) +{ + struct i915_dependency *dep; + + dep = i915_dependency_alloc(); + if (!dep) + return -ENOMEM; + + if (!__i915_sched_node_add_dependency(node, signal, dep, + I915_DEPENDENCY_EXTERNAL | + I915_DEPENDENCY_ALLOC)) + i915_dependency_free(dep); + + return 0; +} + +void i915_sched_node_fini(struct i915_sched_node *node) +{ + struct i915_dependency *dep, *tmp; + + GEM_BUG_ON(!list_empty(&node->link)); + + spin_lock_irq(&schedule_lock); + + /* + * Everyone we depended upon (the fences we wait to be signaled) + * should retire before us and remove themselves from our list. + * However, retirement is run independently on each timeline and + * so we may be called out-of-order. + */ + list_for_each_entry_safe(dep, tmp, &node->signalers_list, signal_link) { + GEM_BUG_ON(!node_signaled(dep->signaler)); + GEM_BUG_ON(!list_empty(&dep->dfs_link)); + + list_del(&dep->wait_link); + if (dep->flags & I915_DEPENDENCY_ALLOC) + i915_dependency_free(dep); + } + + /* Remove ourselves from everyone who depends upon us */ + list_for_each_entry_safe(dep, tmp, &node->waiters_list, wait_link) { + GEM_BUG_ON(dep->signaler != node); + GEM_BUG_ON(!list_empty(&dep->dfs_link)); + + list_del(&dep->signal_link); + if (dep->flags & I915_DEPENDENCY_ALLOC) + i915_dependency_free(dep); + } + + spin_unlock_irq(&schedule_lock); } static void i915_global_scheduler_shrink(void) diff --git a/drivers/gpu/drm/i915/i915_scheduler_types.h b/drivers/gpu/drm/i915/i915_scheduler_types.h index f1af3916a808..4f2b2eb7c3e5 100644 --- a/drivers/gpu/drm/i915/i915_scheduler_types.h +++ b/drivers/gpu/drm/i915/i915_scheduler_types.h @@ -66,7 +66,8 @@ struct i915_dependency { struct list_head wait_link; struct list_head dfs_link; unsigned long flags; -#define I915_DEPENDENCY_ALLOC BIT(0) +#define I915_DEPENDENCY_ALLOC BIT(0) +#define I915_DEPENDENCY_EXTERNAL BIT(1) }; #endif /* _I915_SCHEDULER_TYPES_H_ */ diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c index 4e0a351bfbca..11e5a86610bf 100644 --- a/drivers/gpu/drm/i915/intel_lrc.c +++ b/drivers/gpu/drm/i915/intel_lrc.c @@ -164,7 +164,7 @@ #define WA_TAIL_DWORDS 2 #define WA_TAIL_BYTES (sizeof(u32) * WA_TAIL_DWORDS) -#define ACTIVE_PRIORITY (I915_PRIORITY_NEWCLIENT | I915_PRIORITY_NOSEMAPHORE) +#define ACTIVE_PRIORITY (I915_PRIORITY_NOSEMAPHORE) static int execlists_context_deferred_alloc(struct intel_context *ce, struct intel_engine_cs *engine); diff --git a/drivers/gpu/drm/i915/intel_workarounds.c b/drivers/gpu/drm/i915/intel_workarounds.c index 6decd432f4d3..841b8e515f4d 100644 --- a/drivers/gpu/drm/i915/intel_workarounds.c +++ b/drivers/gpu/drm/i915/intel_workarounds.c @@ -518,6 +518,12 @@ static void icl_ctx_workarounds_init(struct intel_engine_cs *engine) struct drm_i915_private *i915 = engine->i915; struct i915_wa_list *wal = &engine->ctx_wa_list; + /* WaDisableBankHangMode:icl */ + wa_write(wal, + GEN8_L3CNTLREG, + intel_uncore_read(engine->uncore, GEN8_L3CNTLREG) | + GEN8_ERRDETBCTRL); + /* Wa_1604370585:icl (pre-prod) * Formerly known as WaPushConstantDereferenceHoldDisable */ diff --git a/drivers/gpu/drm/i915/selftests/intel_lrc.c b/drivers/gpu/drm/i915/selftests/intel_lrc.c index fbee030db940..e8b0b5dbcb2c 100644 --- a/drivers/gpu/drm/i915/selftests/intel_lrc.c +++ b/drivers/gpu/drm/i915/selftests/intel_lrc.c @@ -99,12 +99,14 @@ static int live_busywait_preempt(void *arg) ctx_hi = kernel_context(i915); if (!ctx_hi) goto err_unlock; - ctx_hi->sched.priority = INT_MAX; + ctx_hi->sched.priority = + I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY); ctx_lo = kernel_context(i915); if (!ctx_lo) goto err_ctx_hi; - ctx_lo->sched.priority = INT_MIN; + ctx_lo->sched.priority = + I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY); obj = i915_gem_object_create_internal(i915, PAGE_SIZE); if (IS_ERR(obj)) { @@ -954,12 +956,14 @@ static int live_preempt_hang(void *arg) ctx_hi = kernel_context(i915); if (!ctx_hi) goto err_spin_lo; - ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY; + ctx_hi->sched.priority = + I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY); ctx_lo = kernel_context(i915); if (!ctx_lo) goto err_ctx_hi; - ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY; + ctx_lo->sched.priority = + I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY); for_each_engine(engine, i915, id) { struct i915_request *rq; diff --git a/drivers/gpu/drm/i915/selftests/lib_sw_fence.h b/drivers/gpu/drm/i915/selftests/lib_sw_fence.h index 1f9927e10f3a..e54d6bc23dc3 100644 --- a/drivers/gpu/drm/i915/selftests/lib_sw_fence.h +++ b/drivers/gpu/drm/i915/selftests/lib_sw_fence.h @@ -1,10 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * lib_sw_fence.h - library routines for testing N:M synchronisation points * * Copyright (C) 2017 Intel Corporation - * - * This file is released under the GPLv2. - * */ #ifndef _LIB_SW_FENCE_H_ |