diff options
Diffstat (limited to 'drivers/gpu/drm/i915/gvt')
36 files changed, 4599 insertions, 2069 deletions
diff --git a/drivers/gpu/drm/i915/gvt/Makefile b/drivers/gpu/drm/i915/gvt/Makefile index 2641ba510a61..347116faa558 100644 --- a/drivers/gpu/drm/i915/gvt/Makefile +++ b/drivers/gpu/drm/i915/gvt/Makefile @@ -2,7 +2,8 @@ GVT_DIR := gvt GVT_SOURCE := gvt.o aperture_gm.o handlers.o vgpu.o trace_points.o firmware.o \ interrupt.o gtt.o cfg_space.o opregion.o mmio.o display.o edid.o \ - execlist.o scheduler.o sched_policy.o render.o cmd_parser.o + execlist.o scheduler.o sched_policy.o mmio_context.o cmd_parser.o debugfs.o \ + fb_decoder.o dmabuf.o ccflags-y += -I$(src) -I$(src)/$(GVT_DIR) i915-y += $(addprefix $(GVT_DIR)/, $(GVT_SOURCE)) diff --git a/drivers/gpu/drm/i915/gvt/cfg_space.c b/drivers/gpu/drm/i915/gvt/cfg_space.c index 4ce2e6bd0680..c62346fdc05d 100644 --- a/drivers/gpu/drm/i915/gvt/cfg_space.c +++ b/drivers/gpu/drm/i915/gvt/cfg_space.c @@ -119,16 +119,6 @@ static int map_aperture(struct intel_vgpu *vgpu, bool map) if (map == vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_APERTURE].tracked) return 0; - if (map) { - vgpu->gm.aperture_va = memremap(aperture_pa, aperture_sz, - MEMREMAP_WC); - if (!vgpu->gm.aperture_va) - return -ENOMEM; - } else { - memunmap(vgpu->gm.aperture_va); - vgpu->gm.aperture_va = NULL; - } - val = vgpu_cfg_space(vgpu)[PCI_BASE_ADDRESS_2]; if (val & PCI_BASE_ADDRESS_MEM_TYPE_64) val = *(u64 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_2); @@ -141,11 +131,8 @@ static int map_aperture(struct intel_vgpu *vgpu, bool map) aperture_pa >> PAGE_SHIFT, aperture_sz >> PAGE_SHIFT, map); - if (ret) { - memunmap(vgpu->gm.aperture_va); - vgpu->gm.aperture_va = NULL; + if (ret) return ret; - } vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_APERTURE].tracked = map; return 0; @@ -335,7 +322,8 @@ int intel_vgpu_emulate_cfg_write(struct intel_vgpu *vgpu, unsigned int offset, case INTEL_GVT_PCI_OPREGION: if (WARN_ON(!IS_ALIGNED(offset, 4))) return -EINVAL; - ret = intel_vgpu_init_opregion(vgpu, *(u32 *)p_data); + ret = intel_vgpu_opregion_base_write_handler(vgpu, + *(u32 *)p_data); if (ret) return ret; diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c index 49af94627c8a..c8454ac43fae 100644 --- a/drivers/gpu/drm/i915/gvt/cmd_parser.c +++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c @@ -709,18 +709,13 @@ static void parser_exec_state_dump(struct parser_exec_state *s) print_opcode(cmd_val(s, 0), s->ring_id); - /* print the whole page to trace */ - pr_err(" ip_va=%p: %08x %08x %08x %08x\n", - s->ip_va, cmd_val(s, 0), cmd_val(s, 1), - cmd_val(s, 2), cmd_val(s, 3)); - s->ip_va = (u32 *)((((u64)s->ip_va) >> 12) << 12); while (cnt < 1024) { - pr_err("ip_va=%p: ", s->ip_va); + gvt_dbg_cmd("ip_va=%p: ", s->ip_va); for (i = 0; i < 8; i++) - pr_err("%08x ", cmd_val(s, i)); - pr_err("\n"); + gvt_dbg_cmd("%08x ", cmd_val(s, i)); + gvt_dbg_cmd("\n"); s->ip_va += 8 * sizeof(u32); cnt += 8; @@ -825,11 +820,26 @@ static int force_nonpriv_reg_handler(struct parser_exec_state *s, if (!intel_gvt_in_force_nonpriv_whitelist(gvt, data)) { gvt_err("Unexpected forcenonpriv 0x%x LRI write, value=0x%x\n", offset, data); - return -EINVAL; + return -EPERM; } return 0; } +static inline bool is_mocs_mmio(unsigned int offset) +{ + return ((offset >= 0xc800) && (offset <= 0xcff8)) || + ((offset >= 0xb020) && (offset <= 0xb0a0)); +} + +static int mocs_cmd_reg_handler(struct parser_exec_state *s, + unsigned int offset, unsigned int index) +{ + if (!is_mocs_mmio(offset)) + return -EINVAL; + vgpu_vreg(s->vgpu, offset) = cmd_val(s, index + 1); + return 0; +} + static int cmd_reg_handler(struct parser_exec_state *s, unsigned int offset, unsigned int index, char *cmd) { @@ -839,7 +849,7 @@ static int cmd_reg_handler(struct parser_exec_state *s, if (offset + 4 > gvt->device_info.mmio_size) { gvt_vgpu_err("%s access to (%x) outside of MMIO range\n", cmd, offset); - return -EINVAL; + return -EFAULT; } if (!intel_gvt_mmio_is_cmd_access(gvt, offset)) { @@ -853,10 +863,14 @@ static int cmd_reg_handler(struct parser_exec_state *s, return 0; } - if (is_force_nonpriv_mmio(offset) && - force_nonpriv_reg_handler(s, offset, index)) + if (is_mocs_mmio(offset) && + mocs_cmd_reg_handler(s, offset, index)) return -EINVAL; + if (is_force_nonpriv_mmio(offset) && + force_nonpriv_reg_handler(s, offset, index)) + return -EPERM; + if (offset == i915_mmio_reg_offset(DERRMR) || offset == i915_mmio_reg_offset(FORCEWAKE_MT)) { /* Writing to HW VGT_PVINFO_PAGE offset will be discarded */ @@ -894,11 +908,14 @@ static int cmd_handler_lri(struct parser_exec_state *s) i915_mmio_reg_offset(DERRMR)) ret |= 0; else - ret |= (cmd_reg_inhibit(s, i)) ? -EINVAL : 0; + ret |= (cmd_reg_inhibit(s, i)) ? + -EBADRQC : 0; } if (ret) break; ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lri"); + if (ret) + break; } return ret; } @@ -912,11 +929,15 @@ static int cmd_handler_lrr(struct parser_exec_state *s) if (IS_BROADWELL(s->vgpu->gvt->dev_priv)) ret |= ((cmd_reg_inhibit(s, i) || (cmd_reg_inhibit(s, i + 1)))) ? - -EINVAL : 0; + -EBADRQC : 0; if (ret) break; ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lrr-src"); + if (ret) + break; ret |= cmd_reg_handler(s, cmd_reg(s, i + 1), i, "lrr-dst"); + if (ret) + break; } return ret; } @@ -934,15 +955,19 @@ static int cmd_handler_lrm(struct parser_exec_state *s) for (i = 1; i < cmd_len;) { if (IS_BROADWELL(gvt->dev_priv)) - ret |= (cmd_reg_inhibit(s, i)) ? -EINVAL : 0; + ret |= (cmd_reg_inhibit(s, i)) ? -EBADRQC : 0; if (ret) break; ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lrm"); + if (ret) + break; if (cmd_val(s, 0) & (1 << 22)) { gma = cmd_gma(s, i + 1); if (gmadr_bytes == 8) gma |= (cmd_gma_hi(s, i + 2)) << 32; ret |= cmd_address_audit(s, gma, sizeof(u32), false); + if (ret) + break; } i += gmadr_dw_number(s) + 1; } @@ -958,11 +983,15 @@ static int cmd_handler_srm(struct parser_exec_state *s) for (i = 1; i < cmd_len;) { ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "srm"); + if (ret) + break; if (cmd_val(s, 0) & (1 << 22)) { gma = cmd_gma(s, i + 1); if (gmadr_bytes == 8) gma |= (cmd_gma_hi(s, i + 2)) << 32; ret |= cmd_address_audit(s, gma, sizeof(u32), false); + if (ret) + break; } i += gmadr_dw_number(s) + 1; } @@ -1116,7 +1145,7 @@ static int gen8_decode_mi_display_flip(struct parser_exec_state *s, v = (dword0 & GENMASK(21, 19)) >> 19; if (WARN_ON(v >= ARRAY_SIZE(gen8_plane_code))) - return -EINVAL; + return -EBADRQC; info->pipe = gen8_plane_code[v].pipe; info->plane = gen8_plane_code[v].plane; @@ -1136,7 +1165,7 @@ static int gen8_decode_mi_display_flip(struct parser_exec_state *s, info->surf_reg = SPRSURF(info->pipe); } else { WARN_ON(1); - return -EINVAL; + return -EBADRQC; } return 0; } @@ -1185,7 +1214,7 @@ static int skl_decode_mi_display_flip(struct parser_exec_state *s, default: gvt_vgpu_err("unknown plane code %d\n", plane); - return -EINVAL; + return -EBADRQC; } info->stride_val = (dword1 & GENMASK(15, 6)) >> 6; @@ -1210,13 +1239,13 @@ static int gen8_check_mi_display_flip(struct parser_exec_state *s, return 0; if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { - stride = vgpu_vreg(s->vgpu, info->stride_reg) & GENMASK(9, 0); - tile = (vgpu_vreg(s->vgpu, info->ctrl_reg) & + stride = vgpu_vreg_t(s->vgpu, info->stride_reg) & GENMASK(9, 0); + tile = (vgpu_vreg_t(s->vgpu, info->ctrl_reg) & GENMASK(12, 10)) >> 10; } else { - stride = (vgpu_vreg(s->vgpu, info->stride_reg) & + stride = (vgpu_vreg_t(s->vgpu, info->stride_reg) & GENMASK(15, 6)) >> 6; - tile = (vgpu_vreg(s->vgpu, info->ctrl_reg) & (1 << 10)) >> 10; + tile = (vgpu_vreg_t(s->vgpu, info->ctrl_reg) & (1 << 10)) >> 10; } if (stride != info->stride_val) @@ -1235,21 +1264,21 @@ static int gen8_update_plane_mmio_from_mi_display_flip( struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv; struct intel_vgpu *vgpu = s->vgpu; - set_mask_bits(&vgpu_vreg(vgpu, info->surf_reg), GENMASK(31, 12), + set_mask_bits(&vgpu_vreg_t(vgpu, info->surf_reg), GENMASK(31, 12), info->surf_val << 12); if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { - set_mask_bits(&vgpu_vreg(vgpu, info->stride_reg), GENMASK(9, 0), + set_mask_bits(&vgpu_vreg_t(vgpu, info->stride_reg), GENMASK(9, 0), info->stride_val); - set_mask_bits(&vgpu_vreg(vgpu, info->ctrl_reg), GENMASK(12, 10), + set_mask_bits(&vgpu_vreg_t(vgpu, info->ctrl_reg), GENMASK(12, 10), info->tile_val << 10); } else { - set_mask_bits(&vgpu_vreg(vgpu, info->stride_reg), GENMASK(15, 6), + set_mask_bits(&vgpu_vreg_t(vgpu, info->stride_reg), GENMASK(15, 6), info->stride_val << 6); - set_mask_bits(&vgpu_vreg(vgpu, info->ctrl_reg), GENMASK(10, 10), + set_mask_bits(&vgpu_vreg_t(vgpu, info->ctrl_reg), GENMASK(10, 10), info->tile_val << 10); } - vgpu_vreg(vgpu, PIPE_FRMCOUNT_G4X(info->pipe))++; + vgpu_vreg_t(vgpu, PIPE_FRMCOUNT_G4X(info->pipe))++; intel_vgpu_trigger_virtual_event(vgpu, info->event); return 0; } @@ -1348,10 +1377,13 @@ static unsigned long get_gma_bb_from_cmd(struct parser_exec_state *s, int index) { unsigned long addr; unsigned long gma_high, gma_low; - int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd; + struct intel_vgpu *vgpu = s->vgpu; + int gmadr_bytes = vgpu->gvt->device_info.gmadr_bytes_in_cmd; - if (WARN_ON(gmadr_bytes != 4 && gmadr_bytes != 8)) + if (WARN_ON(gmadr_bytes != 4 && gmadr_bytes != 8)) { + gvt_vgpu_err("invalid gma bytes %d\n", gmadr_bytes); return INTEL_GVT_INVALID_ADDR; + } gma_low = cmd_val(s, index) & BATCH_BUFFER_ADDR_MASK; if (gmadr_bytes == 4) { @@ -1374,16 +1406,16 @@ static inline int cmd_address_audit(struct parser_exec_state *s, if (op_size > max_surface_size) { gvt_vgpu_err("command address audit fail name %s\n", s->info->name); - return -EINVAL; + return -EFAULT; } if (index_mode) { - if (guest_gma >= GTT_PAGE_SIZE / sizeof(u64)) { - ret = -EINVAL; + if (guest_gma >= I915_GTT_PAGE_SIZE / sizeof(u64)) { + ret = -EFAULT; goto err; } } else if (!intel_gvt_ggtt_validate_range(vgpu, guest_gma, op_size)) { - ret = -EINVAL; + ret = -EFAULT; goto err; } @@ -1439,7 +1471,7 @@ static inline int unexpected_cmd(struct parser_exec_state *s) gvt_vgpu_err("Unexpected %s in command buffer!\n", s->info->name); - return -EINVAL; + return -EBADRQC; } static int cmd_handler_mi_semaphore_wait(struct parser_exec_state *s) @@ -1545,10 +1577,10 @@ static int copy_gma_to_hva(struct intel_vgpu *vgpu, struct intel_vgpu_mm *mm, return -EFAULT; } - offset = gma & (GTT_PAGE_SIZE - 1); + offset = gma & (I915_GTT_PAGE_SIZE - 1); - copy_len = (end_gma - gma) >= (GTT_PAGE_SIZE - offset) ? - GTT_PAGE_SIZE - offset : end_gma - gma; + copy_len = (end_gma - gma) >= (I915_GTT_PAGE_SIZE - offset) ? + I915_GTT_PAGE_SIZE - offset : end_gma - gma; intel_gvt_hypervisor_read_gpa(vgpu, gpa, va + len, copy_len); @@ -1576,110 +1608,113 @@ static int batch_buffer_needs_scan(struct parser_exec_state *s) return 1; } -static int find_bb_size(struct parser_exec_state *s) +static int find_bb_size(struct parser_exec_state *s, unsigned long *bb_size) { unsigned long gma = 0; struct cmd_info *info; - int bb_size = 0; uint32_t cmd_len = 0; - bool met_bb_end = false; + bool bb_end = false; struct intel_vgpu *vgpu = s->vgpu; u32 cmd; + *bb_size = 0; + /* get the start gm address of the batch buffer */ gma = get_gma_bb_from_cmd(s, 1); - cmd = cmd_val(s, 0); + if (gma == INTEL_GVT_INVALID_ADDR) + return -EFAULT; + cmd = cmd_val(s, 0); info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id); if (info == NULL) { gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x\n", cmd, get_opcode(cmd, s->ring_id)); - return -EINVAL; + return -EBADRQC; } do { - copy_gma_to_hva(s->vgpu, s->vgpu->gtt.ggtt_mm, - gma, gma + 4, &cmd); + if (copy_gma_to_hva(s->vgpu, s->vgpu->gtt.ggtt_mm, + gma, gma + 4, &cmd) < 0) + return -EFAULT; info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id); if (info == NULL) { gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x\n", cmd, get_opcode(cmd, s->ring_id)); - return -EINVAL; + return -EBADRQC; } if (info->opcode == OP_MI_BATCH_BUFFER_END) { - met_bb_end = true; + bb_end = true; } else if (info->opcode == OP_MI_BATCH_BUFFER_START) { - if (BATCH_BUFFER_2ND_LEVEL_BIT(cmd) == 0) { + if (BATCH_BUFFER_2ND_LEVEL_BIT(cmd) == 0) /* chained batch buffer */ - met_bb_end = true; - } + bb_end = true; } cmd_len = get_cmd_length(info, cmd) << 2; - bb_size += cmd_len; + *bb_size += cmd_len; gma += cmd_len; + } while (!bb_end); - } while (!met_bb_end); - - return bb_size; + return 0; } static int perform_bb_shadow(struct parser_exec_state *s) { - struct intel_shadow_bb_entry *entry_obj; struct intel_vgpu *vgpu = s->vgpu; + struct intel_vgpu_shadow_bb *bb; unsigned long gma = 0; - int bb_size; - void *dst = NULL; + unsigned long bb_size; int ret = 0; /* get the start gm address of the batch buffer */ gma = get_gma_bb_from_cmd(s, 1); + if (gma == INTEL_GVT_INVALID_ADDR) + return -EFAULT; - /* get the size of the batch buffer */ - bb_size = find_bb_size(s); - if (bb_size < 0) - return -EINVAL; + ret = find_bb_size(s, &bb_size); + if (ret) + return ret; - /* allocate shadow batch buffer */ - entry_obj = kmalloc(sizeof(*entry_obj), GFP_KERNEL); - if (entry_obj == NULL) + bb = kzalloc(sizeof(*bb), GFP_KERNEL); + if (!bb) return -ENOMEM; - entry_obj->obj = - i915_gem_object_create(s->vgpu->gvt->dev_priv, - roundup(bb_size, PAGE_SIZE)); - if (IS_ERR(entry_obj->obj)) { - ret = PTR_ERR(entry_obj->obj); - goto free_entry; + bb->obj = i915_gem_object_create(s->vgpu->gvt->dev_priv, + roundup(bb_size, PAGE_SIZE)); + if (IS_ERR(bb->obj)) { + ret = PTR_ERR(bb->obj); + goto err_free_bb; } - entry_obj->len = bb_size; - INIT_LIST_HEAD(&entry_obj->list); - dst = i915_gem_object_pin_map(entry_obj->obj, I915_MAP_WB); - if (IS_ERR(dst)) { - ret = PTR_ERR(dst); - goto put_obj; - } + ret = i915_gem_obj_prepare_shmem_write(bb->obj, &bb->clflush); + if (ret) + goto err_free_obj; - ret = i915_gem_object_set_to_cpu_domain(entry_obj->obj, false); - if (ret) { - gvt_vgpu_err("failed to set shadow batch to CPU\n"); - goto unmap_src; + bb->va = i915_gem_object_pin_map(bb->obj, I915_MAP_WB); + if (IS_ERR(bb->va)) { + ret = PTR_ERR(bb->va); + goto err_finish_shmem_access; } - entry_obj->va = dst; - entry_obj->bb_start_cmd_va = s->ip_va; + if (bb->clflush & CLFLUSH_BEFORE) { + drm_clflush_virt_range(bb->va, bb->obj->base.size); + bb->clflush &= ~CLFLUSH_BEFORE; + } - /* copy batch buffer to shadow batch buffer*/ ret = copy_gma_to_hva(s->vgpu, s->vgpu->gtt.ggtt_mm, gma, gma + bb_size, - dst); + bb->va); if (ret < 0) { gvt_vgpu_err("fail to copy guest ring buffer\n"); - goto unmap_src; + ret = -EFAULT; + goto err_unmap; } - list_add(&entry_obj->list, &s->workload->shadow_bb); + INIT_LIST_HEAD(&bb->list); + list_add(&bb->list, &s->workload->shadow_bb); + + bb->accessing = true; + bb->bb_start_cmd_va = s->ip_va; + /* * ip_va saves the virtual address of the shadow batch buffer, while * ip_gma saves the graphics address of the original batch buffer. @@ -1688,17 +1723,17 @@ static int perform_bb_shadow(struct parser_exec_state *s) * buffer's gma in pair. After all, we don't want to pin the shadow * buffer here (too early). */ - s->ip_va = dst; + s->ip_va = bb->va; s->ip_gma = gma; - return 0; - -unmap_src: - i915_gem_object_unpin_map(entry_obj->obj); -put_obj: - i915_gem_object_put(entry_obj->obj); -free_entry: - kfree(entry_obj); +err_unmap: + i915_gem_object_unpin_map(bb->obj); +err_finish_shmem_access: + i915_gem_obj_finish_shmem_access(bb->obj); +err_free_obj: + i915_gem_object_put(bb->obj); +err_free_bb: + kfree(bb); return ret; } @@ -1710,13 +1745,13 @@ static int cmd_handler_mi_batch_buffer_start(struct parser_exec_state *s) if (s->buf_type == BATCH_BUFFER_2ND_LEVEL) { gvt_vgpu_err("Found MI_BATCH_BUFFER_START in 2nd level BB\n"); - return -EINVAL; + return -EFAULT; } second_level = BATCH_BUFFER_2ND_LEVEL_BIT(cmd_val(s, 0)) == 1; if (second_level && (s->buf_type != BATCH_BUFFER_INSTRUCTION)) { gvt_vgpu_err("Jumping to 2nd level BB from RB is not allowed\n"); - return -EINVAL; + return -EFAULT; } s->saved_buf_addr_type = s->buf_addr_type; @@ -1740,7 +1775,6 @@ static int cmd_handler_mi_batch_buffer_start(struct parser_exec_state *s) if (ret < 0) return ret; } - return ret; } @@ -2430,7 +2464,7 @@ static int cmd_parser_exec(struct parser_exec_state *s) if (info == NULL) { gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x\n", cmd, get_opcode(cmd, s->ring_id)); - return -EINVAL; + return -EBADRQC; } s->info = info; @@ -2465,6 +2499,10 @@ static inline bool gma_out_of_range(unsigned long gma, return (gma > gma_tail) && (gma < gma_head); } +/* Keep the consistent return type, e.g EBADRQC for unknown + * cmd, EFAULT for invalid address, EPERM for nonpriv. later + * works as the input of VM healthy status. + */ static int command_scan(struct parser_exec_state *s, unsigned long rb_head, unsigned long rb_tail, unsigned long rb_start, unsigned long rb_len) @@ -2487,7 +2525,7 @@ static int command_scan(struct parser_exec_state *s, s->ip_gma, rb_start, gma_bottom); parser_exec_state_dump(s); - return -EINVAL; + return -EFAULT; } if (gma_out_of_range(s->ip_gma, gma_head, gma_tail)) { gvt_vgpu_err("ip_gma %lx out of range." @@ -2516,7 +2554,7 @@ static int scan_workload(struct intel_vgpu_workload *workload) int ret = 0; /* ring base is page aligned */ - if (WARN_ON(!IS_ALIGNED(workload->rb_start, GTT_PAGE_SIZE))) + if (WARN_ON(!IS_ALIGNED(workload->rb_start, I915_GTT_PAGE_SIZE))) return -EINVAL; gma_head = workload->rb_start + workload->rb_head; @@ -2565,7 +2603,8 @@ static int scan_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) wa_ctx); /* ring base is page aligned */ - if (WARN_ON(!IS_ALIGNED(wa_ctx->indirect_ctx.guest_gma, GTT_PAGE_SIZE))) + if (WARN_ON(!IS_ALIGNED(wa_ctx->indirect_ctx.guest_gma, + I915_GTT_PAGE_SIZE))) return -EINVAL; ring_tail = wa_ctx->indirect_ctx.size + 3 * sizeof(uint32_t); @@ -2604,6 +2643,7 @@ out: static int shadow_workload_ring_buffer(struct intel_vgpu_workload *workload) { struct intel_vgpu *vgpu = workload->vgpu; + struct intel_vgpu_submission *s = &vgpu->submission; unsigned long gma_head, gma_tail, gma_top, guest_rb_size; void *shadow_ring_buffer_va; int ring_id = workload->ring_id; @@ -2619,19 +2659,21 @@ static int shadow_workload_ring_buffer(struct intel_vgpu_workload *workload) gma_tail = workload->rb_start + workload->rb_tail; gma_top = workload->rb_start + guest_rb_size; - if (workload->rb_len > vgpu->reserve_ring_buffer_size[ring_id]) { - void *va = vgpu->reserve_ring_buffer_va[ring_id]; + if (workload->rb_len > s->ring_scan_buffer_size[ring_id]) { + void *p; + /* realloc the new ring buffer if needed */ - vgpu->reserve_ring_buffer_va[ring_id] = - krealloc(va, workload->rb_len, GFP_KERNEL); - if (!vgpu->reserve_ring_buffer_va[ring_id]) { - gvt_vgpu_err("fail to alloc reserve ring buffer\n"); + p = krealloc(s->ring_scan_buffer[ring_id], workload->rb_len, + GFP_KERNEL); + if (!p) { + gvt_vgpu_err("fail to re-alloc ring scan buffer\n"); return -ENOMEM; } - vgpu->reserve_ring_buffer_size[ring_id] = workload->rb_len; + s->ring_scan_buffer[ring_id] = p; + s->ring_scan_buffer_size[ring_id] = workload->rb_len; } - shadow_ring_buffer_va = vgpu->reserve_ring_buffer_va[ring_id]; + shadow_ring_buffer_va = s->ring_scan_buffer[ring_id]; /* get shadow ring buffer va */ workload->shadow_ring_buffer_va = shadow_ring_buffer_va; diff --git a/drivers/gpu/drm/i915/gvt/debug.h b/drivers/gpu/drm/i915/gvt/debug.h index b0cff4dc2684..c6027125c1ec 100644 --- a/drivers/gpu/drm/i915/gvt/debug.h +++ b/drivers/gpu/drm/i915/gvt/debug.h @@ -25,41 +25,41 @@ #define __GVT_DEBUG_H__ #define gvt_err(fmt, args...) \ - DRM_ERROR("gvt: "fmt, ##args) + pr_err("gvt: "fmt, ##args) #define gvt_vgpu_err(fmt, args...) \ do { \ if (IS_ERR_OR_NULL(vgpu)) \ - DRM_DEBUG_DRIVER("gvt: "fmt, ##args); \ + pr_err("gvt: "fmt, ##args); \ else \ - DRM_DEBUG_DRIVER("gvt: vgpu %d: "fmt, vgpu->id, ##args);\ + pr_err("gvt: vgpu %d: "fmt, vgpu->id, ##args);\ } while (0) #define gvt_dbg_core(fmt, args...) \ - DRM_DEBUG_DRIVER("gvt: core: "fmt, ##args) + pr_debug("gvt: core: "fmt, ##args) #define gvt_dbg_irq(fmt, args...) \ - DRM_DEBUG_DRIVER("gvt: irq: "fmt, ##args) + pr_debug("gvt: irq: "fmt, ##args) #define gvt_dbg_mm(fmt, args...) \ - DRM_DEBUG_DRIVER("gvt: mm: "fmt, ##args) + pr_debug("gvt: mm: "fmt, ##args) #define gvt_dbg_mmio(fmt, args...) \ - DRM_DEBUG_DRIVER("gvt: mmio: "fmt, ##args) + pr_debug("gvt: mmio: "fmt, ##args) #define gvt_dbg_dpy(fmt, args...) \ - DRM_DEBUG_DRIVER("gvt: dpy: "fmt, ##args) + pr_debug("gvt: dpy: "fmt, ##args) #define gvt_dbg_el(fmt, args...) \ - DRM_DEBUG_DRIVER("gvt: el: "fmt, ##args) + pr_debug("gvt: el: "fmt, ##args) #define gvt_dbg_sched(fmt, args...) \ - DRM_DEBUG_DRIVER("gvt: sched: "fmt, ##args) + pr_debug("gvt: sched: "fmt, ##args) #define gvt_dbg_render(fmt, args...) \ - DRM_DEBUG_DRIVER("gvt: render: "fmt, ##args) + pr_debug("gvt: render: "fmt, ##args) #define gvt_dbg_cmd(fmt, args...) \ - DRM_DEBUG_DRIVER("gvt: cmd: "fmt, ##args) + pr_debug("gvt: cmd: "fmt, ##args) #endif diff --git a/drivers/gpu/drm/i915/gvt/debugfs.c b/drivers/gpu/drm/i915/gvt/debugfs.c new file mode 100644 index 000000000000..32a66dfdf112 --- /dev/null +++ b/drivers/gpu/drm/i915/gvt/debugfs.c @@ -0,0 +1,212 @@ +/* + * Copyright(c) 2011-2017 Intel Corporation. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include <linux/debugfs.h> +#include <linux/list_sort.h> +#include "i915_drv.h" +#include "gvt.h" + +struct mmio_diff_param { + struct intel_vgpu *vgpu; + int total; + int diff; + struct list_head diff_mmio_list; +}; + +struct diff_mmio { + struct list_head node; + u32 offset; + u32 preg; + u32 vreg; +}; + +/* Compare two diff_mmio items. */ +static int mmio_offset_compare(void *priv, + struct list_head *a, struct list_head *b) +{ + struct diff_mmio *ma; + struct diff_mmio *mb; + + ma = container_of(a, struct diff_mmio, node); + mb = container_of(b, struct diff_mmio, node); + if (ma->offset < mb->offset) + return -1; + else if (ma->offset > mb->offset) + return 1; + return 0; +} + +static inline int mmio_diff_handler(struct intel_gvt *gvt, + u32 offset, void *data) +{ + struct drm_i915_private *dev_priv = gvt->dev_priv; + struct mmio_diff_param *param = data; + struct diff_mmio *node; + u32 preg, vreg; + + preg = I915_READ_NOTRACE(_MMIO(offset)); + vreg = vgpu_vreg(param->vgpu, offset); + + if (preg != vreg) { + node = kmalloc(sizeof(*node), GFP_KERNEL); + if (!node) + return -ENOMEM; + + node->offset = offset; + node->preg = preg; + node->vreg = vreg; + list_add(&node->node, ¶m->diff_mmio_list); + param->diff++; + } + param->total++; + return 0; +} + +/* Show the all the different values of tracked mmio. */ +static int vgpu_mmio_diff_show(struct seq_file *s, void *unused) +{ + struct intel_vgpu *vgpu = s->private; + struct intel_gvt *gvt = vgpu->gvt; + struct mmio_diff_param param = { + .vgpu = vgpu, + .total = 0, + .diff = 0, + }; + struct diff_mmio *node, *next; + + INIT_LIST_HEAD(¶m.diff_mmio_list); + + mutex_lock(&gvt->lock); + spin_lock_bh(&gvt->scheduler.mmio_context_lock); + + mmio_hw_access_pre(gvt->dev_priv); + /* Recognize all the diff mmios to list. */ + intel_gvt_for_each_tracked_mmio(gvt, mmio_diff_handler, ¶m); + mmio_hw_access_post(gvt->dev_priv); + + spin_unlock_bh(&gvt->scheduler.mmio_context_lock); + mutex_unlock(&gvt->lock); + + /* In an ascending order by mmio offset. */ + list_sort(NULL, ¶m.diff_mmio_list, mmio_offset_compare); + + seq_printf(s, "%-8s %-8s %-8s %-8s\n", "Offset", "HW", "vGPU", "Diff"); + list_for_each_entry_safe(node, next, ¶m.diff_mmio_list, node) { + u32 diff = node->preg ^ node->vreg; + + seq_printf(s, "%08x %08x %08x %*pbl\n", + node->offset, node->preg, node->vreg, + 32, &diff); + list_del(&node->node); + kfree(node); + } + seq_printf(s, "Total: %d, Diff: %d\n", param.total, param.diff); + return 0; +} + +static int vgpu_mmio_diff_open(struct inode *inode, struct file *file) +{ + return single_open(file, vgpu_mmio_diff_show, inode->i_private); +} + +static const struct file_operations vgpu_mmio_diff_fops = { + .open = vgpu_mmio_diff_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +/** + * intel_gvt_debugfs_add_vgpu - register debugfs entries for a vGPU + * @vgpu: a vGPU + * + * Returns: + * Zero on success, negative error code if failed. + */ +int intel_gvt_debugfs_add_vgpu(struct intel_vgpu *vgpu) +{ + struct dentry *ent; + char name[10] = ""; + + sprintf(name, "vgpu%d", vgpu->id); + vgpu->debugfs = debugfs_create_dir(name, vgpu->gvt->debugfs_root); + if (!vgpu->debugfs) + return -ENOMEM; + + ent = debugfs_create_bool("active", 0444, vgpu->debugfs, + &vgpu->active); + if (!ent) + return -ENOMEM; + + ent = debugfs_create_file("mmio_diff", 0444, vgpu->debugfs, + vgpu, &vgpu_mmio_diff_fops); + if (!ent) + return -ENOMEM; + + return 0; +} + +/** + * intel_gvt_debugfs_remove_vgpu - remove debugfs entries of a vGPU + * @vgpu: a vGPU + */ +void intel_gvt_debugfs_remove_vgpu(struct intel_vgpu *vgpu) +{ + debugfs_remove_recursive(vgpu->debugfs); + vgpu->debugfs = NULL; +} + +/** + * intel_gvt_debugfs_init - register gvt debugfs root entry + * @gvt: GVT device + * + * Returns: + * zero on success, negative if failed. + */ +int intel_gvt_debugfs_init(struct intel_gvt *gvt) +{ + struct drm_minor *minor = gvt->dev_priv->drm.primary; + struct dentry *ent; + + gvt->debugfs_root = debugfs_create_dir("gvt", minor->debugfs_root); + if (!gvt->debugfs_root) { + gvt_err("Cannot create debugfs dir\n"); + return -ENOMEM; + } + + ent = debugfs_create_ulong("num_tracked_mmio", 0444, gvt->debugfs_root, + &gvt->mmio.num_tracked_mmio); + if (!ent) + return -ENOMEM; + + return 0; +} + +/** + * intel_gvt_debugfs_clean - remove debugfs entries + * @gvt: GVT device + */ +void intel_gvt_debugfs_clean(struct intel_gvt *gvt) +{ + debugfs_remove_recursive(gvt->debugfs_root); + gvt->debugfs_root = NULL; +} diff --git a/drivers/gpu/drm/i915/gvt/display.c b/drivers/gpu/drm/i915/gvt/display.c index 309f3fa6794a..dd96ffc878ac 100644 --- a/drivers/gpu/drm/i915/gvt/display.c +++ b/drivers/gpu/drm/i915/gvt/display.c @@ -59,7 +59,7 @@ static int edp_pipe_is_enabled(struct intel_vgpu *vgpu) { struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; - if (!(vgpu_vreg(vgpu, PIPECONF(_PIPE_EDP)) & PIPECONF_ENABLE)) + if (!(vgpu_vreg_t(vgpu, PIPECONF(_PIPE_EDP)) & PIPECONF_ENABLE)) return 0; if (!(vgpu_vreg(vgpu, _TRANS_DDI_FUNC_CTL_EDP) & TRANS_DDI_FUNC_ENABLE)) @@ -67,14 +67,14 @@ static int edp_pipe_is_enabled(struct intel_vgpu *vgpu) return 1; } -static int pipe_is_enabled(struct intel_vgpu *vgpu, int pipe) +int pipe_is_enabled(struct intel_vgpu *vgpu, int pipe) { struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; if (WARN_ON(pipe < PIPE_A || pipe >= I915_MAX_PIPES)) return -EINVAL; - if (vgpu_vreg(vgpu, PIPECONF(pipe)) & PIPECONF_ENABLE) + if (vgpu_vreg_t(vgpu, PIPECONF(pipe)) & PIPECONF_ENABLE) return 1; if (edp_pipe_is_enabled(vgpu) && @@ -169,105 +169,105 @@ static u8 dpcd_fix_data[DPCD_HEADER_SIZE] = { static void emulate_monitor_status_change(struct intel_vgpu *vgpu) { struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; - vgpu_vreg(vgpu, SDEISR) &= ~(SDE_PORTB_HOTPLUG_CPT | + vgpu_vreg_t(vgpu, SDEISR) &= ~(SDE_PORTB_HOTPLUG_CPT | SDE_PORTC_HOTPLUG_CPT | SDE_PORTD_HOTPLUG_CPT); if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { - vgpu_vreg(vgpu, SDEISR) &= ~(SDE_PORTA_HOTPLUG_SPT | + vgpu_vreg_t(vgpu, SDEISR) &= ~(SDE_PORTA_HOTPLUG_SPT | SDE_PORTE_HOTPLUG_SPT); - vgpu_vreg(vgpu, SKL_FUSE_STATUS) |= + vgpu_vreg_t(vgpu, SKL_FUSE_STATUS) |= SKL_FUSE_DOWNLOAD_STATUS | SKL_FUSE_PG_DIST_STATUS(SKL_PG0) | SKL_FUSE_PG_DIST_STATUS(SKL_PG1) | SKL_FUSE_PG_DIST_STATUS(SKL_PG2); - vgpu_vreg(vgpu, LCPLL1_CTL) |= + vgpu_vreg_t(vgpu, LCPLL1_CTL) |= LCPLL_PLL_ENABLE | LCPLL_PLL_LOCK; - vgpu_vreg(vgpu, LCPLL2_CTL) |= LCPLL_PLL_ENABLE; + vgpu_vreg_t(vgpu, LCPLL2_CTL) |= LCPLL_PLL_ENABLE; } if (intel_vgpu_has_monitor_on_port(vgpu, PORT_B)) { - vgpu_vreg(vgpu, SFUSE_STRAP) |= SFUSE_STRAP_DDIB_DETECTED; - vgpu_vreg(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) &= + vgpu_vreg_t(vgpu, SFUSE_STRAP) |= SFUSE_STRAP_DDIB_DETECTED; + vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) &= ~(TRANS_DDI_BPC_MASK | TRANS_DDI_MODE_SELECT_MASK | TRANS_DDI_PORT_MASK); - vgpu_vreg(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |= + vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |= (TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DP_SST | (PORT_B << TRANS_DDI_PORT_SHIFT) | TRANS_DDI_FUNC_ENABLE); if (IS_BROADWELL(dev_priv)) { - vgpu_vreg(vgpu, PORT_CLK_SEL(PORT_B)) &= + vgpu_vreg_t(vgpu, PORT_CLK_SEL(PORT_B)) &= ~PORT_CLK_SEL_MASK; - vgpu_vreg(vgpu, PORT_CLK_SEL(PORT_B)) |= + vgpu_vreg_t(vgpu, PORT_CLK_SEL(PORT_B)) |= PORT_CLK_SEL_LCPLL_810; } - vgpu_vreg(vgpu, DDI_BUF_CTL(PORT_B)) |= DDI_BUF_CTL_ENABLE; - vgpu_vreg(vgpu, DDI_BUF_CTL(PORT_B)) &= ~DDI_BUF_IS_IDLE; - vgpu_vreg(vgpu, SDEISR) |= SDE_PORTB_HOTPLUG_CPT; + vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_B)) |= DDI_BUF_CTL_ENABLE; + vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_B)) &= ~DDI_BUF_IS_IDLE; + vgpu_vreg_t(vgpu, SDEISR) |= SDE_PORTB_HOTPLUG_CPT; } if (intel_vgpu_has_monitor_on_port(vgpu, PORT_C)) { - vgpu_vreg(vgpu, SDEISR) |= SDE_PORTC_HOTPLUG_CPT; - vgpu_vreg(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) &= + vgpu_vreg_t(vgpu, SDEISR) |= SDE_PORTC_HOTPLUG_CPT; + vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) &= ~(TRANS_DDI_BPC_MASK | TRANS_DDI_MODE_SELECT_MASK | TRANS_DDI_PORT_MASK); - vgpu_vreg(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |= + vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |= (TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DP_SST | (PORT_C << TRANS_DDI_PORT_SHIFT) | TRANS_DDI_FUNC_ENABLE); if (IS_BROADWELL(dev_priv)) { - vgpu_vreg(vgpu, PORT_CLK_SEL(PORT_C)) &= + vgpu_vreg_t(vgpu, PORT_CLK_SEL(PORT_C)) &= ~PORT_CLK_SEL_MASK; - vgpu_vreg(vgpu, PORT_CLK_SEL(PORT_C)) |= + vgpu_vreg_t(vgpu, PORT_CLK_SEL(PORT_C)) |= PORT_CLK_SEL_LCPLL_810; } - vgpu_vreg(vgpu, DDI_BUF_CTL(PORT_C)) |= DDI_BUF_CTL_ENABLE; - vgpu_vreg(vgpu, DDI_BUF_CTL(PORT_C)) &= ~DDI_BUF_IS_IDLE; - vgpu_vreg(vgpu, SFUSE_STRAP) |= SFUSE_STRAP_DDIC_DETECTED; + vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_C)) |= DDI_BUF_CTL_ENABLE; + vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_C)) &= ~DDI_BUF_IS_IDLE; + vgpu_vreg_t(vgpu, SFUSE_STRAP) |= SFUSE_STRAP_DDIC_DETECTED; } if (intel_vgpu_has_monitor_on_port(vgpu, PORT_D)) { - vgpu_vreg(vgpu, SDEISR) |= SDE_PORTD_HOTPLUG_CPT; - vgpu_vreg(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) &= + vgpu_vreg_t(vgpu, SDEISR) |= SDE_PORTD_HOTPLUG_CPT; + vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) &= ~(TRANS_DDI_BPC_MASK | TRANS_DDI_MODE_SELECT_MASK | TRANS_DDI_PORT_MASK); - vgpu_vreg(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |= + vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |= (TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DP_SST | (PORT_D << TRANS_DDI_PORT_SHIFT) | TRANS_DDI_FUNC_ENABLE); if (IS_BROADWELL(dev_priv)) { - vgpu_vreg(vgpu, PORT_CLK_SEL(PORT_D)) &= + vgpu_vreg_t(vgpu, PORT_CLK_SEL(PORT_D)) &= ~PORT_CLK_SEL_MASK; - vgpu_vreg(vgpu, PORT_CLK_SEL(PORT_D)) |= + vgpu_vreg_t(vgpu, PORT_CLK_SEL(PORT_D)) |= PORT_CLK_SEL_LCPLL_810; } - vgpu_vreg(vgpu, DDI_BUF_CTL(PORT_D)) |= DDI_BUF_CTL_ENABLE; - vgpu_vreg(vgpu, DDI_BUF_CTL(PORT_D)) &= ~DDI_BUF_IS_IDLE; - vgpu_vreg(vgpu, SFUSE_STRAP) |= SFUSE_STRAP_DDID_DETECTED; + vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_D)) |= DDI_BUF_CTL_ENABLE; + vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_D)) &= ~DDI_BUF_IS_IDLE; + vgpu_vreg_t(vgpu, SFUSE_STRAP) |= SFUSE_STRAP_DDID_DETECTED; } if ((IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) && intel_vgpu_has_monitor_on_port(vgpu, PORT_E)) { - vgpu_vreg(vgpu, SDEISR) |= SDE_PORTE_HOTPLUG_SPT; + vgpu_vreg_t(vgpu, SDEISR) |= SDE_PORTE_HOTPLUG_SPT; } if (intel_vgpu_has_monitor_on_port(vgpu, PORT_A)) { if (IS_BROADWELL(dev_priv)) - vgpu_vreg(vgpu, GEN8_DE_PORT_ISR) |= + vgpu_vreg_t(vgpu, GEN8_DE_PORT_ISR) |= GEN8_PORT_DP_A_HOTPLUG; else - vgpu_vreg(vgpu, SDEISR) |= SDE_PORTA_HOTPLUG_SPT; + vgpu_vreg_t(vgpu, SDEISR) |= SDE_PORTA_HOTPLUG_SPT; - vgpu_vreg(vgpu, DDI_BUF_CTL(PORT_A)) |= DDI_INIT_DISPLAY_DETECTED; + vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_A)) |= DDI_INIT_DISPLAY_DETECTED; } /* Clear host CRT status, so guest couldn't detect this host CRT. */ if (IS_BROADWELL(dev_priv)) - vgpu_vreg(vgpu, PCH_ADPA) &= ~ADPA_CRT_HOTPLUG_MONITOR_MASK; + vgpu_vreg_t(vgpu, PCH_ADPA) &= ~ADPA_CRT_HOTPLUG_MONITOR_MASK; - vgpu_vreg(vgpu, PIPECONF(PIPE_A)) |= PIPECONF_ENABLE; + vgpu_vreg_t(vgpu, PIPECONF(PIPE_A)) |= PIPECONF_ENABLE; } static void clean_virtual_dp_monitor(struct intel_vgpu *vgpu, int port_num) @@ -369,12 +369,12 @@ static void emulate_vblank_on_pipe(struct intel_vgpu *vgpu, int pipe) if (!pipe_is_enabled(vgpu, pipe)) continue; - vgpu_vreg(vgpu, PIPE_FLIPCOUNT_G4X(pipe))++; + vgpu_vreg_t(vgpu, PIPE_FLIPCOUNT_G4X(pipe))++; intel_vgpu_trigger_virtual_event(vgpu, event); } if (pipe_is_enabled(vgpu, pipe)) { - vgpu_vreg(vgpu, PIPE_FRMCOUNT_G4X(pipe))++; + vgpu_vreg_t(vgpu, PIPE_FRMCOUNT_G4X(pipe))++; intel_vgpu_trigger_virtual_event(vgpu, vblank_event[pipe]); } } diff --git a/drivers/gpu/drm/i915/gvt/display.h b/drivers/gpu/drm/i915/gvt/display.h index d73de22102e2..b46b86892d58 100644 --- a/drivers/gpu/drm/i915/gvt/display.h +++ b/drivers/gpu/drm/i915/gvt/display.h @@ -179,4 +179,6 @@ int intel_vgpu_init_display(struct intel_vgpu *vgpu, u64 resolution); void intel_vgpu_reset_display(struct intel_vgpu *vgpu); void intel_vgpu_clean_display(struct intel_vgpu *vgpu); +int pipe_is_enabled(struct intel_vgpu *vgpu, int pipe); + #endif diff --git a/drivers/gpu/drm/i915/gvt/dmabuf.c b/drivers/gpu/drm/i915/gvt/dmabuf.c new file mode 100644 index 000000000000..2fb7b34ef561 --- /dev/null +++ b/drivers/gpu/drm/i915/gvt/dmabuf.c @@ -0,0 +1,536 @@ +/* + * Copyright 2017 Intel Corporation. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Authors: + * Zhiyuan Lv <zhiyuan.lv@intel.com> + * + * Contributors: + * Xiaoguang Chen + * Tina Zhang <tina.zhang@intel.com> + */ + +#include <linux/dma-buf.h> +#include <drm/drmP.h> +#include <linux/vfio.h> + +#include "i915_drv.h" +#include "gvt.h" + +#define GEN8_DECODE_PTE(pte) (pte & GENMASK_ULL(63, 12)) + +static int vgpu_gem_get_pages( + struct drm_i915_gem_object *obj) +{ + struct drm_i915_private *dev_priv = to_i915(obj->base.dev); + struct sg_table *st; + struct scatterlist *sg; + int i, ret; + gen8_pte_t __iomem *gtt_entries; + struct intel_vgpu_fb_info *fb_info; + + fb_info = (struct intel_vgpu_fb_info *)obj->gvt_info; + if (WARN_ON(!fb_info)) + return -ENODEV; + + st = kmalloc(sizeof(*st), GFP_KERNEL); + if (unlikely(!st)) + return -ENOMEM; + + ret = sg_alloc_table(st, fb_info->size, GFP_KERNEL); + if (ret) { + kfree(st); + return ret; + } + gtt_entries = (gen8_pte_t __iomem *)dev_priv->ggtt.gsm + + (fb_info->start >> PAGE_SHIFT); + for_each_sg(st->sgl, sg, fb_info->size, i) { + sg->offset = 0; + sg->length = PAGE_SIZE; + sg_dma_address(sg) = + GEN8_DECODE_PTE(readq(>t_entries[i])); + sg_dma_len(sg) = PAGE_SIZE; + } + + __i915_gem_object_set_pages(obj, st, PAGE_SIZE); + + return 0; +} + +static void vgpu_gem_put_pages(struct drm_i915_gem_object *obj, + struct sg_table *pages) +{ + sg_free_table(pages); + kfree(pages); +} + +static void dmabuf_gem_object_free(struct kref *kref) +{ + struct intel_vgpu_dmabuf_obj *obj = + container_of(kref, struct intel_vgpu_dmabuf_obj, kref); + struct intel_vgpu *vgpu = obj->vgpu; + struct list_head *pos; + struct intel_vgpu_dmabuf_obj *dmabuf_obj; + + if (vgpu && vgpu->active && !list_empty(&vgpu->dmabuf_obj_list_head)) { + list_for_each(pos, &vgpu->dmabuf_obj_list_head) { + dmabuf_obj = container_of(pos, + struct intel_vgpu_dmabuf_obj, list); + if (dmabuf_obj == obj) { + intel_gvt_hypervisor_put_vfio_device(vgpu); + idr_remove(&vgpu->object_idr, + dmabuf_obj->dmabuf_id); + kfree(dmabuf_obj->info); + kfree(dmabuf_obj); + list_del(pos); + break; + } + } + } else { + /* Free the orphan dmabuf_objs here */ + kfree(obj->info); + kfree(obj); + } +} + + +static inline void dmabuf_obj_get(struct intel_vgpu_dmabuf_obj *obj) +{ + kref_get(&obj->kref); +} + +static inline void dmabuf_obj_put(struct intel_vgpu_dmabuf_obj *obj) +{ + kref_put(&obj->kref, dmabuf_gem_object_free); +} + +static void vgpu_gem_release(struct drm_i915_gem_object *gem_obj) +{ + + struct intel_vgpu_fb_info *fb_info = gem_obj->gvt_info; + struct intel_vgpu_dmabuf_obj *obj = fb_info->obj; + struct intel_vgpu *vgpu = obj->vgpu; + + if (vgpu) { + mutex_lock(&vgpu->dmabuf_lock); + gem_obj->base.dma_buf = NULL; + dmabuf_obj_put(obj); + mutex_unlock(&vgpu->dmabuf_lock); + } else { + /* vgpu is NULL, as it has been removed already */ + gem_obj->base.dma_buf = NULL; + dmabuf_obj_put(obj); + } +} + +static const struct drm_i915_gem_object_ops intel_vgpu_gem_ops = { + .flags = I915_GEM_OBJECT_IS_PROXY, + .get_pages = vgpu_gem_get_pages, + .put_pages = vgpu_gem_put_pages, + .release = vgpu_gem_release, +}; + +static struct drm_i915_gem_object *vgpu_create_gem(struct drm_device *dev, + struct intel_vgpu_fb_info *info) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_gem_object *obj; + + obj = i915_gem_object_alloc(dev_priv); + if (obj == NULL) + return NULL; + + drm_gem_private_object_init(dev, &obj->base, + info->size << PAGE_SHIFT); + i915_gem_object_init(obj, &intel_vgpu_gem_ops); + + obj->base.read_domains = I915_GEM_DOMAIN_GTT; + obj->base.write_domain = 0; + if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { + unsigned int tiling_mode = 0; + unsigned int stride = 0; + + switch (info->drm_format_mod << 10) { + case PLANE_CTL_TILED_LINEAR: + tiling_mode = I915_TILING_NONE; + break; + case PLANE_CTL_TILED_X: + tiling_mode = I915_TILING_X; + stride = info->stride; + break; + case PLANE_CTL_TILED_Y: + tiling_mode = I915_TILING_Y; + stride = info->stride; + break; + default: + gvt_dbg_core("not supported tiling mode\n"); + } + obj->tiling_and_stride = tiling_mode | stride; + } else { + obj->tiling_and_stride = info->drm_format_mod ? + I915_TILING_X : 0; + } + + return obj; +} + +static int vgpu_get_plane_info(struct drm_device *dev, + struct intel_vgpu *vgpu, + struct intel_vgpu_fb_info *info, + int plane_id) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_vgpu_primary_plane_format p; + struct intel_vgpu_cursor_plane_format c; + int ret; + + if (plane_id == DRM_PLANE_TYPE_PRIMARY) { + ret = intel_vgpu_decode_primary_plane(vgpu, &p); + if (ret) + return ret; + info->start = p.base; + info->start_gpa = p.base_gpa; + info->width = p.width; + info->height = p.height; + info->stride = p.stride; + info->drm_format = p.drm_format; + info->drm_format_mod = p.tiled; + info->size = (((p.stride * p.height * p.bpp) / 8) + + (PAGE_SIZE - 1)) >> PAGE_SHIFT; + } else if (plane_id == DRM_PLANE_TYPE_CURSOR) { + ret = intel_vgpu_decode_cursor_plane(vgpu, &c); + if (ret) + return ret; + info->start = c.base; + info->start_gpa = c.base_gpa; + info->width = c.width; + info->height = c.height; + info->stride = c.width * (c.bpp / 8); + info->drm_format = c.drm_format; + info->drm_format_mod = 0; + info->x_pos = c.x_pos; + info->y_pos = c.y_pos; + + /* The invalid cursor hotspot value is delivered to host + * until we find a way to get the cursor hotspot info of + * guest OS. + */ + info->x_hot = UINT_MAX; + info->y_hot = UINT_MAX; + info->size = (((info->stride * c.height * c.bpp) / 8) + + (PAGE_SIZE - 1)) >> PAGE_SHIFT; + } else { + gvt_vgpu_err("invalid plane id:%d\n", plane_id); + return -EINVAL; + } + + if (info->size == 0) { + gvt_vgpu_err("fb size is zero\n"); + return -EINVAL; + } + + if (info->start & (PAGE_SIZE - 1)) { + gvt_vgpu_err("Not aligned fb address:0x%llx\n", info->start); + return -EFAULT; + } + if (((info->start >> PAGE_SHIFT) + info->size) > + ggtt_total_entries(&dev_priv->ggtt)) { + gvt_vgpu_err("Invalid GTT offset or size\n"); + return -EFAULT; + } + + if (!intel_gvt_ggtt_validate_range(vgpu, info->start, info->size)) { + gvt_vgpu_err("invalid gma addr\n"); + return -EFAULT; + } + + return 0; +} + +static struct intel_vgpu_dmabuf_obj * +pick_dmabuf_by_info(struct intel_vgpu *vgpu, + struct intel_vgpu_fb_info *latest_info) +{ + struct list_head *pos; + struct intel_vgpu_fb_info *fb_info; + struct intel_vgpu_dmabuf_obj *dmabuf_obj = NULL; + struct intel_vgpu_dmabuf_obj *ret = NULL; + + list_for_each(pos, &vgpu->dmabuf_obj_list_head) { + dmabuf_obj = container_of(pos, struct intel_vgpu_dmabuf_obj, + list); + if ((dmabuf_obj == NULL) || + (dmabuf_obj->info == NULL)) + continue; + + fb_info = (struct intel_vgpu_fb_info *)dmabuf_obj->info; + if ((fb_info->start == latest_info->start) && + (fb_info->start_gpa == latest_info->start_gpa) && + (fb_info->size == latest_info->size) && + (fb_info->drm_format_mod == latest_info->drm_format_mod) && + (fb_info->drm_format == latest_info->drm_format) && + (fb_info->width == latest_info->width) && + (fb_info->height == latest_info->height)) { + ret = dmabuf_obj; + break; + } + } + + return ret; +} + +static struct intel_vgpu_dmabuf_obj * +pick_dmabuf_by_num(struct intel_vgpu *vgpu, u32 id) +{ + struct list_head *pos; + struct intel_vgpu_dmabuf_obj *dmabuf_obj = NULL; + struct intel_vgpu_dmabuf_obj *ret = NULL; + + list_for_each(pos, &vgpu->dmabuf_obj_list_head) { + dmabuf_obj = container_of(pos, struct intel_vgpu_dmabuf_obj, + list); + if (!dmabuf_obj) + continue; + + if (dmabuf_obj->dmabuf_id == id) { + ret = dmabuf_obj; + break; + } + } + + return ret; +} + +static void update_fb_info(struct vfio_device_gfx_plane_info *gvt_dmabuf, + struct intel_vgpu_fb_info *fb_info) +{ + gvt_dmabuf->drm_format = fb_info->drm_format; + gvt_dmabuf->width = fb_info->width; + gvt_dmabuf->height = fb_info->height; + gvt_dmabuf->stride = fb_info->stride; + gvt_dmabuf->size = fb_info->size; + gvt_dmabuf->x_pos = fb_info->x_pos; + gvt_dmabuf->y_pos = fb_info->y_pos; + gvt_dmabuf->x_hot = fb_info->x_hot; + gvt_dmabuf->y_hot = fb_info->y_hot; +} + +int intel_vgpu_query_plane(struct intel_vgpu *vgpu, void *args) +{ + struct drm_device *dev = &vgpu->gvt->dev_priv->drm; + struct vfio_device_gfx_plane_info *gfx_plane_info = args; + struct intel_vgpu_dmabuf_obj *dmabuf_obj; + struct intel_vgpu_fb_info fb_info; + int ret = 0; + + if (gfx_plane_info->flags == (VFIO_GFX_PLANE_TYPE_DMABUF | + VFIO_GFX_PLANE_TYPE_PROBE)) + return ret; + else if ((gfx_plane_info->flags & ~VFIO_GFX_PLANE_TYPE_DMABUF) || + (!gfx_plane_info->flags)) + return -EINVAL; + + ret = vgpu_get_plane_info(dev, vgpu, &fb_info, + gfx_plane_info->drm_plane_type); + if (ret != 0) + goto out; + + mutex_lock(&vgpu->dmabuf_lock); + /* If exists, pick up the exposed dmabuf_obj */ + dmabuf_obj = pick_dmabuf_by_info(vgpu, &fb_info); + if (dmabuf_obj) { + update_fb_info(gfx_plane_info, &fb_info); + gfx_plane_info->dmabuf_id = dmabuf_obj->dmabuf_id; + + /* This buffer may be released between query_plane ioctl and + * get_dmabuf ioctl. Add the refcount to make sure it won't + * be released between the two ioctls. + */ + if (!dmabuf_obj->initref) { + dmabuf_obj->initref = true; + dmabuf_obj_get(dmabuf_obj); + } + ret = 0; + gvt_dbg_dpy("vgpu%d: re-use dmabuf_obj ref %d, id %d\n", + vgpu->id, kref_read(&dmabuf_obj->kref), + gfx_plane_info->dmabuf_id); + mutex_unlock(&vgpu->dmabuf_lock); + goto out; + } + + mutex_unlock(&vgpu->dmabuf_lock); + + /* Need to allocate a new one*/ + dmabuf_obj = kmalloc(sizeof(struct intel_vgpu_dmabuf_obj), GFP_KERNEL); + if (unlikely(!dmabuf_obj)) { + gvt_vgpu_err("alloc dmabuf_obj failed\n"); + ret = -ENOMEM; + goto out; + } + + dmabuf_obj->info = kmalloc(sizeof(struct intel_vgpu_fb_info), + GFP_KERNEL); + if (unlikely(!dmabuf_obj->info)) { + gvt_vgpu_err("allocate intel vgpu fb info failed\n"); + ret = -ENOMEM; + goto out_free_dmabuf; + } + memcpy(dmabuf_obj->info, &fb_info, sizeof(struct intel_vgpu_fb_info)); + + ((struct intel_vgpu_fb_info *)dmabuf_obj->info)->obj = dmabuf_obj; + + dmabuf_obj->vgpu = vgpu; + + ret = idr_alloc(&vgpu->object_idr, dmabuf_obj, 1, 0, GFP_NOWAIT); + if (ret < 0) + goto out_free_info; + gfx_plane_info->dmabuf_id = ret; + dmabuf_obj->dmabuf_id = ret; + + dmabuf_obj->initref = true; + + kref_init(&dmabuf_obj->kref); + + mutex_lock(&vgpu->dmabuf_lock); + if (intel_gvt_hypervisor_get_vfio_device(vgpu)) { + gvt_vgpu_err("get vfio device failed\n"); + mutex_unlock(&vgpu->dmabuf_lock); + goto out_free_info; + } + mutex_unlock(&vgpu->dmabuf_lock); + + update_fb_info(gfx_plane_info, &fb_info); + + INIT_LIST_HEAD(&dmabuf_obj->list); + mutex_lock(&vgpu->dmabuf_lock); + list_add_tail(&dmabuf_obj->list, &vgpu->dmabuf_obj_list_head); + mutex_unlock(&vgpu->dmabuf_lock); + + gvt_dbg_dpy("vgpu%d: %s new dmabuf_obj ref %d, id %d\n", vgpu->id, + __func__, kref_read(&dmabuf_obj->kref), ret); + + return 0; + +out_free_info: + kfree(dmabuf_obj->info); +out_free_dmabuf: + kfree(dmabuf_obj); +out: + /* ENODEV means plane isn't ready, which might be a normal case. */ + return (ret == -ENODEV) ? 0 : ret; +} + +/* To associate an exposed dmabuf with the dmabuf_obj */ +int intel_vgpu_get_dmabuf(struct intel_vgpu *vgpu, unsigned int dmabuf_id) +{ + struct drm_device *dev = &vgpu->gvt->dev_priv->drm; + struct intel_vgpu_dmabuf_obj *dmabuf_obj; + struct drm_i915_gem_object *obj; + struct dma_buf *dmabuf; + int dmabuf_fd; + int ret = 0; + + mutex_lock(&vgpu->dmabuf_lock); + + dmabuf_obj = pick_dmabuf_by_num(vgpu, dmabuf_id); + if (dmabuf_obj == NULL) { + gvt_vgpu_err("invalid dmabuf id:%d\n", dmabuf_id); + ret = -EINVAL; + goto out; + } + + obj = vgpu_create_gem(dev, dmabuf_obj->info); + if (obj == NULL) { + gvt_vgpu_err("create gvt gem obj failed:%d\n", vgpu->id); + ret = -ENOMEM; + goto out; + } + + obj->gvt_info = dmabuf_obj->info; + + dmabuf = i915_gem_prime_export(dev, &obj->base, DRM_CLOEXEC | DRM_RDWR); + if (IS_ERR(dmabuf)) { + gvt_vgpu_err("export dma-buf failed\n"); + ret = PTR_ERR(dmabuf); + goto out_free_gem; + } + + i915_gem_object_put(obj); + + ret = dma_buf_fd(dmabuf, DRM_CLOEXEC | DRM_RDWR); + if (ret < 0) { + gvt_vgpu_err("create dma-buf fd failed ret:%d\n", ret); + goto out_free_dmabuf; + } + dmabuf_fd = ret; + + dmabuf_obj_get(dmabuf_obj); + + if (dmabuf_obj->initref) { + dmabuf_obj->initref = false; + dmabuf_obj_put(dmabuf_obj); + } + + mutex_unlock(&vgpu->dmabuf_lock); + + gvt_dbg_dpy("vgpu%d: dmabuf:%d, dmabuf ref %d, fd:%d\n" + " file count: %ld, GEM ref: %d\n", + vgpu->id, dmabuf_obj->dmabuf_id, + kref_read(&dmabuf_obj->kref), + dmabuf_fd, + file_count(dmabuf->file), + kref_read(&obj->base.refcount)); + + return dmabuf_fd; + +out_free_dmabuf: + dma_buf_put(dmabuf); +out_free_gem: + i915_gem_object_put(obj); +out: + mutex_unlock(&vgpu->dmabuf_lock); + return ret; +} + +void intel_vgpu_dmabuf_cleanup(struct intel_vgpu *vgpu) +{ + struct list_head *pos, *n; + struct intel_vgpu_dmabuf_obj *dmabuf_obj; + + mutex_lock(&vgpu->dmabuf_lock); + list_for_each_safe(pos, n, &vgpu->dmabuf_obj_list_head) { + dmabuf_obj = container_of(pos, struct intel_vgpu_dmabuf_obj, + list); + dmabuf_obj->vgpu = NULL; + + idr_remove(&vgpu->object_idr, dmabuf_obj->dmabuf_id); + intel_gvt_hypervisor_put_vfio_device(vgpu); + list_del(pos); + + /* dmabuf_obj might be freed in dmabuf_obj_put */ + if (dmabuf_obj->initref) { + dmabuf_obj->initref = false; + dmabuf_obj_put(dmabuf_obj); + } + + } + mutex_unlock(&vgpu->dmabuf_lock); +} diff --git a/drivers/gpu/drm/i915/gvt/dmabuf.h b/drivers/gpu/drm/i915/gvt/dmabuf.h new file mode 100644 index 000000000000..5f8f03fb1d1b --- /dev/null +++ b/drivers/gpu/drm/i915/gvt/dmabuf.h @@ -0,0 +1,67 @@ +/* + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Authors: + * Zhiyuan Lv <zhiyuan.lv@intel.com> + * + * Contributors: + * Xiaoguang Chen + * Tina Zhang <tina.zhang@intel.com> + */ + +#ifndef _GVT_DMABUF_H_ +#define _GVT_DMABUF_H_ +#include <linux/vfio.h> + +struct intel_vgpu_fb_info { + __u64 start; + __u64 start_gpa; + __u64 drm_format_mod; + __u32 drm_format; /* drm format of plane */ + __u32 width; /* width of plane */ + __u32 height; /* height of plane */ + __u32 stride; /* stride of plane */ + __u32 size; /* size of plane in bytes, align on page */ + __u32 x_pos; /* horizontal position of cursor plane */ + __u32 y_pos; /* vertical position of cursor plane */ + __u32 x_hot; /* horizontal position of cursor hotspot */ + __u32 y_hot; /* vertical position of cursor hotspot */ + struct intel_vgpu_dmabuf_obj *obj; +}; + +/** + * struct intel_vgpu_dmabuf_obj- Intel vGPU device buffer object + */ +struct intel_vgpu_dmabuf_obj { + struct intel_vgpu *vgpu; + struct intel_vgpu_fb_info *info; + __u32 dmabuf_id; + struct kref kref; + bool initref; + struct list_head list; +}; + +int intel_vgpu_query_plane(struct intel_vgpu *vgpu, void *args); +int intel_vgpu_get_dmabuf(struct intel_vgpu *vgpu, unsigned int dmabuf_id); +void intel_vgpu_dmabuf_cleanup(struct intel_vgpu *vgpu); + +#endif diff --git a/drivers/gpu/drm/i915/gvt/edid.c b/drivers/gpu/drm/i915/gvt/edid.c index 42cd09ec63fa..f61337632969 100644 --- a/drivers/gpu/drm/i915/gvt/edid.c +++ b/drivers/gpu/drm/i915/gvt/edid.c @@ -95,9 +95,9 @@ static inline int get_port_from_gmbus0(u32 gmbus0) static void reset_gmbus_controller(struct intel_vgpu *vgpu) { - vgpu_vreg(vgpu, PCH_GMBUS2) = GMBUS_HW_RDY; + vgpu_vreg_t(vgpu, PCH_GMBUS2) = GMBUS_HW_RDY; if (!vgpu->display.i2c_edid.edid_available) - vgpu_vreg(vgpu, PCH_GMBUS2) |= GMBUS_SATOER; + vgpu_vreg_t(vgpu, PCH_GMBUS2) |= GMBUS_SATOER; vgpu->display.i2c_edid.gmbus.phase = GMBUS_IDLE_PHASE; } @@ -123,16 +123,16 @@ static int gmbus0_mmio_write(struct intel_vgpu *vgpu, vgpu->display.i2c_edid.state = I2C_GMBUS; vgpu->display.i2c_edid.gmbus.phase = GMBUS_IDLE_PHASE; - vgpu_vreg(vgpu, PCH_GMBUS2) &= ~GMBUS_ACTIVE; - vgpu_vreg(vgpu, PCH_GMBUS2) |= GMBUS_HW_RDY | GMBUS_HW_WAIT_PHASE; + vgpu_vreg_t(vgpu, PCH_GMBUS2) &= ~GMBUS_ACTIVE; + vgpu_vreg_t(vgpu, PCH_GMBUS2) |= GMBUS_HW_RDY | GMBUS_HW_WAIT_PHASE; if (intel_vgpu_has_monitor_on_port(vgpu, port) && !intel_vgpu_port_is_dp(vgpu, port)) { vgpu->display.i2c_edid.port = port; vgpu->display.i2c_edid.edid_available = true; - vgpu_vreg(vgpu, PCH_GMBUS2) &= ~GMBUS_SATOER; + vgpu_vreg_t(vgpu, PCH_GMBUS2) &= ~GMBUS_SATOER; } else - vgpu_vreg(vgpu, PCH_GMBUS2) |= GMBUS_SATOER; + vgpu_vreg_t(vgpu, PCH_GMBUS2) |= GMBUS_SATOER; return 0; } @@ -159,8 +159,8 @@ static int gmbus1_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, * 2) HW_RDY bit asserted */ if (wvalue & GMBUS_SW_CLR_INT) { - vgpu_vreg(vgpu, PCH_GMBUS2) &= ~GMBUS_INT; - vgpu_vreg(vgpu, PCH_GMBUS2) |= GMBUS_HW_RDY; + vgpu_vreg_t(vgpu, PCH_GMBUS2) &= ~GMBUS_INT; + vgpu_vreg_t(vgpu, PCH_GMBUS2) |= GMBUS_HW_RDY; } /* For virtualization, we suppose that HW is always ready, @@ -208,7 +208,7 @@ static int gmbus1_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, * visible in gmbus interface) */ i2c_edid->gmbus.phase = GMBUS_IDLE_PHASE; - vgpu_vreg(vgpu, PCH_GMBUS2) &= ~GMBUS_ACTIVE; + vgpu_vreg_t(vgpu, PCH_GMBUS2) &= ~GMBUS_ACTIVE; } break; case NIDX_NS_W: @@ -220,7 +220,7 @@ static int gmbus1_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, * START (-->INDEX) -->DATA */ i2c_edid->gmbus.phase = GMBUS_DATA_PHASE; - vgpu_vreg(vgpu, PCH_GMBUS2) |= GMBUS_ACTIVE; + vgpu_vreg_t(vgpu, PCH_GMBUS2) |= GMBUS_ACTIVE; break; default: gvt_vgpu_err("Unknown/reserved GMBUS cycle detected!\n"); @@ -256,7 +256,7 @@ static int gmbus3_mmio_read(struct intel_vgpu *vgpu, unsigned int offset, u32 reg_data = 0; /* Data can only be recevied if previous settings correct */ - if (vgpu_vreg(vgpu, PCH_GMBUS1) & GMBUS_SLAVE_READ) { + if (vgpu_vreg_t(vgpu, PCH_GMBUS1) & GMBUS_SLAVE_READ) { if (byte_left <= 0) { memcpy(p_data, &vgpu_vreg(vgpu, offset), bytes); return 0; diff --git a/drivers/gpu/drm/i915/gvt/execlist.c b/drivers/gpu/drm/i915/gvt/execlist.c index 940cdaaa3f24..70494e394d2c 100644 --- a/drivers/gpu/drm/i915/gvt/execlist.c +++ b/drivers/gpu/drm/i915/gvt/execlist.c @@ -46,8 +46,6 @@ #define same_context(a, b) (((a)->context_id == (b)->context_id) && \ ((a)->lrca == (b)->lrca)) -static void clean_workloads(struct intel_vgpu *vgpu, unsigned long engine_mask); - static int context_switch_events[] = { [RCS] = RCS_AS_CONTEXT_SWITCH, [BCS] = BCS_AS_CONTEXT_SWITCH, @@ -135,6 +133,8 @@ static void emulate_csb_update(struct intel_vgpu_execlist *execlist, struct execlist_context_status_pointer_format ctx_status_ptr; u32 write_pointer; u32 ctx_status_ptr_reg, ctx_status_buf_reg, offset; + unsigned long hwsp_gpa; + struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; ctx_status_ptr_reg = execlist_ring_mmio(vgpu->gvt, ring_id, _EL_OFFSET_STATUS_PTR); @@ -160,6 +160,20 @@ static void emulate_csb_update(struct intel_vgpu_execlist *execlist, ctx_status_ptr.write_ptr = write_pointer; vgpu_vreg(vgpu, ctx_status_ptr_reg) = ctx_status_ptr.dw; + /* Update the CSB and CSB write pointer in HWSP */ + hwsp_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, + vgpu->hws_pga[ring_id]); + if (hwsp_gpa != INTEL_GVT_INVALID_ADDR) { + intel_gvt_hypervisor_write_gpa(vgpu, + hwsp_gpa + I915_HWS_CSB_BUF0_INDEX * 4 + + write_pointer * 8, + status, 8); + intel_gvt_hypervisor_write_gpa(vgpu, + hwsp_gpa + + intel_hws_csb_write_index(dev_priv) * 4, + &write_pointer, 4); + } + gvt_dbg_el("vgpu%d: w pointer %u reg %x csb l %x csb h %x\n", vgpu->id, write_pointer, offset, status->ldw, status->udw); @@ -358,218 +372,47 @@ static int emulate_execlist_schedule_in(struct intel_vgpu_execlist *execlist, return 0; } -static void free_workload(struct intel_vgpu_workload *workload) -{ - intel_vgpu_unpin_mm(workload->shadow_mm); - intel_gvt_mm_unreference(workload->shadow_mm); - kmem_cache_free(workload->vgpu->workloads, workload); -} - #define get_desc_from_elsp_dwords(ed, i) \ ((struct execlist_ctx_descriptor_format *)&((ed)->data[i * 2])) -static int prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload) -{ - const int gmadr_bytes = workload->vgpu->gvt->device_info.gmadr_bytes_in_cmd; - struct intel_shadow_bb_entry *entry_obj; - - /* pin the gem object to ggtt */ - list_for_each_entry(entry_obj, &workload->shadow_bb, list) { - struct i915_vma *vma; - - vma = i915_gem_object_ggtt_pin(entry_obj->obj, NULL, 0, 4, 0); - if (IS_ERR(vma)) { - return PTR_ERR(vma); - } - - /* FIXME: we are not tracking our pinned VMA leaving it - * up to the core to fix up the stray pin_count upon - * free. - */ - - /* update the relocate gma with shadow batch buffer*/ - entry_obj->bb_start_cmd_va[1] = i915_ggtt_offset(vma); - if (gmadr_bytes == 8) - entry_obj->bb_start_cmd_va[2] = 0; - } - return 0; -} - -static int update_wa_ctx_2_shadow_ctx(struct intel_shadow_wa_ctx *wa_ctx) -{ - struct intel_vgpu_workload *workload = container_of(wa_ctx, - struct intel_vgpu_workload, - wa_ctx); - int ring_id = workload->ring_id; - struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx; - struct drm_i915_gem_object *ctx_obj = - shadow_ctx->engine[ring_id].state->obj; - struct execlist_ring_context *shadow_ring_context; - struct page *page; - - page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN); - shadow_ring_context = kmap_atomic(page); - - shadow_ring_context->bb_per_ctx_ptr.val = - (shadow_ring_context->bb_per_ctx_ptr.val & - (~PER_CTX_ADDR_MASK)) | wa_ctx->per_ctx.shadow_gma; - shadow_ring_context->rcs_indirect_ctx.val = - (shadow_ring_context->rcs_indirect_ctx.val & - (~INDIRECT_CTX_ADDR_MASK)) | wa_ctx->indirect_ctx.shadow_gma; - - kunmap_atomic(shadow_ring_context); - return 0; -} - -static int prepare_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) -{ - struct i915_vma *vma; - unsigned char *per_ctx_va = - (unsigned char *)wa_ctx->indirect_ctx.shadow_va + - wa_ctx->indirect_ctx.size; - - if (wa_ctx->indirect_ctx.size == 0) - return 0; - - vma = i915_gem_object_ggtt_pin(wa_ctx->indirect_ctx.obj, NULL, - 0, CACHELINE_BYTES, 0); - if (IS_ERR(vma)) { - return PTR_ERR(vma); - } - - /* FIXME: we are not tracking our pinned VMA leaving it - * up to the core to fix up the stray pin_count upon - * free. - */ - - wa_ctx->indirect_ctx.shadow_gma = i915_ggtt_offset(vma); - - wa_ctx->per_ctx.shadow_gma = *((unsigned int *)per_ctx_va + 1); - memset(per_ctx_va, 0, CACHELINE_BYTES); - - update_wa_ctx_2_shadow_ctx(wa_ctx); - return 0; -} - -static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload) -{ - /* release all the shadow batch buffer */ - if (!list_empty(&workload->shadow_bb)) { - struct intel_shadow_bb_entry *entry_obj = - list_first_entry(&workload->shadow_bb, - struct intel_shadow_bb_entry, - list); - struct intel_shadow_bb_entry *temp; - - list_for_each_entry_safe(entry_obj, temp, &workload->shadow_bb, - list) { - i915_gem_object_unpin_map(entry_obj->obj); - i915_gem_object_put(entry_obj->obj); - list_del(&entry_obj->list); - kfree(entry_obj); - } - } -} - static int prepare_execlist_workload(struct intel_vgpu_workload *workload) { struct intel_vgpu *vgpu = workload->vgpu; + struct intel_vgpu_submission *s = &vgpu->submission; struct execlist_ctx_descriptor_format ctx[2]; int ring_id = workload->ring_id; int ret; - ret = intel_vgpu_pin_mm(workload->shadow_mm); - if (ret) { - gvt_vgpu_err("fail to vgpu pin mm\n"); - goto out; - } - - ret = intel_vgpu_sync_oos_pages(workload->vgpu); - if (ret) { - gvt_vgpu_err("fail to vgpu sync oos pages\n"); - goto err_unpin_mm; - } - - ret = intel_vgpu_flush_post_shadow(workload->vgpu); - if (ret) { - gvt_vgpu_err("fail to flush post shadow\n"); - goto err_unpin_mm; - } - - ret = intel_gvt_generate_request(workload); - if (ret) { - gvt_vgpu_err("fail to generate request\n"); - goto err_unpin_mm; - } - - ret = prepare_shadow_batch_buffer(workload); - if (ret) { - gvt_vgpu_err("fail to prepare_shadow_batch_buffer\n"); - goto err_unpin_mm; - } - - ret = prepare_shadow_wa_ctx(&workload->wa_ctx); - if (ret) { - gvt_vgpu_err("fail to prepare_shadow_wa_ctx\n"); - goto err_shadow_batch; - } - if (!workload->emulate_schedule_in) return 0; - ctx[0] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 1); - ctx[1] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 0); + ctx[0] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 0); + ctx[1] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 1); - ret = emulate_execlist_schedule_in(&vgpu->execlist[ring_id], ctx); - if (!ret) - goto out; - else + ret = emulate_execlist_schedule_in(&s->execlist[ring_id], ctx); + if (ret) { gvt_vgpu_err("fail to emulate execlist schedule in\n"); - - release_shadow_wa_ctx(&workload->wa_ctx); -err_shadow_batch: - release_shadow_batch_buffer(workload); -err_unpin_mm: - intel_vgpu_unpin_mm(workload->shadow_mm); -out: - return ret; + return ret; + } + return 0; } static int complete_execlist_workload(struct intel_vgpu_workload *workload) { struct intel_vgpu *vgpu = workload->vgpu; int ring_id = workload->ring_id; - struct intel_vgpu_execlist *execlist = &vgpu->execlist[ring_id]; + struct intel_vgpu_submission *s = &vgpu->submission; + struct intel_vgpu_execlist *execlist = &s->execlist[ring_id]; struct intel_vgpu_workload *next_workload; struct list_head *next = workload_q_head(vgpu, ring_id)->next; bool lite_restore = false; - int ret; + int ret = 0; gvt_dbg_el("complete workload %p status %d\n", workload, workload->status); - if (!workload->status) { - release_shadow_batch_buffer(workload); - release_shadow_wa_ctx(&workload->wa_ctx); - } - - if (workload->status || (vgpu->resetting_eng & ENGINE_MASK(ring_id))) { - /* if workload->status is not successful means HW GPU - * has occurred GPU hang or something wrong with i915/GVT, - * and GVT won't inject context switch interrupt to guest. - * So this error is a vGPU hang actually to the guest. - * According to this we should emunlate a vGPU hang. If - * there are pending workloads which are already submitted - * from guest, we should clean them up like HW GPU does. - * - * if it is in middle of engine resetting, the pending - * workloads won't be submitted to HW GPU and will be - * cleaned up during the resetting process later, so doing - * the workload clean up here doesn't have any impact. - **/ - clean_workloads(vgpu, ENGINE_MASK(ring_id)); + if (workload->status || (vgpu->resetting_eng & ENGINE_MASK(ring_id))) goto out; - } if (!list_empty(workload_q_head(vgpu, ring_id))) { struct execlist_ctx_descriptor_format *this_desc, *next_desc; @@ -584,213 +427,60 @@ static int complete_execlist_workload(struct intel_vgpu_workload *workload) if (lite_restore) { gvt_dbg_el("next context == current - no schedule-out\n"); - free_workload(workload); - return 0; + goto out; } ret = emulate_execlist_ctx_schedule_out(execlist, &workload->ctx_desc); - if (ret) - goto err; out: - free_workload(workload); - return 0; -err: - free_workload(workload); + intel_vgpu_unpin_mm(workload->shadow_mm); + intel_vgpu_destroy_workload(workload); return ret; } -#define RING_CTX_OFF(x) \ - offsetof(struct execlist_ring_context, x) - -static void read_guest_pdps(struct intel_vgpu *vgpu, - u64 ring_context_gpa, u32 pdp[8]) -{ - u64 gpa; - int i; - - gpa = ring_context_gpa + RING_CTX_OFF(pdp3_UDW.val); - - for (i = 0; i < 8; i++) - intel_gvt_hypervisor_read_gpa(vgpu, - gpa + i * 8, &pdp[7 - i], 4); -} - -static int prepare_mm(struct intel_vgpu_workload *workload) -{ - struct execlist_ctx_descriptor_format *desc = &workload->ctx_desc; - struct intel_vgpu_mm *mm; - struct intel_vgpu *vgpu = workload->vgpu; - int page_table_level; - u32 pdp[8]; - - if (desc->addressing_mode == 1) { /* legacy 32-bit */ - page_table_level = 3; - } else if (desc->addressing_mode == 3) { /* legacy 64 bit */ - page_table_level = 4; - } else { - gvt_vgpu_err("Advanced Context mode(SVM) is not supported!\n"); - return -EINVAL; - } - - read_guest_pdps(workload->vgpu, workload->ring_context_gpa, pdp); - - mm = intel_vgpu_find_ppgtt_mm(workload->vgpu, page_table_level, pdp); - if (mm) { - intel_gvt_mm_reference(mm); - } else { - - mm = intel_vgpu_create_mm(workload->vgpu, INTEL_GVT_MM_PPGTT, - pdp, page_table_level, 0); - if (IS_ERR(mm)) { - gvt_vgpu_err("fail to create mm object.\n"); - return PTR_ERR(mm); - } - } - workload->shadow_mm = mm; - return 0; -} - -#define get_last_workload(q) \ - (list_empty(q) ? NULL : container_of(q->prev, \ - struct intel_vgpu_workload, list)) - static int submit_context(struct intel_vgpu *vgpu, int ring_id, struct execlist_ctx_descriptor_format *desc, bool emulate_schedule_in) { - struct list_head *q = workload_q_head(vgpu, ring_id); - struct intel_vgpu_workload *last_workload = get_last_workload(q); + struct intel_vgpu_submission *s = &vgpu->submission; struct intel_vgpu_workload *workload = NULL; - struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; - u64 ring_context_gpa; - u32 head, tail, start, ctl, ctx_ctl, per_ctx, indirect_ctx; - int ret; - - ring_context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, - (u32)((desc->lrca + 1) << GTT_PAGE_SHIFT)); - if (ring_context_gpa == INTEL_GVT_INVALID_ADDR) { - gvt_vgpu_err("invalid guest context LRCA: %x\n", desc->lrca); - return -EINVAL; - } - intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + - RING_CTX_OFF(ring_header.val), &head, 4); + workload = intel_vgpu_create_workload(vgpu, ring_id, desc); + if (IS_ERR(workload)) + return PTR_ERR(workload); - intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + - RING_CTX_OFF(ring_tail.val), &tail, 4); - - head &= RB_HEAD_OFF_MASK; - tail &= RB_TAIL_OFF_MASK; - - if (last_workload && same_context(&last_workload->ctx_desc, desc)) { - gvt_dbg_el("ring id %d cur workload == last\n", ring_id); - gvt_dbg_el("ctx head %x real head %lx\n", head, - last_workload->rb_tail); - /* - * cannot use guest context head pointer here, - * as it might not be updated at this time - */ - head = last_workload->rb_tail; - } - - gvt_dbg_el("ring id %d begin a new workload\n", ring_id); - - workload = kmem_cache_zalloc(vgpu->workloads, GFP_KERNEL); - if (!workload) - return -ENOMEM; - - /* record some ring buffer register values for scan and shadow */ - intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + - RING_CTX_OFF(rb_start.val), &start, 4); - intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + - RING_CTX_OFF(rb_ctrl.val), &ctl, 4); - intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + - RING_CTX_OFF(ctx_ctrl.val), &ctx_ctl, 4); - - INIT_LIST_HEAD(&workload->list); - INIT_LIST_HEAD(&workload->shadow_bb); - - init_waitqueue_head(&workload->shadow_ctx_status_wq); - atomic_set(&workload->shadow_ctx_active, 0); - - workload->vgpu = vgpu; - workload->ring_id = ring_id; - workload->ctx_desc = *desc; - workload->ring_context_gpa = ring_context_gpa; - workload->rb_head = head; - workload->rb_tail = tail; - workload->rb_start = start; - workload->rb_ctl = ctl; workload->prepare = prepare_execlist_workload; workload->complete = complete_execlist_workload; - workload->status = -EINPROGRESS; workload->emulate_schedule_in = emulate_schedule_in; - workload->shadowed = false; - - if (ring_id == RCS) { - intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + - RING_CTX_OFF(bb_per_ctx_ptr.val), &per_ctx, 4); - intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + - RING_CTX_OFF(rcs_indirect_ctx.val), &indirect_ctx, 4); - - workload->wa_ctx.indirect_ctx.guest_gma = - indirect_ctx & INDIRECT_CTX_ADDR_MASK; - workload->wa_ctx.indirect_ctx.size = - (indirect_ctx & INDIRECT_CTX_SIZE_MASK) * - CACHELINE_BYTES; - workload->wa_ctx.per_ctx.guest_gma = - per_ctx & PER_CTX_ADDR_MASK; - workload->wa_ctx.per_ctx.valid = per_ctx & 1; - } if (emulate_schedule_in) - workload->elsp_dwords = vgpu->execlist[ring_id].elsp_dwords; - - gvt_dbg_el("workload %p ring id %d head %x tail %x start %x ctl %x\n", - workload, ring_id, head, tail, start, ctl); + workload->elsp_dwords = s->execlist[ring_id].elsp_dwords; gvt_dbg_el("workload %p emulate schedule_in %d\n", workload, emulate_schedule_in); - ret = prepare_mm(workload); - if (ret) { - kmem_cache_free(vgpu->workloads, workload); - return ret; - } - - /* Only scan and shadow the first workload in the queue - * as there is only one pre-allocated buf-obj for shadow. - */ - if (list_empty(workload_q_head(vgpu, ring_id))) { - intel_runtime_pm_get(dev_priv); - mutex_lock(&dev_priv->drm.struct_mutex); - intel_gvt_scan_and_shadow_workload(workload); - mutex_unlock(&dev_priv->drm.struct_mutex); - intel_runtime_pm_put(dev_priv); - } - - queue_workload(workload); + intel_vgpu_queue_workload(workload); return 0; } int intel_vgpu_submit_execlist(struct intel_vgpu *vgpu, int ring_id) { - struct intel_vgpu_execlist *execlist = &vgpu->execlist[ring_id]; - struct execlist_ctx_descriptor_format desc[2]; + struct intel_vgpu_submission *s = &vgpu->submission; + struct intel_vgpu_execlist *execlist = &s->execlist[ring_id]; + struct execlist_ctx_descriptor_format *desc[2]; int i, ret; - desc[0] = *get_desc_from_elsp_dwords(&execlist->elsp_dwords, 1); - desc[1] = *get_desc_from_elsp_dwords(&execlist->elsp_dwords, 0); + desc[0] = get_desc_from_elsp_dwords(&execlist->elsp_dwords, 0); + desc[1] = get_desc_from_elsp_dwords(&execlist->elsp_dwords, 1); - if (!desc[0].valid) { + if (!desc[0]->valid) { gvt_vgpu_err("invalid elsp submission, desc0 is invalid\n"); goto inv_desc; } for (i = 0; i < ARRAY_SIZE(desc); i++) { - if (!desc[i].valid) + if (!desc[i]->valid) continue; - if (!desc[i].privilege_access) { + if (!desc[i]->privilege_access) { gvt_vgpu_err("unexpected GGTT elsp submission\n"); goto inv_desc; } @@ -798,9 +488,9 @@ int intel_vgpu_submit_execlist(struct intel_vgpu *vgpu, int ring_id) /* submit workload */ for (i = 0; i < ARRAY_SIZE(desc); i++) { - if (!desc[i].valid) + if (!desc[i]->valid) continue; - ret = submit_context(vgpu, ring_id, &desc[i], i == 0); + ret = submit_context(vgpu, ring_id, desc[i], i == 0); if (ret) { gvt_vgpu_err("failed to submit desc %d\n", i); return ret; @@ -811,13 +501,14 @@ int intel_vgpu_submit_execlist(struct intel_vgpu *vgpu, int ring_id) inv_desc: gvt_vgpu_err("descriptors content: desc0 %08x %08x desc1 %08x %08x\n", - desc[0].udw, desc[0].ldw, desc[1].udw, desc[1].ldw); + desc[0]->udw, desc[0]->ldw, desc[1]->udw, desc[1]->ldw); return -EINVAL; } static void init_vgpu_execlist(struct intel_vgpu *vgpu, int ring_id) { - struct intel_vgpu_execlist *execlist = &vgpu->execlist[ring_id]; + struct intel_vgpu_submission *s = &vgpu->submission; + struct intel_vgpu_execlist *execlist = &s->execlist[ring_id]; struct execlist_context_status_pointer_format ctx_status_ptr; u32 ctx_status_ptr_reg; @@ -830,98 +521,47 @@ static void init_vgpu_execlist(struct intel_vgpu *vgpu, int ring_id) ctx_status_ptr_reg = execlist_ring_mmio(vgpu->gvt, ring_id, _EL_OFFSET_STATUS_PTR); - ctx_status_ptr.dw = vgpu_vreg(vgpu, ctx_status_ptr_reg); ctx_status_ptr.read_ptr = 0; ctx_status_ptr.write_ptr = 0x7; vgpu_vreg(vgpu, ctx_status_ptr_reg) = ctx_status_ptr.dw; } -static void clean_workloads(struct intel_vgpu *vgpu, unsigned long engine_mask) +static void clean_execlist(struct intel_vgpu *vgpu, unsigned long engine_mask) { + unsigned int tmp; struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; struct intel_engine_cs *engine; - struct intel_vgpu_workload *pos, *n; - unsigned int tmp; + struct intel_vgpu_submission *s = &vgpu->submission; - /* free the unsubmited workloads in the queues. */ for_each_engine_masked(engine, dev_priv, engine_mask, tmp) { - list_for_each_entry_safe(pos, n, - &vgpu->workload_q_head[engine->id], list) { - list_del_init(&pos->list); - free_workload(pos); - } - - clear_bit(engine->id, vgpu->shadow_ctx_desc_updated); - } -} - -void intel_vgpu_clean_execlist(struct intel_vgpu *vgpu) -{ - enum intel_engine_id i; - struct intel_engine_cs *engine; - - clean_workloads(vgpu, ALL_ENGINES); - kmem_cache_destroy(vgpu->workloads); - - for_each_engine(engine, vgpu->gvt->dev_priv, i) { - kfree(vgpu->reserve_ring_buffer_va[i]); - vgpu->reserve_ring_buffer_va[i] = NULL; - vgpu->reserve_ring_buffer_size[i] = 0; - } - -} - -#define RESERVE_RING_BUFFER_SIZE ((1 * PAGE_SIZE)/8) -int intel_vgpu_init_execlist(struct intel_vgpu *vgpu) -{ - enum intel_engine_id i; - struct intel_engine_cs *engine; - - /* each ring has a virtual execlist engine */ - for_each_engine(engine, vgpu->gvt->dev_priv, i) { - init_vgpu_execlist(vgpu, i); - INIT_LIST_HEAD(&vgpu->workload_q_head[i]); + kfree(s->ring_scan_buffer[engine->id]); + s->ring_scan_buffer[engine->id] = NULL; + s->ring_scan_buffer_size[engine->id] = 0; } - - vgpu->workloads = kmem_cache_create("gvt-g_vgpu_workload", - sizeof(struct intel_vgpu_workload), 0, - SLAB_HWCACHE_ALIGN, - NULL); - - if (!vgpu->workloads) - return -ENOMEM; - - /* each ring has a shadow ring buffer until vgpu destroyed */ - for_each_engine(engine, vgpu->gvt->dev_priv, i) { - vgpu->reserve_ring_buffer_va[i] = - kmalloc(RESERVE_RING_BUFFER_SIZE, GFP_KERNEL); - if (!vgpu->reserve_ring_buffer_va[i]) { - gvt_vgpu_err("fail to alloc reserve ring buffer\n"); - goto out; - } - vgpu->reserve_ring_buffer_size[i] = RESERVE_RING_BUFFER_SIZE; - } - return 0; -out: - for_each_engine(engine, vgpu->gvt->dev_priv, i) { - if (vgpu->reserve_ring_buffer_size[i]) { - kfree(vgpu->reserve_ring_buffer_va[i]); - vgpu->reserve_ring_buffer_va[i] = NULL; - vgpu->reserve_ring_buffer_size[i] = 0; - } - } - return -ENOMEM; } -void intel_vgpu_reset_execlist(struct intel_vgpu *vgpu, +static void reset_execlist(struct intel_vgpu *vgpu, unsigned long engine_mask) { struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; struct intel_engine_cs *engine; unsigned int tmp; - clean_workloads(vgpu, engine_mask); for_each_engine_masked(engine, dev_priv, engine_mask, tmp) init_vgpu_execlist(vgpu, engine->id); } + +static int init_execlist(struct intel_vgpu *vgpu, + unsigned long engine_mask) +{ + reset_execlist(vgpu, engine_mask); + return 0; +} + +const struct intel_vgpu_submission_ops intel_vgpu_execlist_submission_ops = { + .name = "execlist", + .init = init_execlist, + .reset = reset_execlist, + .clean = clean_execlist, +}; diff --git a/drivers/gpu/drm/i915/gvt/execlist.h b/drivers/gpu/drm/i915/gvt/execlist.h index 7eced40a1e30..427e40e64d41 100644 --- a/drivers/gpu/drm/i915/gvt/execlist.h +++ b/drivers/gpu/drm/i915/gvt/execlist.h @@ -37,10 +37,6 @@ struct execlist_ctx_descriptor_format { union { - u32 udw; - u32 context_id; - }; - union { u32 ldw; struct { u32 valid : 1; @@ -54,6 +50,10 @@ struct execlist_ctx_descriptor_format { u32 lrca : 20; }; }; + union { + u32 udw; + u32 context_id; + }; }; struct execlist_status_format { diff --git a/drivers/gpu/drm/i915/gvt/fb_decoder.c b/drivers/gpu/drm/i915/gvt/fb_decoder.c new file mode 100644 index 000000000000..6b50fe78dc1b --- /dev/null +++ b/drivers/gpu/drm/i915/gvt/fb_decoder.c @@ -0,0 +1,514 @@ +/* + * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Authors: + * Kevin Tian <kevin.tian@intel.com> + * + * Contributors: + * Bing Niu <bing.niu@intel.com> + * Xu Han <xu.han@intel.com> + * Ping Gao <ping.a.gao@intel.com> + * Xiaoguang Chen <xiaoguang.chen@intel.com> + * Yang Liu <yang2.liu@intel.com> + * Tina Zhang <tina.zhang@intel.com> + * + */ + +#include <uapi/drm/drm_fourcc.h> +#include "i915_drv.h" +#include "gvt.h" + +#define PRIMARY_FORMAT_NUM 16 +struct pixel_format { + int drm_format; /* Pixel format in DRM definition */ + int bpp; /* Bits per pixel, 0 indicates invalid */ + char *desc; /* The description */ +}; + +static struct pixel_format bdw_pixel_formats[] = { + {DRM_FORMAT_C8, 8, "8-bit Indexed"}, + {DRM_FORMAT_RGB565, 16, "16-bit BGRX (5:6:5 MSB-R:G:B)"}, + {DRM_FORMAT_XRGB8888, 32, "32-bit BGRX (8:8:8:8 MSB-X:R:G:B)"}, + {DRM_FORMAT_XBGR2101010, 32, "32-bit RGBX (2:10:10:10 MSB-X:B:G:R)"}, + + {DRM_FORMAT_XRGB2101010, 32, "32-bit BGRX (2:10:10:10 MSB-X:R:G:B)"}, + {DRM_FORMAT_XBGR8888, 32, "32-bit RGBX (8:8:8:8 MSB-X:B:G:R)"}, + + /* non-supported format has bpp default to 0 */ + {0, 0, NULL}, +}; + +static struct pixel_format skl_pixel_formats[] = { + {DRM_FORMAT_YUYV, 16, "16-bit packed YUYV (8:8:8:8 MSB-V:Y2:U:Y1)"}, + {DRM_FORMAT_UYVY, 16, "16-bit packed UYVY (8:8:8:8 MSB-Y2:V:Y1:U)"}, + {DRM_FORMAT_YVYU, 16, "16-bit packed YVYU (8:8:8:8 MSB-U:Y2:V:Y1)"}, + {DRM_FORMAT_VYUY, 16, "16-bit packed VYUY (8:8:8:8 MSB-Y2:U:Y1:V)"}, + + {DRM_FORMAT_C8, 8, "8-bit Indexed"}, + {DRM_FORMAT_RGB565, 16, "16-bit BGRX (5:6:5 MSB-R:G:B)"}, + {DRM_FORMAT_ABGR8888, 32, "32-bit RGBA (8:8:8:8 MSB-A:B:G:R)"}, + {DRM_FORMAT_XBGR8888, 32, "32-bit RGBX (8:8:8:8 MSB-X:B:G:R)"}, + + {DRM_FORMAT_ARGB8888, 32, "32-bit BGRA (8:8:8:8 MSB-A:R:G:B)"}, + {DRM_FORMAT_XRGB8888, 32, "32-bit BGRX (8:8:8:8 MSB-X:R:G:B)"}, + {DRM_FORMAT_XBGR2101010, 32, "32-bit RGBX (2:10:10:10 MSB-X:B:G:R)"}, + {DRM_FORMAT_XRGB2101010, 32, "32-bit BGRX (2:10:10:10 MSB-X:R:G:B)"}, + + /* non-supported format has bpp default to 0 */ + {0, 0, NULL}, +}; + +static int bdw_format_to_drm(int format) +{ + int bdw_pixel_formats_index = 6; + + switch (format) { + case DISPPLANE_8BPP: + bdw_pixel_formats_index = 0; + break; + case DISPPLANE_BGRX565: + bdw_pixel_formats_index = 1; + break; + case DISPPLANE_BGRX888: + bdw_pixel_formats_index = 2; + break; + case DISPPLANE_RGBX101010: + bdw_pixel_formats_index = 3; + break; + case DISPPLANE_BGRX101010: + bdw_pixel_formats_index = 4; + break; + case DISPPLANE_RGBX888: + bdw_pixel_formats_index = 5; + break; + + default: + break; + } + + return bdw_pixel_formats_index; +} + +static int skl_format_to_drm(int format, bool rgb_order, bool alpha, + int yuv_order) +{ + int skl_pixel_formats_index = 12; + + switch (format) { + case PLANE_CTL_FORMAT_INDEXED: + skl_pixel_formats_index = 4; + break; + case PLANE_CTL_FORMAT_RGB_565: + skl_pixel_formats_index = 5; + break; + case PLANE_CTL_FORMAT_XRGB_8888: + if (rgb_order) + skl_pixel_formats_index = alpha ? 6 : 7; + else + skl_pixel_formats_index = alpha ? 8 : 9; + break; + case PLANE_CTL_FORMAT_XRGB_2101010: + skl_pixel_formats_index = rgb_order ? 10 : 11; + break; + case PLANE_CTL_FORMAT_YUV422: + skl_pixel_formats_index = yuv_order >> 16; + if (skl_pixel_formats_index > 3) + return -EINVAL; + break; + + default: + break; + } + + return skl_pixel_formats_index; +} + +static u32 intel_vgpu_get_stride(struct intel_vgpu *vgpu, int pipe, + u32 tiled, int stride_mask, int bpp) +{ + struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; + + u32 stride_reg = vgpu_vreg_t(vgpu, DSPSTRIDE(pipe)) & stride_mask; + u32 stride = stride_reg; + + if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { + switch (tiled) { + case PLANE_CTL_TILED_LINEAR: + stride = stride_reg * 64; + break; + case PLANE_CTL_TILED_X: + stride = stride_reg * 512; + break; + case PLANE_CTL_TILED_Y: + stride = stride_reg * 128; + break; + case PLANE_CTL_TILED_YF: + if (bpp == 8) + stride = stride_reg * 64; + else if (bpp == 16 || bpp == 32 || bpp == 64) + stride = stride_reg * 128; + else + gvt_dbg_core("skl: unsupported bpp:%d\n", bpp); + break; + default: + gvt_dbg_core("skl: unsupported tile format:%x\n", + tiled); + } + } + + return stride; +} + +static int get_active_pipe(struct intel_vgpu *vgpu) +{ + int i; + + for (i = 0; i < I915_MAX_PIPES; i++) + if (pipe_is_enabled(vgpu, i)) + break; + + return i; +} + +/** + * intel_vgpu_decode_primary_plane - Decode primary plane + * @vgpu: input vgpu + * @plane: primary plane to save decoded info + * This function is called for decoding plane + * + * Returns: + * 0 on success, non-zero if failed. + */ +int intel_vgpu_decode_primary_plane(struct intel_vgpu *vgpu, + struct intel_vgpu_primary_plane_format *plane) +{ + u32 val, fmt; + struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; + int pipe; + + pipe = get_active_pipe(vgpu); + if (pipe >= I915_MAX_PIPES) + return -ENODEV; + + val = vgpu_vreg_t(vgpu, DSPCNTR(pipe)); + plane->enabled = !!(val & DISPLAY_PLANE_ENABLE); + if (!plane->enabled) + return -ENODEV; + + if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { + plane->tiled = (val & PLANE_CTL_TILED_MASK) >> + _PLANE_CTL_TILED_SHIFT; + fmt = skl_format_to_drm( + val & PLANE_CTL_FORMAT_MASK, + val & PLANE_CTL_ORDER_RGBX, + val & PLANE_CTL_ALPHA_MASK, + val & PLANE_CTL_YUV422_ORDER_MASK); + + if (fmt >= ARRAY_SIZE(skl_pixel_formats)) { + gvt_vgpu_err("Out-of-bounds pixel format index\n"); + return -EINVAL; + } + + plane->bpp = skl_pixel_formats[fmt].bpp; + plane->drm_format = skl_pixel_formats[fmt].drm_format; + } else { + plane->tiled = !!(val & DISPPLANE_TILED); + fmt = bdw_format_to_drm(val & DISPPLANE_PIXFORMAT_MASK); + plane->bpp = bdw_pixel_formats[fmt].bpp; + plane->drm_format = bdw_pixel_formats[fmt].drm_format; + } + + if (!plane->bpp) { + gvt_vgpu_err("Non-supported pixel format (0x%x)\n", fmt); + return -EINVAL; + } + + plane->hw_format = fmt; + + plane->base = vgpu_vreg_t(vgpu, DSPSURF(pipe)) & I915_GTT_PAGE_MASK; + if (!intel_gvt_ggtt_validate_range(vgpu, plane->base, 0)) { + gvt_vgpu_err("invalid gma address: %lx\n", + (unsigned long)plane->base); + return -EINVAL; + } + + plane->base_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, plane->base); + if (plane->base_gpa == INTEL_GVT_INVALID_ADDR) { + gvt_vgpu_err("invalid gma address: %lx\n", + (unsigned long)plane->base); + return -EINVAL; + } + + plane->stride = intel_vgpu_get_stride(vgpu, pipe, (plane->tiled << 10), + (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) ? + (_PRI_PLANE_STRIDE_MASK >> 6) : + _PRI_PLANE_STRIDE_MASK, plane->bpp); + + plane->width = (vgpu_vreg_t(vgpu, PIPESRC(pipe)) & _PIPE_H_SRCSZ_MASK) >> + _PIPE_H_SRCSZ_SHIFT; + plane->width += 1; + plane->height = (vgpu_vreg_t(vgpu, PIPESRC(pipe)) & + _PIPE_V_SRCSZ_MASK) >> _PIPE_V_SRCSZ_SHIFT; + plane->height += 1; /* raw height is one minus the real value */ + + val = vgpu_vreg_t(vgpu, DSPTILEOFF(pipe)); + plane->x_offset = (val & _PRI_PLANE_X_OFF_MASK) >> + _PRI_PLANE_X_OFF_SHIFT; + plane->y_offset = (val & _PRI_PLANE_Y_OFF_MASK) >> + _PRI_PLANE_Y_OFF_SHIFT; + + return 0; +} + +#define CURSOR_FORMAT_NUM (1 << 6) +struct cursor_mode_format { + int drm_format; /* Pixel format in DRM definition */ + u8 bpp; /* Bits per pixel; 0 indicates invalid */ + u32 width; /* In pixel */ + u32 height; /* In lines */ + char *desc; /* The description */ +}; + +static struct cursor_mode_format cursor_pixel_formats[] = { + {DRM_FORMAT_ARGB8888, 32, 128, 128, "128x128 32bpp ARGB"}, + {DRM_FORMAT_ARGB8888, 32, 256, 256, "256x256 32bpp ARGB"}, + {DRM_FORMAT_ARGB8888, 32, 64, 64, "64x64 32bpp ARGB"}, + {DRM_FORMAT_ARGB8888, 32, 64, 64, "64x64 32bpp ARGB"}, + + /* non-supported format has bpp default to 0 */ + {0, 0, 0, 0, NULL}, +}; + +static int cursor_mode_to_drm(int mode) +{ + int cursor_pixel_formats_index = 4; + + switch (mode) { + case CURSOR_MODE_128_ARGB_AX: + cursor_pixel_formats_index = 0; + break; + case CURSOR_MODE_256_ARGB_AX: + cursor_pixel_formats_index = 1; + break; + case CURSOR_MODE_64_ARGB_AX: + cursor_pixel_formats_index = 2; + break; + case CURSOR_MODE_64_32B_AX: + cursor_pixel_formats_index = 3; + break; + + default: + break; + } + + return cursor_pixel_formats_index; +} + +/** + * intel_vgpu_decode_cursor_plane - Decode sprite plane + * @vgpu: input vgpu + * @plane: cursor plane to save decoded info + * This function is called for decoding plane + * + * Returns: + * 0 on success, non-zero if failed. + */ +int intel_vgpu_decode_cursor_plane(struct intel_vgpu *vgpu, + struct intel_vgpu_cursor_plane_format *plane) +{ + u32 val, mode, index; + u32 alpha_plane, alpha_force; + struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; + int pipe; + + pipe = get_active_pipe(vgpu); + if (pipe >= I915_MAX_PIPES) + return -ENODEV; + + val = vgpu_vreg_t(vgpu, CURCNTR(pipe)); + mode = val & CURSOR_MODE; + plane->enabled = (mode != CURSOR_MODE_DISABLE); + if (!plane->enabled) + return -ENODEV; + + index = cursor_mode_to_drm(mode); + + if (!cursor_pixel_formats[index].bpp) { + gvt_vgpu_err("Non-supported cursor mode (0x%x)\n", mode); + return -EINVAL; + } + plane->mode = mode; + plane->bpp = cursor_pixel_formats[index].bpp; + plane->drm_format = cursor_pixel_formats[index].drm_format; + plane->width = cursor_pixel_formats[index].width; + plane->height = cursor_pixel_formats[index].height; + + alpha_plane = (val & _CURSOR_ALPHA_PLANE_MASK) >> + _CURSOR_ALPHA_PLANE_SHIFT; + alpha_force = (val & _CURSOR_ALPHA_FORCE_MASK) >> + _CURSOR_ALPHA_FORCE_SHIFT; + if (alpha_plane || alpha_force) + gvt_dbg_core("alpha_plane=0x%x, alpha_force=0x%x\n", + alpha_plane, alpha_force); + + plane->base = vgpu_vreg_t(vgpu, CURBASE(pipe)) & I915_GTT_PAGE_MASK; + if (!intel_gvt_ggtt_validate_range(vgpu, plane->base, 0)) { + gvt_vgpu_err("invalid gma address: %lx\n", + (unsigned long)plane->base); + return -EINVAL; + } + + plane->base_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, plane->base); + if (plane->base_gpa == INTEL_GVT_INVALID_ADDR) { + gvt_vgpu_err("invalid gma address: %lx\n", + (unsigned long)plane->base); + return -EINVAL; + } + + val = vgpu_vreg_t(vgpu, CURPOS(pipe)); + plane->x_pos = (val & _CURSOR_POS_X_MASK) >> _CURSOR_POS_X_SHIFT; + plane->x_sign = (val & _CURSOR_SIGN_X_MASK) >> _CURSOR_SIGN_X_SHIFT; + plane->y_pos = (val & _CURSOR_POS_Y_MASK) >> _CURSOR_POS_Y_SHIFT; + plane->y_sign = (val & _CURSOR_SIGN_Y_MASK) >> _CURSOR_SIGN_Y_SHIFT; + + return 0; +} + +#define SPRITE_FORMAT_NUM (1 << 3) + +static struct pixel_format sprite_pixel_formats[SPRITE_FORMAT_NUM] = { + [0x0] = {DRM_FORMAT_YUV422, 16, "YUV 16-bit 4:2:2 packed"}, + [0x1] = {DRM_FORMAT_XRGB2101010, 32, "RGB 32-bit 2:10:10:10"}, + [0x2] = {DRM_FORMAT_XRGB8888, 32, "RGB 32-bit 8:8:8:8"}, + [0x4] = {DRM_FORMAT_AYUV, 32, + "YUV 32-bit 4:4:4 packed (8:8:8:8 MSB-X:Y:U:V)"}, +}; + +/** + * intel_vgpu_decode_sprite_plane - Decode sprite plane + * @vgpu: input vgpu + * @plane: sprite plane to save decoded info + * This function is called for decoding plane + * + * Returns: + * 0 on success, non-zero if failed. + */ +int intel_vgpu_decode_sprite_plane(struct intel_vgpu *vgpu, + struct intel_vgpu_sprite_plane_format *plane) +{ + u32 val, fmt; + u32 color_order, yuv_order; + int drm_format; + int pipe; + + pipe = get_active_pipe(vgpu); + if (pipe >= I915_MAX_PIPES) + return -ENODEV; + + val = vgpu_vreg_t(vgpu, SPRCTL(pipe)); + plane->enabled = !!(val & SPRITE_ENABLE); + if (!plane->enabled) + return -ENODEV; + + plane->tiled = !!(val & SPRITE_TILED); + color_order = !!(val & SPRITE_RGB_ORDER_RGBX); + yuv_order = (val & SPRITE_YUV_BYTE_ORDER_MASK) >> + _SPRITE_YUV_ORDER_SHIFT; + + fmt = (val & SPRITE_PIXFORMAT_MASK) >> _SPRITE_FMT_SHIFT; + if (!sprite_pixel_formats[fmt].bpp) { + gvt_vgpu_err("Non-supported pixel format (0x%x)\n", fmt); + return -EINVAL; + } + plane->hw_format = fmt; + plane->bpp = sprite_pixel_formats[fmt].bpp; + drm_format = sprite_pixel_formats[fmt].drm_format; + + /* Order of RGB values in an RGBxxx buffer may be ordered RGB or + * BGR depending on the state of the color_order field + */ + if (!color_order) { + if (drm_format == DRM_FORMAT_XRGB2101010) + drm_format = DRM_FORMAT_XBGR2101010; + else if (drm_format == DRM_FORMAT_XRGB8888) + drm_format = DRM_FORMAT_XBGR8888; + } + + if (drm_format == DRM_FORMAT_YUV422) { + switch (yuv_order) { + case 0: + drm_format = DRM_FORMAT_YUYV; + break; + case 1: + drm_format = DRM_FORMAT_UYVY; + break; + case 2: + drm_format = DRM_FORMAT_YVYU; + break; + case 3: + drm_format = DRM_FORMAT_VYUY; + break; + default: + /* yuv_order has only 2 bits */ + break; + } + } + + plane->drm_format = drm_format; + + plane->base = vgpu_vreg_t(vgpu, SPRSURF(pipe)) & I915_GTT_PAGE_MASK; + if (!intel_gvt_ggtt_validate_range(vgpu, plane->base, 0)) { + gvt_vgpu_err("invalid gma address: %lx\n", + (unsigned long)plane->base); + return -EINVAL; + } + + plane->base_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, plane->base); + if (plane->base_gpa == INTEL_GVT_INVALID_ADDR) { + gvt_vgpu_err("invalid gma address: %lx\n", + (unsigned long)plane->base); + return -EINVAL; + } + + plane->stride = vgpu_vreg_t(vgpu, SPRSTRIDE(pipe)) & + _SPRITE_STRIDE_MASK; + + val = vgpu_vreg_t(vgpu, SPRSIZE(pipe)); + plane->height = (val & _SPRITE_SIZE_HEIGHT_MASK) >> + _SPRITE_SIZE_HEIGHT_SHIFT; + plane->width = (val & _SPRITE_SIZE_WIDTH_MASK) >> + _SPRITE_SIZE_WIDTH_SHIFT; + plane->height += 1; /* raw height is one minus the real value */ + plane->width += 1; /* raw width is one minus the real value */ + + val = vgpu_vreg_t(vgpu, SPRPOS(pipe)); + plane->x_pos = (val & _SPRITE_POS_X_MASK) >> _SPRITE_POS_X_SHIFT; + plane->y_pos = (val & _SPRITE_POS_Y_MASK) >> _SPRITE_POS_Y_SHIFT; + + val = vgpu_vreg_t(vgpu, SPROFFSET(pipe)); + plane->x_offset = (val & _SPRITE_OFFSET_START_X_MASK) >> + _SPRITE_OFFSET_START_X_SHIFT; + plane->y_offset = (val & _SPRITE_OFFSET_START_Y_MASK) >> + _SPRITE_OFFSET_START_Y_SHIFT; + + return 0; +} diff --git a/drivers/gpu/drm/i915/gvt/fb_decoder.h b/drivers/gpu/drm/i915/gvt/fb_decoder.h new file mode 100644 index 000000000000..cb055f3c81a2 --- /dev/null +++ b/drivers/gpu/drm/i915/gvt/fb_decoder.h @@ -0,0 +1,169 @@ +/* + * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Authors: + * Kevin Tian <kevin.tian@intel.com> + * + * Contributors: + * Bing Niu <bing.niu@intel.com> + * Xu Han <xu.han@intel.com> + * Ping Gao <ping.a.gao@intel.com> + * Xiaoguang Chen <xiaoguang.chen@intel.com> + * Yang Liu <yang2.liu@intel.com> + * Tina Zhang <tina.zhang@intel.com> + * + */ + +#ifndef _GVT_FB_DECODER_H_ +#define _GVT_FB_DECODER_H_ + +#define _PLANE_CTL_FORMAT_SHIFT 24 +#define _PLANE_CTL_TILED_SHIFT 10 +#define _PIPE_V_SRCSZ_SHIFT 0 +#define _PIPE_V_SRCSZ_MASK (0xfff << _PIPE_V_SRCSZ_SHIFT) +#define _PIPE_H_SRCSZ_SHIFT 16 +#define _PIPE_H_SRCSZ_MASK (0x1fff << _PIPE_H_SRCSZ_SHIFT) + +#define _PRI_PLANE_FMT_SHIFT 26 +#define _PRI_PLANE_STRIDE_MASK (0x3ff << 6) +#define _PRI_PLANE_X_OFF_SHIFT 0 +#define _PRI_PLANE_X_OFF_MASK (0x1fff << _PRI_PLANE_X_OFF_SHIFT) +#define _PRI_PLANE_Y_OFF_SHIFT 16 +#define _PRI_PLANE_Y_OFF_MASK (0xfff << _PRI_PLANE_Y_OFF_SHIFT) + +#define _CURSOR_MODE 0x3f +#define _CURSOR_ALPHA_FORCE_SHIFT 8 +#define _CURSOR_ALPHA_FORCE_MASK (0x3 << _CURSOR_ALPHA_FORCE_SHIFT) +#define _CURSOR_ALPHA_PLANE_SHIFT 10 +#define _CURSOR_ALPHA_PLANE_MASK (0x3 << _CURSOR_ALPHA_PLANE_SHIFT) +#define _CURSOR_POS_X_SHIFT 0 +#define _CURSOR_POS_X_MASK (0x1fff << _CURSOR_POS_X_SHIFT) +#define _CURSOR_SIGN_X_SHIFT 15 +#define _CURSOR_SIGN_X_MASK (1 << _CURSOR_SIGN_X_SHIFT) +#define _CURSOR_POS_Y_SHIFT 16 +#define _CURSOR_POS_Y_MASK (0xfff << _CURSOR_POS_Y_SHIFT) +#define _CURSOR_SIGN_Y_SHIFT 31 +#define _CURSOR_SIGN_Y_MASK (1 << _CURSOR_SIGN_Y_SHIFT) + +#define _SPRITE_FMT_SHIFT 25 +#define _SPRITE_COLOR_ORDER_SHIFT 20 +#define _SPRITE_YUV_ORDER_SHIFT 16 +#define _SPRITE_STRIDE_SHIFT 6 +#define _SPRITE_STRIDE_MASK (0x1ff << _SPRITE_STRIDE_SHIFT) +#define _SPRITE_SIZE_WIDTH_SHIFT 0 +#define _SPRITE_SIZE_HEIGHT_SHIFT 16 +#define _SPRITE_SIZE_WIDTH_MASK (0x1fff << _SPRITE_SIZE_WIDTH_SHIFT) +#define _SPRITE_SIZE_HEIGHT_MASK (0xfff << _SPRITE_SIZE_HEIGHT_SHIFT) +#define _SPRITE_POS_X_SHIFT 0 +#define _SPRITE_POS_Y_SHIFT 16 +#define _SPRITE_POS_X_MASK (0x1fff << _SPRITE_POS_X_SHIFT) +#define _SPRITE_POS_Y_MASK (0xfff << _SPRITE_POS_Y_SHIFT) +#define _SPRITE_OFFSET_START_X_SHIFT 0 +#define _SPRITE_OFFSET_START_Y_SHIFT 16 +#define _SPRITE_OFFSET_START_X_MASK (0x1fff << _SPRITE_OFFSET_START_X_SHIFT) +#define _SPRITE_OFFSET_START_Y_MASK (0xfff << _SPRITE_OFFSET_START_Y_SHIFT) + +enum GVT_FB_EVENT { + FB_MODE_SET_START = 1, + FB_MODE_SET_END, + FB_DISPLAY_FLIP, +}; + +enum DDI_PORT { + DDI_PORT_NONE = 0, + DDI_PORT_B = 1, + DDI_PORT_C = 2, + DDI_PORT_D = 3, + DDI_PORT_E = 4 +}; + +struct intel_gvt; + +/* color space conversion and gamma correction are not included */ +struct intel_vgpu_primary_plane_format { + u8 enabled; /* plane is enabled */ + u8 tiled; /* X-tiled */ + u8 bpp; /* bits per pixel */ + u32 hw_format; /* format field in the PRI_CTL register */ + u32 drm_format; /* format in DRM definition */ + u32 base; /* framebuffer base in graphics memory */ + u64 base_gpa; + u32 x_offset; /* in pixels */ + u32 y_offset; /* in lines */ + u32 width; /* in pixels */ + u32 height; /* in lines */ + u32 stride; /* in bytes */ +}; + +struct intel_vgpu_sprite_plane_format { + u8 enabled; /* plane is enabled */ + u8 tiled; /* X-tiled */ + u8 bpp; /* bits per pixel */ + u32 hw_format; /* format field in the SPR_CTL register */ + u32 drm_format; /* format in DRM definition */ + u32 base; /* sprite base in graphics memory */ + u64 base_gpa; + u32 x_pos; /* in pixels */ + u32 y_pos; /* in lines */ + u32 x_offset; /* in pixels */ + u32 y_offset; /* in lines */ + u32 width; /* in pixels */ + u32 height; /* in lines */ + u32 stride; /* in bytes */ +}; + +struct intel_vgpu_cursor_plane_format { + u8 enabled; + u8 mode; /* cursor mode select */ + u8 bpp; /* bits per pixel */ + u32 drm_format; /* format in DRM definition */ + u32 base; /* cursor base in graphics memory */ + u64 base_gpa; + u32 x_pos; /* in pixels */ + u32 y_pos; /* in lines */ + u8 x_sign; /* X Position Sign */ + u8 y_sign; /* Y Position Sign */ + u32 width; /* in pixels */ + u32 height; /* in lines */ + u32 x_hot; /* in pixels */ + u32 y_hot; /* in pixels */ +}; + +struct intel_vgpu_pipe_format { + struct intel_vgpu_primary_plane_format primary; + struct intel_vgpu_sprite_plane_format sprite; + struct intel_vgpu_cursor_plane_format cursor; + enum DDI_PORT ddi_port; /* the DDI port that pipe is connected to */ +}; + +struct intel_vgpu_fb_format { + struct intel_vgpu_pipe_format pipes[I915_MAX_PIPES]; +}; + +int intel_vgpu_decode_primary_plane(struct intel_vgpu *vgpu, + struct intel_vgpu_primary_plane_format *plane); +int intel_vgpu_decode_cursor_plane(struct intel_vgpu *vgpu, + struct intel_vgpu_cursor_plane_format *plane); +int intel_vgpu_decode_sprite_plane(struct intel_vgpu *vgpu, + struct intel_vgpu_sprite_plane_format *plane); + +#endif diff --git a/drivers/gpu/drm/i915/gvt/firmware.c b/drivers/gpu/drm/i915/gvt/firmware.c index a26c1705430e..a73e1d418c22 100644 --- a/drivers/gpu/drm/i915/gvt/firmware.c +++ b/drivers/gpu/drm/i915/gvt/firmware.c @@ -66,20 +66,23 @@ static struct bin_attribute firmware_attr = { .mmap = NULL, }; -static int expose_firmware_sysfs(struct intel_gvt *gvt) +static int mmio_snapshot_handler(struct intel_gvt *gvt, u32 offset, void *data) { struct drm_i915_private *dev_priv = gvt->dev_priv; + + *(u32 *)(data + offset) = I915_READ_NOTRACE(_MMIO(offset)); + return 0; +} + +static int expose_firmware_sysfs(struct intel_gvt *gvt) +{ struct intel_gvt_device_info *info = &gvt->device_info; struct pci_dev *pdev = gvt->dev_priv->drm.pdev; - struct intel_gvt_mmio_info *e; - struct gvt_mmio_block *block = gvt->mmio.mmio_block; - int num = gvt->mmio.num_mmio_block; struct gvt_firmware_header *h; void *firmware; void *p; unsigned long size, crc32_start; - int i, j; - int ret; + int i, ret; size = sizeof(*h) + info->mmio_size + info->cfg_space_size; firmware = vzalloc(size); @@ -104,15 +107,8 @@ static int expose_firmware_sysfs(struct intel_gvt *gvt) p = firmware + h->mmio_offset; - hash_for_each(gvt->mmio.mmio_info_table, i, e, node) - *(u32 *)(p + e->offset) = I915_READ_NOTRACE(_MMIO(e->offset)); - - for (i = 0; i < num; i++, block++) { - for (j = 0; j < block->size; j += 4) - *(u32 *)(p + INTEL_GVT_MMIO_OFFSET(block->offset) + j) = - I915_READ_NOTRACE(_MMIO(INTEL_GVT_MMIO_OFFSET( - block->offset) + j)); - } + /* Take a snapshot of hw mmio registers. */ + intel_gvt_for_each_tracked_mmio(gvt, mmio_snapshot_handler, p); memcpy(gvt->firmware.mmio, p, info->mmio_size); diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c index 64d67ff9bf08..8d5317d0122d 100644 --- a/drivers/gpu/drm/i915/gvt/gtt.c +++ b/drivers/gpu/drm/i915/gvt/gtt.c @@ -94,12 +94,12 @@ int intel_gvt_ggtt_index_g2h(struct intel_vgpu *vgpu, unsigned long g_index, u64 h_addr; int ret; - ret = intel_gvt_ggtt_gmadr_g2h(vgpu, g_index << GTT_PAGE_SHIFT, + ret = intel_gvt_ggtt_gmadr_g2h(vgpu, g_index << I915_GTT_PAGE_SHIFT, &h_addr); if (ret) return ret; - *h_index = h_addr >> GTT_PAGE_SHIFT; + *h_index = h_addr >> I915_GTT_PAGE_SHIFT; return 0; } @@ -109,12 +109,12 @@ int intel_gvt_ggtt_h2g_index(struct intel_vgpu *vgpu, unsigned long h_index, u64 g_addr; int ret; - ret = intel_gvt_ggtt_gmadr_h2g(vgpu, h_index << GTT_PAGE_SHIFT, + ret = intel_gvt_ggtt_gmadr_h2g(vgpu, h_index << I915_GTT_PAGE_SHIFT, &g_addr); if (ret) return ret; - *g_index = g_addr >> GTT_PAGE_SHIFT; + *g_index = g_addr >> I915_GTT_PAGE_SHIFT; return 0; } @@ -156,13 +156,15 @@ int intel_gvt_ggtt_h2g_index(struct intel_vgpu *vgpu, unsigned long h_index, struct gtt_type_table_entry { int entry_type; + int pt_type; int next_pt_type; int pse_entry_type; }; -#define GTT_TYPE_TABLE_ENTRY(type, e_type, npt_type, pse_type) \ +#define GTT_TYPE_TABLE_ENTRY(type, e_type, cpt_type, npt_type, pse_type) \ [type] = { \ .entry_type = e_type, \ + .pt_type = cpt_type, \ .next_pt_type = npt_type, \ .pse_entry_type = pse_type, \ } @@ -170,55 +172,68 @@ struct gtt_type_table_entry { static struct gtt_type_table_entry gtt_type_table[] = { GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_ROOT_L4_ENTRY, GTT_TYPE_PPGTT_ROOT_L4_ENTRY, + GTT_TYPE_INVALID, GTT_TYPE_PPGTT_PML4_PT, GTT_TYPE_INVALID), GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PML4_PT, GTT_TYPE_PPGTT_PML4_ENTRY, + GTT_TYPE_PPGTT_PML4_PT, GTT_TYPE_PPGTT_PDP_PT, GTT_TYPE_INVALID), GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PML4_ENTRY, GTT_TYPE_PPGTT_PML4_ENTRY, + GTT_TYPE_PPGTT_PML4_PT, GTT_TYPE_PPGTT_PDP_PT, GTT_TYPE_INVALID), GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDP_PT, GTT_TYPE_PPGTT_PDP_ENTRY, + GTT_TYPE_PPGTT_PDP_PT, GTT_TYPE_PPGTT_PDE_PT, GTT_TYPE_PPGTT_PTE_1G_ENTRY), GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_ROOT_L3_ENTRY, GTT_TYPE_PPGTT_ROOT_L3_ENTRY, + GTT_TYPE_INVALID, GTT_TYPE_PPGTT_PDE_PT, GTT_TYPE_PPGTT_PTE_1G_ENTRY), GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDP_ENTRY, GTT_TYPE_PPGTT_PDP_ENTRY, + GTT_TYPE_PPGTT_PDP_PT, GTT_TYPE_PPGTT_PDE_PT, GTT_TYPE_PPGTT_PTE_1G_ENTRY), GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDE_PT, GTT_TYPE_PPGTT_PDE_ENTRY, + GTT_TYPE_PPGTT_PDE_PT, GTT_TYPE_PPGTT_PTE_PT, GTT_TYPE_PPGTT_PTE_2M_ENTRY), GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDE_ENTRY, GTT_TYPE_PPGTT_PDE_ENTRY, + GTT_TYPE_PPGTT_PDE_PT, GTT_TYPE_PPGTT_PTE_PT, GTT_TYPE_PPGTT_PTE_2M_ENTRY), GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_PT, GTT_TYPE_PPGTT_PTE_4K_ENTRY, + GTT_TYPE_PPGTT_PTE_PT, GTT_TYPE_INVALID, GTT_TYPE_INVALID), GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_4K_ENTRY, GTT_TYPE_PPGTT_PTE_4K_ENTRY, + GTT_TYPE_PPGTT_PTE_PT, GTT_TYPE_INVALID, GTT_TYPE_INVALID), GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_2M_ENTRY, GTT_TYPE_PPGTT_PDE_ENTRY, + GTT_TYPE_PPGTT_PDE_PT, GTT_TYPE_INVALID, GTT_TYPE_PPGTT_PTE_2M_ENTRY), GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_1G_ENTRY, GTT_TYPE_PPGTT_PDP_ENTRY, + GTT_TYPE_PPGTT_PDP_PT, GTT_TYPE_INVALID, GTT_TYPE_PPGTT_PTE_1G_ENTRY), GTT_TYPE_TABLE_ENTRY(GTT_TYPE_GGTT_PTE, GTT_TYPE_GGTT_PTE, GTT_TYPE_INVALID, + GTT_TYPE_INVALID, GTT_TYPE_INVALID), }; @@ -227,6 +242,11 @@ static inline int get_next_pt_type(int type) return gtt_type_table[type].next_pt_type; } +static inline int get_pt_type(int type) +{ + return gtt_type_table[type].pt_type; +} + static inline int get_entry_type(int type) { return gtt_type_table[type].entry_type; @@ -351,7 +371,7 @@ static bool gen8_gtt_test_pse(struct intel_gvt_gtt_entry *e) return false; e->type = get_entry_type(e->type); - if (!(e->val64 & (1 << 7))) + if (!(e->val64 & BIT(7))) return false; e->type = get_pse_type(e->type); @@ -369,12 +389,17 @@ static bool gen8_gtt_test_present(struct intel_gvt_gtt_entry *e) || e->type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY) return (e->val64 != 0); else - return (e->val64 & (1 << 0)); + return (e->val64 & BIT(0)); } static void gtt_entry_clear_present(struct intel_gvt_gtt_entry *e) { - e->val64 &= ~(1 << 0); + e->val64 &= ~BIT(0); +} + +static void gtt_entry_set_present(struct intel_gvt_gtt_entry *e) +{ + e->val64 |= BIT(0); } /* @@ -382,7 +407,7 @@ static void gtt_entry_clear_present(struct intel_gvt_gtt_entry *e) */ static unsigned long gma_to_ggtt_pte_index(unsigned long gma) { - unsigned long x = (gma >> GTT_PAGE_SHIFT); + unsigned long x = (gma >> I915_GTT_PAGE_SHIFT); trace_gma_index(__func__, gma, x); return x; @@ -406,6 +431,7 @@ static struct intel_gvt_gtt_pte_ops gen8_gtt_pte_ops = { .get_entry = gtt_get_entry64, .set_entry = gtt_set_entry64, .clear_present = gtt_entry_clear_present, + .set_present = gtt_entry_set_present, .test_present = gen8_gtt_test_present, .test_pse = gen8_gtt_test_pse, .get_pfn = gen8_gtt_get_pfn, @@ -494,7 +520,7 @@ static inline int ppgtt_spt_get_entry( return -EINVAL; ret = ops->get_entry(page_table, e, index, guest, - spt->guest_page.gfn << GTT_PAGE_SHIFT, + spt->guest_page.track.gfn << I915_GTT_PAGE_SHIFT, spt->vgpu); if (ret) return ret; @@ -516,7 +542,7 @@ static inline int ppgtt_spt_set_entry( return -EINVAL; return ops->set_entry(page_table, e, index, guest, - spt->guest_page.gfn << GTT_PAGE_SHIFT, + spt->guest_page.track.gfn << I915_GTT_PAGE_SHIFT, spt->vgpu); } @@ -537,88 +563,103 @@ static inline int ppgtt_spt_set_entry( spt->shadow_page.type, e, index, false) /** - * intel_vgpu_init_guest_page - init a guest page data structure + * intel_vgpu_init_page_track - init a page track data structure * @vgpu: a vGPU - * @p: a guest page data structure + * @t: a page track data structure * @gfn: guest memory page frame number - * @handler: function will be called when target guest memory page has + * @handler: the function will be called when target guest memory page has * been modified. * - * This function is called when user wants to track a guest memory page. + * This function is called when a user wants to prepare a page track data + * structure to track a guest memory page. * * Returns: * Zero on success, negative error code if failed. */ -int intel_vgpu_init_guest_page(struct intel_vgpu *vgpu, - struct intel_vgpu_guest_page *p, +int intel_vgpu_init_page_track(struct intel_vgpu *vgpu, + struct intel_vgpu_page_track *t, unsigned long gfn, int (*handler)(void *, u64, void *, int), void *data) { - INIT_HLIST_NODE(&p->node); + INIT_HLIST_NODE(&t->node); - p->writeprotection = false; - p->gfn = gfn; - p->handler = handler; - p->data = data; - p->oos_page = NULL; - p->write_cnt = 0; + t->tracked = false; + t->gfn = gfn; + t->handler = handler; + t->data = data; - hash_add(vgpu->gtt.guest_page_hash_table, &p->node, p->gfn); + hash_add(vgpu->gtt.tracked_guest_page_hash_table, &t->node, t->gfn); return 0; } -static int detach_oos_page(struct intel_vgpu *vgpu, - struct intel_vgpu_oos_page *oos_page); - /** - * intel_vgpu_clean_guest_page - release the resource owned by guest page data - * structure + * intel_vgpu_clean_page_track - release a page track data structure * @vgpu: a vGPU - * @p: a tracked guest page + * @t: a page track data structure * - * This function is called when user tries to stop tracking a guest memory - * page. + * This function is called before a user frees a page track data structure. */ -void intel_vgpu_clean_guest_page(struct intel_vgpu *vgpu, - struct intel_vgpu_guest_page *p) +void intel_vgpu_clean_page_track(struct intel_vgpu *vgpu, + struct intel_vgpu_page_track *t) { - if (!hlist_unhashed(&p->node)) - hash_del(&p->node); + if (!hlist_unhashed(&t->node)) + hash_del(&t->node); - if (p->oos_page) - detach_oos_page(vgpu, p->oos_page); - - if (p->writeprotection) - intel_gvt_hypervisor_unset_wp_page(vgpu, p); + if (t->tracked) + intel_gvt_hypervisor_disable_page_track(vgpu, t); } /** - * intel_vgpu_find_guest_page - find a guest page data structure by GFN. + * intel_vgpu_find_tracked_page - find a tracked guest page * @vgpu: a vGPU * @gfn: guest memory page frame number * - * This function is called when emulation logic wants to know if a trapped GFN - * is a tracked guest page. + * This function is called when the emulation layer wants to figure out if a + * trapped GFN is a tracked guest page. * * Returns: - * Pointer to guest page data structure, NULL if failed. + * Pointer to page track data structure, NULL if not found. */ -struct intel_vgpu_guest_page *intel_vgpu_find_guest_page( +struct intel_vgpu_page_track *intel_vgpu_find_tracked_page( struct intel_vgpu *vgpu, unsigned long gfn) { - struct intel_vgpu_guest_page *p; + struct intel_vgpu_page_track *t; - hash_for_each_possible(vgpu->gtt.guest_page_hash_table, - p, node, gfn) { - if (p->gfn == gfn) - return p; + hash_for_each_possible(vgpu->gtt.tracked_guest_page_hash_table, + t, node, gfn) { + if (t->gfn == gfn) + return t; } return NULL; } +static int init_guest_page(struct intel_vgpu *vgpu, + struct intel_vgpu_guest_page *p, + unsigned long gfn, + int (*handler)(void *, u64, void *, int), + void *data) +{ + p->oos_page = NULL; + p->write_cnt = 0; + + return intel_vgpu_init_page_track(vgpu, &p->track, gfn, handler, data); +} + +static int detach_oos_page(struct intel_vgpu *vgpu, + struct intel_vgpu_oos_page *oos_page); + +static void clean_guest_page(struct intel_vgpu *vgpu, + struct intel_vgpu_guest_page *p) +{ + if (p->oos_page) + detach_oos_page(vgpu, p->oos_page); + + intel_vgpu_clean_page_track(vgpu, &p->track); +} + static inline int init_shadow_page(struct intel_vgpu *vgpu, - struct intel_vgpu_shadow_page *p, int type) + struct intel_vgpu_shadow_page *p, int type, bool hash) { struct device *kdev = &vgpu->gvt->dev_priv->drm.pdev->dev; dma_addr_t daddr; @@ -634,8 +675,9 @@ static inline int init_shadow_page(struct intel_vgpu *vgpu, INIT_HLIST_NODE(&p->node); - p->mfn = daddr >> GTT_PAGE_SHIFT; - hash_add(vgpu->gtt.shadow_page_hash_table, &p->node, p->mfn); + p->mfn = daddr >> I915_GTT_PAGE_SHIFT; + if (hash) + hash_add(vgpu->gtt.shadow_page_hash_table, &p->node, p->mfn); return 0; } @@ -644,7 +686,7 @@ static inline void clean_shadow_page(struct intel_vgpu *vgpu, { struct device *kdev = &vgpu->gvt->dev_priv->drm.pdev->dev; - dma_unmap_page(kdev, p->mfn << GTT_PAGE_SHIFT, 4096, + dma_unmap_page(kdev, p->mfn << I915_GTT_PAGE_SHIFT, 4096, PCI_DMA_BIDIRECTIONAL); if (!hlist_unhashed(&p->node)) @@ -664,6 +706,9 @@ static inline struct intel_vgpu_shadow_page *find_shadow_page( return NULL; } +#define page_track_to_guest_page(ptr) \ + container_of(ptr, struct intel_vgpu_guest_page, track) + #define guest_page_to_ppgtt_spt(ptr) \ container_of(ptr, struct intel_vgpu_ppgtt_spt, guest_page) @@ -697,7 +742,7 @@ static void ppgtt_free_shadow_page(struct intel_vgpu_ppgtt_spt *spt) trace_spt_free(spt->vgpu->id, spt, spt->shadow_page.type); clean_shadow_page(spt->vgpu, &spt->shadow_page); - intel_vgpu_clean_guest_page(spt->vgpu, &spt->guest_page); + clean_guest_page(spt->vgpu, &spt->guest_page); list_del_init(&spt->post_shadow_list); free_spt(spt); @@ -713,22 +758,24 @@ static void ppgtt_free_all_shadow_page(struct intel_vgpu *vgpu) ppgtt_free_shadow_page(shadow_page_to_ppgtt_spt(sp)); } -static int ppgtt_handle_guest_write_page_table_bytes(void *gp, +static int ppgtt_handle_guest_write_page_table_bytes( + struct intel_vgpu_guest_page *gpt, u64 pa, void *p_data, int bytes); -static int ppgtt_write_protection_handler(void *gp, u64 pa, +static int ppgtt_write_protection_handler(void *data, u64 pa, void *p_data, int bytes) { - struct intel_vgpu_guest_page *gpt = (struct intel_vgpu_guest_page *)gp; + struct intel_vgpu_page_track *t = data; + struct intel_vgpu_guest_page *p = page_track_to_guest_page(t); int ret; if (bytes != 4 && bytes != 8) return -EINVAL; - if (!gpt->writeprotection) + if (!t->tracked) return -EINVAL; - ret = ppgtt_handle_guest_write_page_table_bytes(gp, + ret = ppgtt_handle_guest_write_page_table_bytes(p, pa, p_data, bytes); if (ret) return ret; @@ -762,13 +809,13 @@ retry: * TODO: guest page type may be different with shadow page type, * when we support PSE page in future. */ - ret = init_shadow_page(vgpu, &spt->shadow_page, type); + ret = init_shadow_page(vgpu, &spt->shadow_page, type, true); if (ret) { gvt_vgpu_err("fail to initialize shadow page for spt\n"); goto err; } - ret = intel_vgpu_init_guest_page(vgpu, &spt->guest_page, + ret = init_guest_page(vgpu, &spt->guest_page, gfn, ppgtt_write_protection_handler, NULL); if (ret) { gvt_vgpu_err("fail to initialize guest page for spt\n"); @@ -798,7 +845,7 @@ static struct intel_vgpu_ppgtt_spt *ppgtt_find_shadow_page( ((spt)->vgpu->gvt->device_info.gtt_entry_size_shift) #define pt_entries(spt) \ - (GTT_PAGE_SIZE >> pt_entry_size_shift(spt)) + (I915_GTT_PAGE_SIZE >> pt_entry_size_shift(spt)) #define for_each_present_guest_entry(spt, e, i) \ for (i = 0; i < pt_entries(spt); i++) \ @@ -856,7 +903,7 @@ static int ppgtt_invalidate_shadow_page(struct intel_vgpu_ppgtt_spt *spt) int v = atomic_read(&spt->refcount); trace_spt_change(spt->vgpu->id, "die", spt, - spt->guest_page.gfn, spt->shadow_page.type); + spt->guest_page.track.gfn, spt->shadow_page.type); trace_spt_refcount(spt->vgpu->id, "dec", spt, v, (v - 1)); @@ -878,7 +925,7 @@ static int ppgtt_invalidate_shadow_page(struct intel_vgpu_ppgtt_spt *spt) } release: trace_spt_change(spt->vgpu->id, "release", spt, - spt->guest_page.gfn, spt->shadow_page.type); + spt->guest_page.track.gfn, spt->shadow_page.type); ppgtt_free_shadow_page(spt); return 0; fail: @@ -895,6 +942,7 @@ static struct intel_vgpu_ppgtt_spt *ppgtt_populate_shadow_page_by_guest_entry( struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; struct intel_vgpu_ppgtt_spt *s = NULL; struct intel_vgpu_guest_page *g; + struct intel_vgpu_page_track *t; int ret; if (WARN_ON(!gtt_type_is_pt(get_next_pt_type(we->type)))) { @@ -902,8 +950,9 @@ static struct intel_vgpu_ppgtt_spt *ppgtt_populate_shadow_page_by_guest_entry( goto fail; } - g = intel_vgpu_find_guest_page(vgpu, ops->get_pfn(we)); - if (g) { + t = intel_vgpu_find_tracked_page(vgpu, ops->get_pfn(we)); + if (t) { + g = page_track_to_guest_page(t); s = guest_page_to_ppgtt_spt(g); ppgtt_get_shadow_page(s); } else { @@ -915,7 +964,8 @@ static struct intel_vgpu_ppgtt_spt *ppgtt_populate_shadow_page_by_guest_entry( goto fail; } - ret = intel_gvt_hypervisor_set_wp_page(vgpu, &s->guest_page); + ret = intel_gvt_hypervisor_enable_page_track(vgpu, + &s->guest_page.track); if (ret) goto fail; @@ -923,7 +973,7 @@ static struct intel_vgpu_ppgtt_spt *ppgtt_populate_shadow_page_by_guest_entry( if (ret) goto fail; - trace_spt_change(vgpu->id, "new", s, s->guest_page.gfn, + trace_spt_change(vgpu->id, "new", s, s->guest_page.track.gfn, s->shadow_page.type); } return s; @@ -947,19 +997,22 @@ static inline void ppgtt_generate_shadow_entry(struct intel_gvt_gtt_entry *se, static int ppgtt_populate_shadow_page(struct intel_vgpu_ppgtt_spt *spt) { struct intel_vgpu *vgpu = spt->vgpu; + struct intel_gvt *gvt = vgpu->gvt; + struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops; struct intel_vgpu_ppgtt_spt *s; struct intel_gvt_gtt_entry se, ge; - unsigned long i; + unsigned long gfn, i; int ret; trace_spt_change(spt->vgpu->id, "born", spt, - spt->guest_page.gfn, spt->shadow_page.type); + spt->guest_page.track.gfn, spt->shadow_page.type); if (gtt_type_is_pte_pt(spt->shadow_page.type)) { for_each_present_guest_entry(spt, &ge, i) { - ret = gtt_entry_p2m(vgpu, &ge, &se); - if (ret) - goto fail; + gfn = ops->get_pfn(&ge); + if (!intel_gvt_hypervisor_is_valid_gfn(vgpu, gfn) || + gtt_entry_p2m(vgpu, &ge, &se)) + ops->set_pfn(&se, gvt->gtt.scratch_mfn); ppgtt_set_shadow_entry(spt, &se, i); } return 0; @@ -1078,11 +1131,11 @@ static int sync_oos_page(struct intel_vgpu *vgpu, old.type = new.type = get_entry_type(spt->guest_page_type); old.val64 = new.val64 = 0; - for (index = 0; index < (GTT_PAGE_SIZE >> info->gtt_entry_size_shift); - index++) { + for (index = 0; index < (I915_GTT_PAGE_SIZE >> + info->gtt_entry_size_shift); index++) { ops->get_entry(oos_page->mem, &old, index, false, 0, vgpu); ops->get_entry(NULL, &new, index, true, - oos_page->guest_page->gfn << PAGE_SHIFT, vgpu); + oos_page->guest_page->track.gfn << PAGE_SHIFT, vgpu); if (old.val64 == new.val64 && !test_and_clear_bit(index, spt->post_shadow_bitmap)) @@ -1132,8 +1185,9 @@ static int attach_oos_page(struct intel_vgpu *vgpu, struct intel_gvt *gvt = vgpu->gvt; int ret; - ret = intel_gvt_hypervisor_read_gpa(vgpu, gpt->gfn << GTT_PAGE_SHIFT, - oos_page->mem, GTT_PAGE_SIZE); + ret = intel_gvt_hypervisor_read_gpa(vgpu, + gpt->track.gfn << I915_GTT_PAGE_SHIFT, + oos_page->mem, I915_GTT_PAGE_SIZE); if (ret) return ret; @@ -1152,7 +1206,7 @@ static int ppgtt_set_guest_page_sync(struct intel_vgpu *vgpu, { int ret; - ret = intel_gvt_hypervisor_set_wp_page(vgpu, gpt); + ret = intel_gvt_hypervisor_enable_page_track(vgpu, &gpt->track); if (ret) return ret; @@ -1200,7 +1254,7 @@ static int ppgtt_set_guest_page_oos(struct intel_vgpu *vgpu, gpt, guest_page_to_ppgtt_spt(gpt)->guest_page_type); list_add_tail(&oos_page->vm_list, &vgpu->gtt.oos_page_list_head); - return intel_gvt_hypervisor_unset_wp_page(vgpu, gpt); + return intel_gvt_hypervisor_disable_page_track(vgpu, &gpt->track); } /** @@ -1335,10 +1389,10 @@ int intel_vgpu_flush_post_shadow(struct intel_vgpu *vgpu) return 0; } -static int ppgtt_handle_guest_write_page_table_bytes(void *gp, +static int ppgtt_handle_guest_write_page_table_bytes( + struct intel_vgpu_guest_page *gpt, u64 pa, void *p_data, int bytes) { - struct intel_vgpu_guest_page *gpt = (struct intel_vgpu_guest_page *)gp; struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt); struct intel_vgpu *vgpu = spt->vgpu; struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; @@ -1418,7 +1472,7 @@ static int gen8_mm_alloc_page_table(struct intel_vgpu_mm *mm) mm->shadow_page_table = mem + mm->page_table_entry_size; } else if (mm->type == INTEL_GVT_MM_GGTT) { mm->page_table_entry_cnt = - (gvt_ggtt_gm_sz(gvt) >> GTT_PAGE_SHIFT); + (gvt_ggtt_gm_sz(gvt) >> I915_GTT_PAGE_SHIFT); mm->page_table_entry_size = mm->page_table_entry_cnt * info->gtt_entry_size; mem = vzalloc(mm->page_table_entry_size); @@ -1740,8 +1794,8 @@ unsigned long intel_vgpu_gma_to_gpa(struct intel_vgpu_mm *mm, unsigned long gma) gma_ops->gma_to_ggtt_pte_index(gma)); if (ret) goto err; - gpa = (pte_ops->get_pfn(&e) << GTT_PAGE_SHIFT) - + (gma & ~GTT_PAGE_MASK); + gpa = (pte_ops->get_pfn(&e) << I915_GTT_PAGE_SHIFT) + + (gma & ~I915_GTT_PAGE_MASK); trace_gma_translate(vgpu->id, "ggtt", 0, 0, gma, gpa); return gpa; @@ -1793,8 +1847,8 @@ unsigned long intel_vgpu_gma_to_gpa(struct intel_vgpu_mm *mm, unsigned long gma) } } - gpa = (pte_ops->get_pfn(&e) << GTT_PAGE_SHIFT) - + (gma & ~GTT_PAGE_MASK); + gpa = (pte_ops->get_pfn(&e) << I915_GTT_PAGE_SHIFT) + + (gma & ~I915_GTT_PAGE_MASK); trace_gma_translate(vgpu->id, "ppgtt", 0, mm->page_table_level, gma, gpa); @@ -1855,14 +1909,14 @@ static int emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off, struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm; struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops; unsigned long g_gtt_index = off >> info->gtt_entry_size_shift; - unsigned long gma; + unsigned long gma, gfn; struct intel_gvt_gtt_entry e, m; int ret; if (bytes != 4 && bytes != 8) return -EINVAL; - gma = g_gtt_index << GTT_PAGE_SHIFT; + gma = g_gtt_index << I915_GTT_PAGE_SHIFT; /* the VM may configure the whole GM space when ballooning is used */ if (!vgpu_gmadr_is_valid(vgpu, gma)) @@ -1874,6 +1928,16 @@ static int emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off, bytes); if (ops->test_present(&e)) { + gfn = ops->get_pfn(&e); + + /* one PTE update may be issued in multiple writes and the + * first write may not construct a valid gfn + */ + if (!intel_gvt_hypervisor_is_valid_gfn(vgpu, gfn)) { + ops->set_pfn(&m, gvt->gtt.scratch_mfn); + goto out; + } + ret = gtt_entry_p2m(vgpu, &e, &m); if (ret) { gvt_vgpu_err("fail to translate guest gtt entry\n"); @@ -1881,13 +1945,14 @@ static int emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off, * update the entry in this situation p2m will fail * settting the shadow entry to point to a scratch page */ - ops->set_pfn(&m, gvt->gtt.scratch_ggtt_mfn); + ops->set_pfn(&m, gvt->gtt.scratch_mfn); } } else { m = e; - ops->set_pfn(&m, gvt->gtt.scratch_ggtt_mfn); + ops->set_pfn(&m, gvt->gtt.scratch_mfn); } +out: ggtt_set_shadow_entry(ggtt_mm, &m, g_gtt_index); gtt_invalidate(gvt->dev_priv); ggtt_set_guest_entry(ggtt_mm, &e, g_gtt_index); @@ -1920,12 +1985,45 @@ int intel_vgpu_emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off, return ret; } +int intel_vgpu_write_protect_handler(struct intel_vgpu *vgpu, u64 pa, + void *p_data, unsigned int bytes) +{ + struct intel_gvt *gvt = vgpu->gvt; + int ret = 0; + + if (atomic_read(&vgpu->gtt.n_tracked_guest_page)) { + struct intel_vgpu_page_track *t; + + mutex_lock(&gvt->lock); + + t = intel_vgpu_find_tracked_page(vgpu, pa >> PAGE_SHIFT); + if (t) { + if (unlikely(vgpu->failsafe)) { + /* remove write protection to prevent furture traps */ + intel_vgpu_clean_page_track(vgpu, t); + } else { + ret = t->handler(t, pa, p_data, bytes); + if (ret) { + gvt_err("guest page write error %d, " + "gfn 0x%lx, pa 0x%llx, " + "var 0x%x, len %d\n", + ret, t->gfn, pa, + *(u32 *)p_data, bytes); + } + } + } + mutex_unlock(&gvt->lock); + } + return ret; +} + + static int alloc_scratch_pages(struct intel_vgpu *vgpu, intel_gvt_gtt_type_t type) { struct intel_vgpu_gtt *gtt = &vgpu->gtt; struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; - int page_entry_num = GTT_PAGE_SIZE >> + int page_entry_num = I915_GTT_PAGE_SIZE >> vgpu->gvt->device_info.gtt_entry_size_shift; void *scratch_pt; int i; @@ -1949,7 +2047,7 @@ static int alloc_scratch_pages(struct intel_vgpu *vgpu, return -ENOMEM; } gtt->scratch_pt[type].page_mfn = - (unsigned long)(daddr >> GTT_PAGE_SHIFT); + (unsigned long)(daddr >> I915_GTT_PAGE_SHIFT); gtt->scratch_pt[type].page = virt_to_page(scratch_pt); gvt_dbg_mm("vgpu%d create scratch_pt: type %d mfn=0x%lx\n", vgpu->id, type, gtt->scratch_pt[type].page_mfn); @@ -1992,7 +2090,7 @@ static int release_scratch_page_tree(struct intel_vgpu *vgpu) for (i = GTT_TYPE_PPGTT_PTE_PT; i < GTT_TYPE_MAX; i++) { if (vgpu->gtt.scratch_pt[i].page != NULL) { daddr = (dma_addr_t)(vgpu->gtt.scratch_pt[i].page_mfn << - GTT_PAGE_SHIFT); + I915_GTT_PAGE_SHIFT); dma_unmap_page(dev, daddr, 4096, PCI_DMA_BIDIRECTIONAL); __free_page(vgpu->gtt.scratch_pt[i].page); vgpu->gtt.scratch_pt[i].page = NULL; @@ -2035,7 +2133,7 @@ int intel_vgpu_init_gtt(struct intel_vgpu *vgpu) struct intel_vgpu_gtt *gtt = &vgpu->gtt; struct intel_vgpu_mm *ggtt_mm; - hash_init(gtt->guest_page_hash_table); + hash_init(gtt->tracked_guest_page_hash_table); hash_init(gtt->shadow_page_hash_table); INIT_LIST_HEAD(>t->mm_list_head); @@ -2196,7 +2294,7 @@ struct intel_vgpu_mm *intel_vgpu_find_ppgtt_mm(struct intel_vgpu *vgpu, int intel_vgpu_g2v_create_ppgtt_mm(struct intel_vgpu *vgpu, int page_table_level) { - u64 *pdp = (u64 *)&vgpu_vreg64(vgpu, vgtif_reg(pdp[0])); + u64 *pdp = (u64 *)&vgpu_vreg64_t(vgpu, vgtif_reg(pdp[0])); struct intel_vgpu_mm *mm; if (WARN_ON((page_table_level != 4) && (page_table_level != 3))) @@ -2231,7 +2329,7 @@ int intel_vgpu_g2v_create_ppgtt_mm(struct intel_vgpu *vgpu, int intel_vgpu_g2v_destroy_ppgtt_mm(struct intel_vgpu *vgpu, int page_table_level) { - u64 *pdp = (u64 *)&vgpu_vreg64(vgpu, vgtif_reg(pdp[0])); + u64 *pdp = (u64 *)&vgpu_vreg64_t(vgpu, vgtif_reg(pdp[0])); struct intel_vgpu_mm *mm; if (WARN_ON((page_table_level != 4) && (page_table_level != 3))) @@ -2288,15 +2386,16 @@ int intel_gvt_init_gtt(struct intel_gvt *gvt) __free_page(virt_to_page(page)); return -ENOMEM; } - gvt->gtt.scratch_ggtt_page = virt_to_page(page); - gvt->gtt.scratch_ggtt_mfn = (unsigned long)(daddr >> GTT_PAGE_SHIFT); + + gvt->gtt.scratch_page = virt_to_page(page); + gvt->gtt.scratch_mfn = (unsigned long)(daddr >> I915_GTT_PAGE_SHIFT); if (enable_out_of_sync) { ret = setup_spt_oos(gvt); if (ret) { gvt_err("fail to initialize SPT oos\n"); dma_unmap_page(dev, daddr, 4096, PCI_DMA_BIDIRECTIONAL); - __free_page(gvt->gtt.scratch_ggtt_page); + __free_page(gvt->gtt.scratch_page); return ret; } } @@ -2315,12 +2414,12 @@ int intel_gvt_init_gtt(struct intel_gvt *gvt) void intel_gvt_clean_gtt(struct intel_gvt *gvt) { struct device *dev = &gvt->dev_priv->drm.pdev->dev; - dma_addr_t daddr = (dma_addr_t)(gvt->gtt.scratch_ggtt_mfn << - GTT_PAGE_SHIFT); + dma_addr_t daddr = (dma_addr_t)(gvt->gtt.scratch_mfn << + I915_GTT_PAGE_SHIFT); dma_unmap_page(dev, daddr, 4096, PCI_DMA_BIDIRECTIONAL); - __free_page(gvt->gtt.scratch_ggtt_page); + __free_page(gvt->gtt.scratch_page); if (enable_out_of_sync) clean_spt_oos(gvt); @@ -2346,7 +2445,7 @@ void intel_vgpu_reset_ggtt(struct intel_vgpu *vgpu) memset(&e, 0, sizeof(struct intel_gvt_gtt_entry)); e.type = GTT_TYPE_GGTT_PTE; - ops->set_pfn(&e, gvt->gtt.scratch_ggtt_mfn); + ops->set_pfn(&e, gvt->gtt.scratch_mfn); e.val64 |= _PAGE_PRESENT; index = vgpu_aperture_gmadr_base(vgpu) >> PAGE_SHIFT; @@ -2372,8 +2471,6 @@ void intel_vgpu_reset_ggtt(struct intel_vgpu *vgpu) */ void intel_vgpu_reset_gtt(struct intel_vgpu *vgpu) { - int i; - ppgtt_free_all_shadow_page(vgpu); /* Shadow pages are only created when there is no page @@ -2383,11 +2480,4 @@ void intel_vgpu_reset_gtt(struct intel_vgpu *vgpu) intel_vgpu_free_mm(vgpu, INTEL_GVT_MM_PPGTT); intel_vgpu_reset_ggtt(vgpu); - - /* clear scratch page for security */ - for (i = GTT_TYPE_PPGTT_PTE_PT; i < GTT_TYPE_MAX; i++) { - if (vgpu->gtt.scratch_pt[i].page != NULL) - memset(page_address(vgpu->gtt.scratch_pt[i].page), - 0, PAGE_SIZE); - } } diff --git a/drivers/gpu/drm/i915/gvt/gtt.h b/drivers/gpu/drm/i915/gvt/gtt.h index 30a4c8d16026..4cc13b5934f1 100644 --- a/drivers/gpu/drm/i915/gvt/gtt.h +++ b/drivers/gpu/drm/i915/gvt/gtt.h @@ -34,9 +34,8 @@ #ifndef _GVT_GTT_H_ #define _GVT_GTT_H_ -#define GTT_PAGE_SHIFT 12 -#define GTT_PAGE_SIZE (1UL << GTT_PAGE_SHIFT) -#define GTT_PAGE_MASK (~(GTT_PAGE_SIZE-1)) +#define I915_GTT_PAGE_SHIFT 12 +#define I915_GTT_PAGE_MASK (~(I915_GTT_PAGE_SIZE - 1)) struct intel_vgpu_mm; @@ -63,6 +62,7 @@ struct intel_gvt_gtt_pte_ops { struct intel_vgpu *vgpu); bool (*test_present)(struct intel_gvt_gtt_entry *e); void (*clear_present)(struct intel_gvt_gtt_entry *e); + void (*set_present)(struct intel_gvt_gtt_entry *e); bool (*test_pse)(struct intel_gvt_gtt_entry *e); void (*set_pfn)(struct intel_gvt_gtt_entry *e, unsigned long pfn); unsigned long (*get_pfn)(struct intel_gvt_gtt_entry *e); @@ -86,8 +86,8 @@ struct intel_gvt_gtt { struct list_head oos_page_free_list_head; struct list_head mm_lru_list_head; - struct page *scratch_ggtt_page; - unsigned long scratch_ggtt_mfn; + struct page *scratch_page; + unsigned long scratch_mfn; }; enum { @@ -193,18 +193,16 @@ struct intel_vgpu_scratch_pt { unsigned long page_mfn; }; - struct intel_vgpu_gtt { struct intel_vgpu_mm *ggtt_mm; unsigned long active_ppgtt_mm_bitmap; struct list_head mm_list_head; DECLARE_HASHTABLE(shadow_page_hash_table, INTEL_GVT_GTT_HASH_BITS); - DECLARE_HASHTABLE(guest_page_hash_table, INTEL_GVT_GTT_HASH_BITS); - atomic_t n_write_protected_guest_page; + DECLARE_HASHTABLE(tracked_guest_page_hash_table, INTEL_GVT_GTT_HASH_BITS); + atomic_t n_tracked_guest_page; struct list_head oos_page_list_head; struct list_head post_shadow_list_head; struct intel_vgpu_scratch_pt scratch_pt[GTT_TYPE_MAX]; - }; extern int intel_vgpu_init_gtt(struct intel_vgpu *vgpu); @@ -228,12 +226,16 @@ struct intel_vgpu_shadow_page { unsigned long mfn; }; -struct intel_vgpu_guest_page { +struct intel_vgpu_page_track { struct hlist_node node; - bool writeprotection; + bool tracked; unsigned long gfn; int (*handler)(void *, u64, void *, int); void *data; +}; + +struct intel_vgpu_guest_page { + struct intel_vgpu_page_track track; unsigned long write_cnt; struct intel_vgpu_oos_page *oos_page; }; @@ -243,7 +245,7 @@ struct intel_vgpu_oos_page { struct list_head list; struct list_head vm_list; int id; - unsigned char mem[GTT_PAGE_SIZE]; + unsigned char mem[I915_GTT_PAGE_SIZE]; }; #define GTT_ENTRY_NUM_IN_ONE_PAGE 512 @@ -258,22 +260,16 @@ struct intel_vgpu_ppgtt_spt { struct list_head post_shadow_list; }; -int intel_vgpu_init_guest_page(struct intel_vgpu *vgpu, - struct intel_vgpu_guest_page *guest_page, +int intel_vgpu_init_page_track(struct intel_vgpu *vgpu, + struct intel_vgpu_page_track *t, unsigned long gfn, int (*handler)(void *gp, u64, void *, int), void *data); -void intel_vgpu_clean_guest_page(struct intel_vgpu *vgpu, - struct intel_vgpu_guest_page *guest_page); - -int intel_vgpu_set_guest_page_writeprotection(struct intel_vgpu *vgpu, - struct intel_vgpu_guest_page *guest_page); +void intel_vgpu_clean_page_track(struct intel_vgpu *vgpu, + struct intel_vgpu_page_track *t); -void intel_vgpu_clear_guest_page_writeprotection(struct intel_vgpu *vgpu, - struct intel_vgpu_guest_page *guest_page); - -struct intel_vgpu_guest_page *intel_vgpu_find_guest_page( +struct intel_vgpu_page_track *intel_vgpu_find_tracked_page( struct intel_vgpu *vgpu, unsigned long gfn); int intel_vgpu_sync_oos_pages(struct intel_vgpu *vgpu); @@ -312,4 +308,7 @@ int intel_vgpu_emulate_gtt_mmio_read(struct intel_vgpu *vgpu, int intel_vgpu_emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off, void *p_data, unsigned int bytes); +int intel_vgpu_write_protect_handler(struct intel_vgpu *vgpu, u64 pa, + void *p_data, unsigned int bytes); + #endif /* _GVT_GTT_H_ */ diff --git a/drivers/gpu/drm/i915/gvt/gvt.c b/drivers/gpu/drm/i915/gvt/gvt.c index aaa347f8620c..fac54f32d33f 100644 --- a/drivers/gpu/drm/i915/gvt/gvt.c +++ b/drivers/gpu/drm/i915/gvt/gvt.c @@ -36,6 +36,8 @@ #include "i915_drv.h" #include "gvt.h" +#include <linux/vfio.h> +#include <linux/mdev.h> struct intel_gvt_host intel_gvt_host; @@ -44,6 +46,129 @@ static const char * const supported_hypervisors[] = { [INTEL_GVT_HYPERVISOR_KVM] = "KVM", }; +static struct intel_vgpu_type *intel_gvt_find_vgpu_type(struct intel_gvt *gvt, + const char *name) +{ + int i; + struct intel_vgpu_type *t; + const char *driver_name = dev_driver_string( + &gvt->dev_priv->drm.pdev->dev); + + for (i = 0; i < gvt->num_types; i++) { + t = &gvt->types[i]; + if (!strncmp(t->name, name + strlen(driver_name) + 1, + sizeof(t->name))) + return t; + } + + return NULL; +} + +static ssize_t available_instances_show(struct kobject *kobj, + struct device *dev, char *buf) +{ + struct intel_vgpu_type *type; + unsigned int num = 0; + void *gvt = kdev_to_i915(dev)->gvt; + + type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj)); + if (!type) + num = 0; + else + num = type->avail_instance; + + return sprintf(buf, "%u\n", num); +} + +static ssize_t device_api_show(struct kobject *kobj, struct device *dev, + char *buf) +{ + return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING); +} + +static ssize_t description_show(struct kobject *kobj, struct device *dev, + char *buf) +{ + struct intel_vgpu_type *type; + void *gvt = kdev_to_i915(dev)->gvt; + + type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj)); + if (!type) + return 0; + + return sprintf(buf, "low_gm_size: %dMB\nhigh_gm_size: %dMB\n" + "fence: %d\nresolution: %s\n" + "weight: %d\n", + BYTES_TO_MB(type->low_gm_size), + BYTES_TO_MB(type->high_gm_size), + type->fence, vgpu_edid_str(type->resolution), + type->weight); +} + +static MDEV_TYPE_ATTR_RO(available_instances); +static MDEV_TYPE_ATTR_RO(device_api); +static MDEV_TYPE_ATTR_RO(description); + +static struct attribute *gvt_type_attrs[] = { + &mdev_type_attr_available_instances.attr, + &mdev_type_attr_device_api.attr, + &mdev_type_attr_description.attr, + NULL, +}; + +static struct attribute_group *gvt_vgpu_type_groups[] = { + [0 ... NR_MAX_INTEL_VGPU_TYPES - 1] = NULL, +}; + +static bool intel_get_gvt_attrs(struct attribute ***type_attrs, + struct attribute_group ***intel_vgpu_type_groups) +{ + *type_attrs = gvt_type_attrs; + *intel_vgpu_type_groups = gvt_vgpu_type_groups; + return true; +} + +static bool intel_gvt_init_vgpu_type_groups(struct intel_gvt *gvt) +{ + int i, j; + struct intel_vgpu_type *type; + struct attribute_group *group; + + for (i = 0; i < gvt->num_types; i++) { + type = &gvt->types[i]; + + group = kzalloc(sizeof(struct attribute_group), GFP_KERNEL); + if (WARN_ON(!group)) + goto unwind; + + group->name = type->name; + group->attrs = gvt_type_attrs; + gvt_vgpu_type_groups[i] = group; + } + + return true; + +unwind: + for (j = 0; j < i; j++) { + group = gvt_vgpu_type_groups[j]; + kfree(group); + } + + return false; +} + +static void intel_gvt_cleanup_vgpu_type_groups(struct intel_gvt *gvt) +{ + int i; + struct attribute_group *group; + + for (i = 0; i < gvt->num_types; i++) { + group = gvt_vgpu_type_groups[i]; + gvt_vgpu_type_groups[i] = NULL; + kfree(group); + } +} + static const struct intel_gvt_ops intel_gvt_ops = { .emulate_cfg_read = intel_vgpu_emulate_cfg_read, .emulate_cfg_write = intel_vgpu_emulate_cfg_write, @@ -54,6 +179,11 @@ static const struct intel_gvt_ops intel_gvt_ops = { .vgpu_reset = intel_gvt_reset_vgpu, .vgpu_activate = intel_gvt_activate_vgpu, .vgpu_deactivate = intel_gvt_deactivate_vgpu, + .gvt_find_vgpu_type = intel_gvt_find_vgpu_type, + .get_gvt_attrs = intel_get_gvt_attrs, + .vgpu_query_plane = intel_vgpu_query_plane, + .vgpu_get_dmabuf = intel_vgpu_get_dmabuf, + .write_protect_handler = intel_vgpu_write_protect_handler, }; /** @@ -191,17 +321,18 @@ void intel_gvt_clean_device(struct drm_i915_private *dev_priv) if (WARN_ON(!gvt)) return; + intel_gvt_debugfs_clean(gvt); clean_service_thread(gvt); intel_gvt_clean_cmd_parser(gvt); intel_gvt_clean_sched_policy(gvt); intel_gvt_clean_workload_scheduler(gvt); - intel_gvt_clean_opregion(gvt); intel_gvt_clean_gtt(gvt); intel_gvt_clean_irq(gvt); intel_gvt_clean_mmio_info(gvt); intel_gvt_free_firmware(gvt); intel_gvt_hypervisor_host_exit(&dev_priv->drm.pdev->dev, gvt); + intel_gvt_cleanup_vgpu_type_groups(gvt); intel_gvt_clean_vgpu_types(gvt); idr_destroy(&gvt->vgpu_idr); @@ -256,6 +387,8 @@ int intel_gvt_init_device(struct drm_i915_private *dev_priv) if (ret) goto out_clean_idr; + intel_gvt_init_engine_mmio_context(gvt); + ret = intel_gvt_load_firmware(gvt); if (ret) goto out_clean_mmio_info; @@ -268,13 +401,9 @@ int intel_gvt_init_device(struct drm_i915_private *dev_priv) if (ret) goto out_clean_irq; - ret = intel_gvt_init_opregion(gvt); - if (ret) - goto out_clean_gtt; - ret = intel_gvt_init_workload_scheduler(gvt); if (ret) - goto out_clean_opregion; + goto out_clean_gtt; ret = intel_gvt_init_sched_policy(gvt); if (ret) @@ -292,6 +421,12 @@ int intel_gvt_init_device(struct drm_i915_private *dev_priv) if (ret) goto out_clean_thread; + ret = intel_gvt_init_vgpu_type_groups(gvt); + if (ret == false) { + gvt_err("failed to init vgpu type groups: %d\n", ret); + goto out_clean_types; + } + ret = intel_gvt_hypervisor_host_init(&dev_priv->drm.pdev->dev, gvt, &intel_gvt_ops); if (ret) { @@ -307,6 +442,10 @@ int intel_gvt_init_device(struct drm_i915_private *dev_priv) } gvt->idle_vgpu = vgpu; + ret = intel_gvt_debugfs_init(gvt); + if (ret) + gvt_err("debugfs registeration failed, go on.\n"); + gvt_dbg_core("gvt device initialization is done\n"); dev_priv->gvt = gvt; return 0; @@ -321,8 +460,6 @@ out_clean_sched_policy: intel_gvt_clean_sched_policy(gvt); out_clean_workload_scheduler: intel_gvt_clean_workload_scheduler(gvt); -out_clean_opregion: - intel_gvt_clean_opregion(gvt); out_clean_gtt: intel_gvt_clean_gtt(gvt); out_clean_irq: diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h index 9c2e7c0aa38f..c6197d990818 100644 --- a/drivers/gpu/drm/i915/gvt/gvt.h +++ b/drivers/gpu/drm/i915/gvt/gvt.h @@ -44,8 +44,10 @@ #include "execlist.h" #include "scheduler.h" #include "sched_policy.h" -#include "render.h" +#include "mmio_context.h" #include "cmd_parser.h" +#include "fb_decoder.h" +#include "dmabuf.h" #define GVT_MAX_VGPU 8 @@ -80,7 +82,6 @@ struct intel_gvt_device_info { struct intel_vgpu_gm { u64 aperture_sz; u64 hidden_sz; - void *aperture_va; struct drm_mm_node low_gm_node; struct drm_mm_node high_gm_node; }; @@ -123,9 +124,9 @@ struct intel_vgpu_irq { }; struct intel_vgpu_opregion { + bool mapped; void *va; u32 gfn[INTEL_GVT_OPREGION_PAGES]; - struct page *pages[INTEL_GVT_OPREGION_PAGES]; }; #define vgpu_opregion(vgpu) (&(vgpu->opregion)) @@ -142,6 +143,33 @@ struct vgpu_sched_ctl { int weight; }; +enum { + INTEL_VGPU_EXECLIST_SUBMISSION = 1, + INTEL_VGPU_GUC_SUBMISSION, +}; + +struct intel_vgpu_submission_ops { + const char *name; + int (*init)(struct intel_vgpu *vgpu, unsigned long engine_mask); + void (*clean)(struct intel_vgpu *vgpu, unsigned long engine_mask); + void (*reset)(struct intel_vgpu *vgpu, unsigned long engine_mask); +}; + +struct intel_vgpu_submission { + struct intel_vgpu_execlist execlist[I915_NUM_ENGINES]; + struct list_head workload_q_head[I915_NUM_ENGINES]; + struct kmem_cache *workloads; + atomic_t running_workload_num; + struct i915_gem_context *shadow_ctx; + DECLARE_BITMAP(shadow_ctx_desc_updated, I915_NUM_ENGINES); + DECLARE_BITMAP(tlb_handle_pending, I915_NUM_ENGINES); + void *ring_scan_buffer[I915_NUM_ENGINES]; + int ring_scan_buffer_size[I915_NUM_ENGINES]; + const struct intel_vgpu_submission_ops *ops; + int virtual_submission_interface; + bool active; +}; + struct intel_vgpu { struct intel_gvt *gvt; int id; @@ -161,16 +189,10 @@ struct intel_vgpu { struct intel_vgpu_gtt gtt; struct intel_vgpu_opregion opregion; struct intel_vgpu_display display; - struct intel_vgpu_execlist execlist[I915_NUM_ENGINES]; - struct list_head workload_q_head[I915_NUM_ENGINES]; - struct kmem_cache *workloads; - atomic_t running_workload_num; - /* 1/2K for each reserve ring buffer */ - void *reserve_ring_buffer_va[I915_NUM_ENGINES]; - int reserve_ring_buffer_size[I915_NUM_ENGINES]; - DECLARE_BITMAP(tlb_handle_pending, I915_NUM_ENGINES); - struct i915_gem_context *shadow_ctx; - DECLARE_BITMAP(shadow_ctx_desc_updated, I915_NUM_ENGINES); + struct intel_vgpu_submission submission; + u32 hws_pga[I915_NUM_ENGINES]; + + struct dentry *debugfs; #if IS_ENABLED(CONFIG_DRM_I915_GVT_KVMGT) struct { @@ -186,10 +208,22 @@ struct intel_vgpu { struct kvm *kvm; struct work_struct release_work; atomic_t released; + struct vfio_device *vfio_device; } vdev; #endif + + struct list_head dmabuf_obj_list_head; + struct mutex dmabuf_lock; + struct idr object_idr; + + struct completion vblank_done; + }; +/* validating GM healthy status*/ +#define vgpu_is_vm_unhealthy(ret_val) \ + (((ret_val) == -EBADRQC) || ((ret_val) == -EFAULT)) + struct intel_gvt_gm { unsigned long vgpu_allocated_low_gm_size; unsigned long vgpu_allocated_high_gm_size; @@ -231,7 +265,7 @@ struct intel_gvt_mmio { unsigned int num_mmio_block; DECLARE_HASHTABLE(mmio_info_table, INTEL_GVT_MMIO_HASH_BITS); - unsigned int num_tracked_mmio; + unsigned long num_tracked_mmio; }; struct intel_gvt_firmware { @@ -240,11 +274,6 @@ struct intel_gvt_firmware { bool firmware_loaded; }; -struct intel_gvt_opregion { - void *opregion_va; - u32 opregion_pa; -}; - #define NR_MAX_INTEL_VGPU_TYPES 20 struct intel_vgpu_type { char name[16]; @@ -268,7 +297,6 @@ struct intel_gvt { struct intel_gvt_firmware firmware; struct intel_gvt_irq irq; struct intel_gvt_gtt gtt; - struct intel_gvt_opregion opregion; struct intel_gvt_workload_scheduler scheduler; struct notifier_block shadow_ctx_notifier_block[I915_NUM_ENGINES]; DECLARE_HASHTABLE(cmd_table, GVT_CMD_HASH_BITS); @@ -279,6 +307,10 @@ struct intel_gvt { struct task_struct *service_thread; wait_queue_head_t service_thread_wq; unsigned long service_request; + + struct engine_mmio *engine_mmio_list; + + struct dentry *debugfs_root; }; static inline struct intel_gvt *to_gvt(struct drm_i915_private *i915) @@ -316,7 +348,7 @@ int intel_gvt_load_firmware(struct intel_gvt *gvt); /* Aperture/GM space definitions for GVT device */ #define gvt_aperture_sz(gvt) (gvt->dev_priv->ggtt.mappable_end) -#define gvt_aperture_pa_base(gvt) (gvt->dev_priv->ggtt.mappable_base) +#define gvt_aperture_pa_base(gvt) (gvt->dev_priv->ggtt.gmadr.start) #define gvt_ggtt_gm_sz(gvt) (gvt->dev_priv->ggtt.base.total) #define gvt_ggtt_sz(gvt) \ @@ -378,23 +410,20 @@ void intel_vgpu_free_resource(struct intel_vgpu *vgpu); void intel_vgpu_write_fence(struct intel_vgpu *vgpu, u32 fence, u64 value); -/* Macros for easily accessing vGPU virtual/shadow register */ -#define vgpu_vreg(vgpu, reg) \ - (*(u32 *)(vgpu->mmio.vreg + INTEL_GVT_MMIO_OFFSET(reg))) -#define vgpu_vreg8(vgpu, reg) \ - (*(u8 *)(vgpu->mmio.vreg + INTEL_GVT_MMIO_OFFSET(reg))) -#define vgpu_vreg16(vgpu, reg) \ - (*(u16 *)(vgpu->mmio.vreg + INTEL_GVT_MMIO_OFFSET(reg))) -#define vgpu_vreg64(vgpu, reg) \ - (*(u64 *)(vgpu->mmio.vreg + INTEL_GVT_MMIO_OFFSET(reg))) -#define vgpu_sreg(vgpu, reg) \ - (*(u32 *)(vgpu->mmio.sreg + INTEL_GVT_MMIO_OFFSET(reg))) -#define vgpu_sreg8(vgpu, reg) \ - (*(u8 *)(vgpu->mmio.sreg + INTEL_GVT_MMIO_OFFSET(reg))) -#define vgpu_sreg16(vgpu, reg) \ - (*(u16 *)(vgpu->mmio.sreg + INTEL_GVT_MMIO_OFFSET(reg))) -#define vgpu_sreg64(vgpu, reg) \ - (*(u64 *)(vgpu->mmio.sreg + INTEL_GVT_MMIO_OFFSET(reg))) +/* Macros for easily accessing vGPU virtual/shadow register. + Explicitly seperate use for typed MMIO reg or real offset.*/ +#define vgpu_vreg_t(vgpu, reg) \ + (*(u32 *)(vgpu->mmio.vreg + i915_mmio_reg_offset(reg))) +#define vgpu_vreg(vgpu, offset) \ + (*(u32 *)(vgpu->mmio.vreg + (offset))) +#define vgpu_vreg64_t(vgpu, reg) \ + (*(u64 *)(vgpu->mmio.vreg + i915_mmio_reg_offset(reg))) +#define vgpu_vreg64(vgpu, offset) \ + (*(u64 *)(vgpu->mmio.vreg + (offset))) +#define vgpu_sreg_t(vgpu, reg) \ + (*(u32 *)(vgpu->mmio.sreg + i915_mmio_reg_offset(reg))) +#define vgpu_sreg(vgpu, offset) \ + (*(u32 *)(vgpu->mmio.sreg + (offset))) #define for_each_active_vgpu(gvt, vgpu, id) \ idr_for_each_entry((&(gvt)->vgpu_idr), (vgpu), (id)) \ @@ -484,16 +513,15 @@ static inline u64 intel_vgpu_get_bar_gpa(struct intel_vgpu *vgpu, int bar) PCI_BASE_ADDRESS_MEM_MASK; } -void intel_gvt_clean_opregion(struct intel_gvt *gvt); -int intel_gvt_init_opregion(struct intel_gvt *gvt); - void intel_vgpu_clean_opregion(struct intel_vgpu *vgpu); -int intel_vgpu_init_opregion(struct intel_vgpu *vgpu, u32 gpa); +int intel_vgpu_init_opregion(struct intel_vgpu *vgpu); +int intel_vgpu_opregion_base_write_handler(struct intel_vgpu *vgpu, u32 gpa); int intel_vgpu_emulate_opregion_request(struct intel_vgpu *vgpu, u32 swsci); void populate_pvinfo_page(struct intel_vgpu *vgpu); int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload); +void enter_failsafe_mode(struct intel_vgpu *vgpu, int reason); struct intel_gvt_ops { int (*emulate_cfg_read)(struct intel_vgpu *, unsigned int, void *, @@ -510,12 +538,21 @@ struct intel_gvt_ops { void (*vgpu_reset)(struct intel_vgpu *); void (*vgpu_activate)(struct intel_vgpu *); void (*vgpu_deactivate)(struct intel_vgpu *); + struct intel_vgpu_type *(*gvt_find_vgpu_type)(struct intel_gvt *gvt, + const char *name); + bool (*get_gvt_attrs)(struct attribute ***type_attrs, + struct attribute_group ***intel_vgpu_type_groups); + int (*vgpu_query_plane)(struct intel_vgpu *vgpu, void *); + int (*vgpu_get_dmabuf)(struct intel_vgpu *vgpu, unsigned int); + int (*write_protect_handler)(struct intel_vgpu *, u64, void *, + unsigned int); }; enum { GVT_FAILSAFE_UNSUPPORTED_GUEST, GVT_FAILSAFE_INSUFFICIENT_RESOURCE, + GVT_FAILSAFE_GUEST_ERR, }; static inline void mmio_hw_access_pre(struct drm_i915_private *dev_priv) @@ -591,6 +628,12 @@ static inline bool intel_gvt_mmio_has_mode_mask( return gvt->mmio.mmio_attribute[offset >> 2] & F_MODE_MASK; } +int intel_gvt_debugfs_add_vgpu(struct intel_vgpu *vgpu); +void intel_gvt_debugfs_remove_vgpu(struct intel_vgpu *vgpu); +int intel_gvt_debugfs_init(struct intel_gvt *gvt); +void intel_gvt_debugfs_clean(struct intel_gvt *gvt); + + #include "trace.h" #include "mpt.h" diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c index 1f840f6b81bb..9be639aa3b55 100644 --- a/drivers/gpu/drm/i915/gvt/handlers.c +++ b/drivers/gpu/drm/i915/gvt/handlers.c @@ -166,7 +166,7 @@ int intel_gvt_render_mmio_to_ring_id(struct intel_gvt *gvt, (num * 8 + i915_mmio_reg_offset(FENCE_REG_GEN6_LO(0))) -static void enter_failsafe_mode(struct intel_vgpu *vgpu, int reason) +void enter_failsafe_mode(struct intel_vgpu *vgpu, int reason) { switch (reason) { case GVT_FAILSAFE_UNSUPPORTED_GUEST: @@ -174,6 +174,10 @@ static void enter_failsafe_mode(struct intel_vgpu *vgpu, int reason) break; case GVT_FAILSAFE_INSUFFICIENT_RESOURCE: pr_err("Graphics resource is not enough for the guest\n"); + break; + case GVT_FAILSAFE_GUEST_ERR: + pr_err("GVT Internal error for the guest\n"); + break; default: break; } @@ -339,13 +343,13 @@ static int pch_pp_control_mmio_write(struct intel_vgpu *vgpu, write_vreg(vgpu, offset, p_data, bytes); if (vgpu_vreg(vgpu, offset) & PANEL_POWER_ON) { - vgpu_vreg(vgpu, PCH_PP_STATUS) |= PP_ON; - vgpu_vreg(vgpu, PCH_PP_STATUS) |= PP_SEQUENCE_STATE_ON_IDLE; - vgpu_vreg(vgpu, PCH_PP_STATUS) &= ~PP_SEQUENCE_POWER_DOWN; - vgpu_vreg(vgpu, PCH_PP_STATUS) &= ~PP_CYCLE_DELAY_ACTIVE; + vgpu_vreg_t(vgpu, PCH_PP_STATUS) |= PP_ON; + vgpu_vreg_t(vgpu, PCH_PP_STATUS) |= PP_SEQUENCE_STATE_ON_IDLE; + vgpu_vreg_t(vgpu, PCH_PP_STATUS) &= ~PP_SEQUENCE_POWER_DOWN; + vgpu_vreg_t(vgpu, PCH_PP_STATUS) &= ~PP_CYCLE_DELAY_ACTIVE; } else - vgpu_vreg(vgpu, PCH_PP_STATUS) &= + vgpu_vreg_t(vgpu, PCH_PP_STATUS) &= ~(PP_ON | PP_SEQUENCE_POWER_DOWN | PP_CYCLE_DELAY_ACTIVE); return 0; @@ -499,7 +503,7 @@ static int ddi_buf_ctl_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, } else { vgpu_vreg(vgpu, offset) |= DDI_BUF_IS_IDLE; if (offset == i915_mmio_reg_offset(DDI_BUF_CTL(PORT_E))) - vgpu_vreg(vgpu, DP_TP_STATUS(PORT_E)) + vgpu_vreg_t(vgpu, DP_TP_STATUS(PORT_E)) &= ~DP_TP_STATUS_AUTOTRAIN_DONE; } return 0; @@ -517,9 +521,9 @@ static int fdi_rx_iir_mmio_write(struct intel_vgpu *vgpu, static int fdi_auto_training_started(struct intel_vgpu *vgpu) { - u32 ddi_buf_ctl = vgpu_vreg(vgpu, DDI_BUF_CTL(PORT_E)); + u32 ddi_buf_ctl = vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_E)); u32 rx_ctl = vgpu_vreg(vgpu, _FDI_RXA_CTL); - u32 tx_ctl = vgpu_vreg(vgpu, DP_TP_CTL(PORT_E)); + u32 tx_ctl = vgpu_vreg_t(vgpu, DP_TP_CTL(PORT_E)); if ((ddi_buf_ctl & DDI_BUF_CTL_ENABLE) && (rx_ctl & FDI_RX_ENABLE) && @@ -560,12 +564,12 @@ static int check_fdi_rx_train_status(struct intel_vgpu *vgpu, fdi_tx_check_bits = FDI_TX_ENABLE | fdi_tx_train_bits; /* If imr bit has been masked */ - if (vgpu_vreg(vgpu, fdi_rx_imr) & fdi_iir_check_bits) + if (vgpu_vreg_t(vgpu, fdi_rx_imr) & fdi_iir_check_bits) return 0; - if (((vgpu_vreg(vgpu, fdi_tx_ctl) & fdi_tx_check_bits) + if (((vgpu_vreg_t(vgpu, fdi_tx_ctl) & fdi_tx_check_bits) == fdi_tx_check_bits) - && ((vgpu_vreg(vgpu, fdi_rx_ctl) & fdi_rx_check_bits) + && ((vgpu_vreg_t(vgpu, fdi_rx_ctl) & fdi_rx_check_bits) == fdi_rx_check_bits)) return 1; else @@ -622,17 +626,17 @@ static int update_fdi_rx_iir_status(struct intel_vgpu *vgpu, if (ret < 0) return ret; if (ret) - vgpu_vreg(vgpu, fdi_rx_iir) |= FDI_RX_BIT_LOCK; + vgpu_vreg_t(vgpu, fdi_rx_iir) |= FDI_RX_BIT_LOCK; ret = check_fdi_rx_train_status(vgpu, index, FDI_LINK_TRAIN_PATTERN2); if (ret < 0) return ret; if (ret) - vgpu_vreg(vgpu, fdi_rx_iir) |= FDI_RX_SYMBOL_LOCK; + vgpu_vreg_t(vgpu, fdi_rx_iir) |= FDI_RX_SYMBOL_LOCK; if (offset == _FDI_RXA_CTL) if (fdi_auto_training_started(vgpu)) - vgpu_vreg(vgpu, DP_TP_STATUS(PORT_E)) |= + vgpu_vreg_t(vgpu, DP_TP_STATUS(PORT_E)) |= DP_TP_STATUS_AUTOTRAIN_DONE; return 0; } @@ -653,7 +657,7 @@ static int dp_tp_ctl_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, data = (vgpu_vreg(vgpu, offset) & GENMASK(10, 8)) >> 8; if (data == 0x2) { status_reg = DP_TP_STATUS(index); - vgpu_vreg(vgpu, status_reg) |= (1 << 25); + vgpu_vreg_t(vgpu, status_reg) |= (1 << 25); } return 0; } @@ -717,7 +721,7 @@ static int pri_surf_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, }; write_vreg(vgpu, offset, p_data, bytes); - vgpu_vreg(vgpu, surflive_reg) = vgpu_vreg(vgpu, offset); + vgpu_vreg_t(vgpu, surflive_reg) = vgpu_vreg(vgpu, offset); set_bit(flip_event[index], vgpu->irq.flip_done_event[index]); return 0; @@ -738,7 +742,7 @@ static int spr_surf_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, }; write_vreg(vgpu, offset, p_data, bytes); - vgpu_vreg(vgpu, surflive_reg) = vgpu_vreg(vgpu, offset); + vgpu_vreg_t(vgpu, surflive_reg) = vgpu_vreg(vgpu, offset); set_bit(flip_event[index], vgpu->irq.flip_done_event[index]); return 0; @@ -1060,9 +1064,9 @@ static void write_virtual_sbi_register(struct intel_vgpu *vgpu, static int sbi_data_mmio_read(struct intel_vgpu *vgpu, unsigned int offset, void *p_data, unsigned int bytes) { - if (((vgpu_vreg(vgpu, SBI_CTL_STAT) & SBI_OPCODE_MASK) >> + if (((vgpu_vreg_t(vgpu, SBI_CTL_STAT) & SBI_OPCODE_MASK) >> SBI_OPCODE_SHIFT) == SBI_CMD_CRRD) { - unsigned int sbi_offset = (vgpu_vreg(vgpu, SBI_ADDR) & + unsigned int sbi_offset = (vgpu_vreg_t(vgpu, SBI_ADDR) & SBI_ADDR_OFFSET_MASK) >> SBI_ADDR_OFFSET_SHIFT; vgpu_vreg(vgpu, offset) = read_virtual_sbi_register(vgpu, sbi_offset); @@ -1087,13 +1091,13 @@ static int sbi_ctl_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, vgpu_vreg(vgpu, offset) = data; - if (((vgpu_vreg(vgpu, SBI_CTL_STAT) & SBI_OPCODE_MASK) >> + if (((vgpu_vreg_t(vgpu, SBI_CTL_STAT) & SBI_OPCODE_MASK) >> SBI_OPCODE_SHIFT) == SBI_CMD_CRWR) { - unsigned int sbi_offset = (vgpu_vreg(vgpu, SBI_ADDR) & + unsigned int sbi_offset = (vgpu_vreg_t(vgpu, SBI_ADDR) & SBI_ADDR_OFFSET_MASK) >> SBI_ADDR_OFFSET_SHIFT; write_virtual_sbi_register(vgpu, sbi_offset, - vgpu_vreg(vgpu, SBI_DATA)); + vgpu_vreg_t(vgpu, SBI_DATA)); } return 0; } @@ -1339,7 +1343,7 @@ static int mailbox_write(struct intel_vgpu *vgpu, unsigned int offset, { u32 value = *(u32 *)p_data; u32 cmd = value & 0xff; - u32 *data0 = &vgpu_vreg(vgpu, GEN6_PCODE_DATA); + u32 *data0 = &vgpu_vreg_t(vgpu, GEN6_PCODE_DATA); switch (cmd) { case GEN9_PCODE_READ_MEM_LATENCY: @@ -1378,6 +1382,34 @@ static int mailbox_write(struct intel_vgpu *vgpu, unsigned int offset, return intel_vgpu_default_mmio_write(vgpu, offset, &value, bytes); } +static int hws_pga_write(struct intel_vgpu *vgpu, unsigned int offset, + void *p_data, unsigned int bytes) +{ + u32 value = *(u32 *)p_data; + int ring_id = intel_gvt_render_mmio_to_ring_id(vgpu->gvt, offset); + + if (!intel_gvt_ggtt_validate_range(vgpu, value, I915_GTT_PAGE_SIZE)) { + gvt_vgpu_err("VM(%d) write invalid HWSP address, reg:0x%x, value:0x%x\n", + vgpu->id, offset, value); + return -EINVAL; + } + /* + * Need to emulate all the HWSP register write to ensure host can + * update the VM CSB status correctly. Here listed registers can + * support BDW, SKL or other platforms with same HWSP registers. + */ + if (unlikely(ring_id < 0 || ring_id >= I915_NUM_ENGINES)) { + gvt_vgpu_err("VM(%d) access unknown hardware status page register:0x%x\n", + vgpu->id, offset); + return -EINVAL; + } + vgpu->hws_pga[ring_id] = value; + gvt_dbg_mmio("VM(%d) write: 0x%x to HWSP: 0x%x\n", + vgpu->id, value, offset); + + return intel_vgpu_default_mmio_write(vgpu, offset, &value, bytes); +} + static int skl_power_well_ctl_write(struct intel_vgpu *vgpu, unsigned int offset, void *p_data, unsigned int bytes) { @@ -1441,12 +1473,12 @@ static int elsp_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, u32 data = *(u32 *)p_data; int ret = 0; - if (WARN_ON(ring_id < 0 || ring_id > I915_NUM_ENGINES - 1)) + if (WARN_ON(ring_id < 0 || ring_id >= I915_NUM_ENGINES)) return -EINVAL; - execlist = &vgpu->execlist[ring_id]; + execlist = &vgpu->submission.execlist[ring_id]; - execlist->elsp_dwords.data[execlist->elsp_dwords.index] = data; + execlist->elsp_dwords.data[3 - execlist->elsp_dwords.index] = data; if (execlist->elsp_dwords.index == 3) { ret = intel_vgpu_submit_execlist(vgpu, ring_id); if(ret) @@ -1465,6 +1497,7 @@ static int ring_mode_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, u32 data = *(u32 *)p_data; int ring_id = intel_gvt_render_mmio_to_ring_id(vgpu->gvt, offset); bool enable_execlist; + int ret; write_vreg(vgpu, offset, p_data, bytes); @@ -1486,8 +1519,16 @@ static int ring_mode_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, (enable_execlist ? "enabling" : "disabling"), ring_id); - if (enable_execlist) - intel_vgpu_start_schedule(vgpu); + if (!enable_execlist) + return 0; + + ret = intel_vgpu_select_submission_ops(vgpu, + ENGINE_MASK(ring_id), + INTEL_VGPU_EXECLIST_SUBMISSION); + if (ret) + return ret; + + intel_vgpu_start_schedule(vgpu); } return 0; } @@ -1519,7 +1560,7 @@ static int gvt_reg_tlb_control_handler(struct intel_vgpu *vgpu, default: return -EINVAL; } - set_bit(id, (void *)vgpu->tlb_handle_pending); + set_bit(id, (void *)vgpu->submission.tlb_handle_pending); return 0; } @@ -1542,7 +1583,7 @@ static int ring_reset_ctl_write(struct intel_vgpu *vgpu, } #define MMIO_F(reg, s, f, am, rm, d, r, w) do { \ - ret = new_mmio_info(gvt, INTEL_GVT_MMIO_OFFSET(reg), \ + ret = new_mmio_info(gvt, i915_mmio_reg_offset(reg), \ f, s, am, rm, d, r, w); \ if (ret) \ return ret; \ @@ -1610,22 +1651,22 @@ static int init_generic_mmio_info(struct intel_gvt *gvt) MMIO_GM_RDR(BLT_HWS_PGA_GEN7, D_ALL, NULL, NULL); MMIO_GM_RDR(VEBOX_HWS_PGA_GEN7, D_ALL, NULL, NULL); -#define RING_REG(base) (base + 0x28) +#define RING_REG(base) _MMIO((base) + 0x28) MMIO_RING_DFH(RING_REG, D_ALL, F_CMD_ACCESS, NULL, NULL); #undef RING_REG -#define RING_REG(base) (base + 0x134) +#define RING_REG(base) _MMIO((base) + 0x134) MMIO_RING_DFH(RING_REG, D_ALL, F_CMD_ACCESS, NULL, NULL); #undef RING_REG -#define RING_REG(base) (base + 0x6c) +#define RING_REG(base) _MMIO((base) + 0x6c) MMIO_RING_DFH(RING_REG, D_ALL, 0, mmio_read_from_hw, NULL); #undef RING_REG MMIO_DH(GEN7_SC_INSTDONE, D_BDW_PLUS, mmio_read_from_hw, NULL); - MMIO_GM_RDR(0x2148, D_ALL, NULL, NULL); + MMIO_GM_RDR(_MMIO(0x2148), D_ALL, NULL, NULL); MMIO_GM_RDR(CCID, D_ALL, NULL, NULL); - MMIO_GM_RDR(0x12198, D_ALL, NULL, NULL); + MMIO_GM_RDR(_MMIO(0x12198), D_ALL, NULL, NULL); MMIO_D(GEN7_CXT_SIZE, D_ALL); MMIO_RING_DFH(RING_TAIL, D_ALL, F_CMD_ACCESS, NULL, NULL); @@ -1635,7 +1676,7 @@ static int init_generic_mmio_info(struct intel_gvt *gvt) MMIO_RING_GM_RDR(RING_START, D_ALL, NULL, NULL); /* RING MODE */ -#define RING_REG(base) (base + 0x29c) +#define RING_REG(base) _MMIO((base) + 0x29c) MMIO_RING_DFH(RING_REG, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, ring_mode_mmio_write); #undef RING_REG @@ -1654,37 +1695,37 @@ static int init_generic_mmio_info(struct intel_gvt *gvt) NULL, NULL); MMIO_DFH(CACHE_MODE_1, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); MMIO_DFH(CACHE_MODE_0, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x2124, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x2124), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x20dc, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x20dc), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); MMIO_DFH(_3D_CHICKEN3, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x2088, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x20e4, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x2470, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x2088), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x20e4), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x2470), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); MMIO_DFH(GAM_ECOCHK, D_ALL, F_CMD_ACCESS, NULL, NULL); MMIO_DFH(GEN7_COMMON_SLICE_CHICKEN1, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); MMIO_DFH(COMMON_SLICE_CHICKEN2, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x9030, D_ALL, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x20a0, D_ALL, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x2420, D_ALL, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x2430, D_ALL, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x2434, D_ALL, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x2438, D_ALL, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x243c, D_ALL, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x7018, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x9030), D_ALL, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x20a0), D_ALL, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x2420), D_ALL, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x2430), D_ALL, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x2434), D_ALL, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x2438), D_ALL, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x243c), D_ALL, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x7018), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); MMIO_DFH(HALF_SLICE_CHICKEN3, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); MMIO_DFH(GEN7_HALF_SLICE_CHICKEN1, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); /* display */ - MMIO_F(0x60220, 0x20, 0, 0, 0, D_ALL, NULL, NULL); - MMIO_D(0x602a0, D_ALL); + MMIO_F(_MMIO(0x60220), 0x20, 0, 0, 0, D_ALL, NULL, NULL); + MMIO_D(_MMIO(0x602a0), D_ALL); - MMIO_D(0x65050, D_ALL); - MMIO_D(0x650b4, D_ALL); + MMIO_D(_MMIO(0x65050), D_ALL); + MMIO_D(_MMIO(0x650b4), D_ALL); - MMIO_D(0xc4040, D_ALL); + MMIO_D(_MMIO(0xc4040), D_ALL); MMIO_D(DERRMR, D_ALL); MMIO_D(PIPEDSL(PIPE_A), D_ALL); @@ -1724,14 +1765,14 @@ static int init_generic_mmio_info(struct intel_gvt *gvt) MMIO_D(CURBASE(PIPE_B), D_ALL); MMIO_D(CURBASE(PIPE_C), D_ALL); - MMIO_D(0x700ac, D_ALL); - MMIO_D(0x710ac, D_ALL); - MMIO_D(0x720ac, D_ALL); + MMIO_D(_MMIO(0x700ac), D_ALL); + MMIO_D(_MMIO(0x710ac), D_ALL); + MMIO_D(_MMIO(0x720ac), D_ALL); - MMIO_D(0x70090, D_ALL); - MMIO_D(0x70094, D_ALL); - MMIO_D(0x70098, D_ALL); - MMIO_D(0x7009c, D_ALL); + MMIO_D(_MMIO(0x70090), D_ALL); + MMIO_D(_MMIO(0x70094), D_ALL); + MMIO_D(_MMIO(0x70098), D_ALL); + MMIO_D(_MMIO(0x7009c), D_ALL); MMIO_D(DSPCNTR(PIPE_A), D_ALL); MMIO_D(DSPADDR(PIPE_A), D_ALL); @@ -1907,24 +1948,24 @@ static int init_generic_mmio_info(struct intel_gvt *gvt) MMIO_D(BLC_PWM_PCH_CTL1, D_ALL); MMIO_D(BLC_PWM_PCH_CTL2, D_ALL); - MMIO_D(0x48268, D_ALL); + MMIO_D(_MMIO(0x48268), D_ALL); MMIO_F(PCH_GMBUS0, 4 * 4, 0, 0, 0, D_ALL, gmbus_mmio_read, gmbus_mmio_write); MMIO_F(PCH_GPIOA, 6 * 4, F_UNALIGN, 0, 0, D_ALL, NULL, NULL); - MMIO_F(0xe4f00, 0x28, 0, 0, 0, D_ALL, NULL, NULL); + MMIO_F(_MMIO(0xe4f00), 0x28, 0, 0, 0, D_ALL, NULL, NULL); - MMIO_F(_PCH_DPB_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_PRE_SKL, NULL, + MMIO_F(_MMIO(_PCH_DPB_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_PRE_SKL, NULL, dp_aux_ch_ctl_mmio_write); - MMIO_F(_PCH_DPC_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_PRE_SKL, NULL, + MMIO_F(_MMIO(_PCH_DPC_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_PRE_SKL, NULL, dp_aux_ch_ctl_mmio_write); - MMIO_F(_PCH_DPD_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_PRE_SKL, NULL, + MMIO_F(_MMIO(_PCH_DPD_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_PRE_SKL, NULL, dp_aux_ch_ctl_mmio_write); MMIO_DH(PCH_ADPA, D_PRE_SKL, NULL, pch_adpa_mmio_write); - MMIO_DH(_PCH_TRANSACONF, D_ALL, NULL, transconf_mmio_write); - MMIO_DH(_PCH_TRANSBCONF, D_ALL, NULL, transconf_mmio_write); + MMIO_DH(_MMIO(_PCH_TRANSACONF), D_ALL, NULL, transconf_mmio_write); + MMIO_DH(_MMIO(_PCH_TRANSBCONF), D_ALL, NULL, transconf_mmio_write); MMIO_DH(FDI_RX_IIR(PIPE_A), D_ALL, NULL, fdi_rx_iir_mmio_write); MMIO_DH(FDI_RX_IIR(PIPE_B), D_ALL, NULL, fdi_rx_iir_mmio_write); @@ -1936,30 +1977,30 @@ static int init_generic_mmio_info(struct intel_gvt *gvt) MMIO_DH(FDI_RX_CTL(PIPE_B), D_ALL, NULL, update_fdi_rx_iir_status); MMIO_DH(FDI_RX_CTL(PIPE_C), D_ALL, NULL, update_fdi_rx_iir_status); - MMIO_D(_PCH_TRANS_HTOTAL_A, D_ALL); - MMIO_D(_PCH_TRANS_HBLANK_A, D_ALL); - MMIO_D(_PCH_TRANS_HSYNC_A, D_ALL); - MMIO_D(_PCH_TRANS_VTOTAL_A, D_ALL); - MMIO_D(_PCH_TRANS_VBLANK_A, D_ALL); - MMIO_D(_PCH_TRANS_VSYNC_A, D_ALL); - MMIO_D(_PCH_TRANS_VSYNCSHIFT_A, D_ALL); - - MMIO_D(_PCH_TRANS_HTOTAL_B, D_ALL); - MMIO_D(_PCH_TRANS_HBLANK_B, D_ALL); - MMIO_D(_PCH_TRANS_HSYNC_B, D_ALL); - MMIO_D(_PCH_TRANS_VTOTAL_B, D_ALL); - MMIO_D(_PCH_TRANS_VBLANK_B, D_ALL); - MMIO_D(_PCH_TRANS_VSYNC_B, D_ALL); - MMIO_D(_PCH_TRANS_VSYNCSHIFT_B, D_ALL); - - MMIO_D(_PCH_TRANSA_DATA_M1, D_ALL); - MMIO_D(_PCH_TRANSA_DATA_N1, D_ALL); - MMIO_D(_PCH_TRANSA_DATA_M2, D_ALL); - MMIO_D(_PCH_TRANSA_DATA_N2, D_ALL); - MMIO_D(_PCH_TRANSA_LINK_M1, D_ALL); - MMIO_D(_PCH_TRANSA_LINK_N1, D_ALL); - MMIO_D(_PCH_TRANSA_LINK_M2, D_ALL); - MMIO_D(_PCH_TRANSA_LINK_N2, D_ALL); + MMIO_D(_MMIO(_PCH_TRANS_HTOTAL_A), D_ALL); + MMIO_D(_MMIO(_PCH_TRANS_HBLANK_A), D_ALL); + MMIO_D(_MMIO(_PCH_TRANS_HSYNC_A), D_ALL); + MMIO_D(_MMIO(_PCH_TRANS_VTOTAL_A), D_ALL); + MMIO_D(_MMIO(_PCH_TRANS_VBLANK_A), D_ALL); + MMIO_D(_MMIO(_PCH_TRANS_VSYNC_A), D_ALL); + MMIO_D(_MMIO(_PCH_TRANS_VSYNCSHIFT_A), D_ALL); + + MMIO_D(_MMIO(_PCH_TRANS_HTOTAL_B), D_ALL); + MMIO_D(_MMIO(_PCH_TRANS_HBLANK_B), D_ALL); + MMIO_D(_MMIO(_PCH_TRANS_HSYNC_B), D_ALL); + MMIO_D(_MMIO(_PCH_TRANS_VTOTAL_B), D_ALL); + MMIO_D(_MMIO(_PCH_TRANS_VBLANK_B), D_ALL); + MMIO_D(_MMIO(_PCH_TRANS_VSYNC_B), D_ALL); + MMIO_D(_MMIO(_PCH_TRANS_VSYNCSHIFT_B), D_ALL); + + MMIO_D(_MMIO(_PCH_TRANSA_DATA_M1), D_ALL); + MMIO_D(_MMIO(_PCH_TRANSA_DATA_N1), D_ALL); + MMIO_D(_MMIO(_PCH_TRANSA_DATA_M2), D_ALL); + MMIO_D(_MMIO(_PCH_TRANSA_DATA_N2), D_ALL); + MMIO_D(_MMIO(_PCH_TRANSA_LINK_M1), D_ALL); + MMIO_D(_MMIO(_PCH_TRANSA_LINK_N1), D_ALL); + MMIO_D(_MMIO(_PCH_TRANSA_LINK_M2), D_ALL); + MMIO_D(_MMIO(_PCH_TRANSA_LINK_N2), D_ALL); MMIO_D(TRANS_DP_CTL(PIPE_A), D_ALL); MMIO_D(TRANS_DP_CTL(PIPE_B), D_ALL); @@ -1977,38 +2018,38 @@ static int init_generic_mmio_info(struct intel_gvt *gvt) MMIO_D(TVIDEO_DIP_DATA(PIPE_C), D_ALL); MMIO_D(TVIDEO_DIP_GCP(PIPE_C), D_ALL); - MMIO_D(_FDI_RXA_MISC, D_ALL); - MMIO_D(_FDI_RXB_MISC, D_ALL); - MMIO_D(_FDI_RXA_TUSIZE1, D_ALL); - MMIO_D(_FDI_RXA_TUSIZE2, D_ALL); - MMIO_D(_FDI_RXB_TUSIZE1, D_ALL); - MMIO_D(_FDI_RXB_TUSIZE2, D_ALL); + MMIO_D(_MMIO(_FDI_RXA_MISC), D_ALL); + MMIO_D(_MMIO(_FDI_RXB_MISC), D_ALL); + MMIO_D(_MMIO(_FDI_RXA_TUSIZE1), D_ALL); + MMIO_D(_MMIO(_FDI_RXA_TUSIZE2), D_ALL); + MMIO_D(_MMIO(_FDI_RXB_TUSIZE1), D_ALL); + MMIO_D(_MMIO(_FDI_RXB_TUSIZE2), D_ALL); MMIO_DH(PCH_PP_CONTROL, D_ALL, NULL, pch_pp_control_mmio_write); MMIO_D(PCH_PP_DIVISOR, D_ALL); MMIO_D(PCH_PP_STATUS, D_ALL); MMIO_D(PCH_LVDS, D_ALL); - MMIO_D(_PCH_DPLL_A, D_ALL); - MMIO_D(_PCH_DPLL_B, D_ALL); - MMIO_D(_PCH_FPA0, D_ALL); - MMIO_D(_PCH_FPA1, D_ALL); - MMIO_D(_PCH_FPB0, D_ALL); - MMIO_D(_PCH_FPB1, D_ALL); + MMIO_D(_MMIO(_PCH_DPLL_A), D_ALL); + MMIO_D(_MMIO(_PCH_DPLL_B), D_ALL); + MMIO_D(_MMIO(_PCH_FPA0), D_ALL); + MMIO_D(_MMIO(_PCH_FPA1), D_ALL); + MMIO_D(_MMIO(_PCH_FPB0), D_ALL); + MMIO_D(_MMIO(_PCH_FPB1), D_ALL); MMIO_D(PCH_DREF_CONTROL, D_ALL); MMIO_D(PCH_RAWCLK_FREQ, D_ALL); MMIO_D(PCH_DPLL_SEL, D_ALL); - MMIO_D(0x61208, D_ALL); - MMIO_D(0x6120c, D_ALL); + MMIO_D(_MMIO(0x61208), D_ALL); + MMIO_D(_MMIO(0x6120c), D_ALL); MMIO_D(PCH_PP_ON_DELAYS, D_ALL); MMIO_D(PCH_PP_OFF_DELAYS, D_ALL); - MMIO_DH(0xe651c, D_ALL, dpy_reg_mmio_read, NULL); - MMIO_DH(0xe661c, D_ALL, dpy_reg_mmio_read, NULL); - MMIO_DH(0xe671c, D_ALL, dpy_reg_mmio_read, NULL); - MMIO_DH(0xe681c, D_ALL, dpy_reg_mmio_read, NULL); - MMIO_DH(0xe6c04, D_ALL, dpy_reg_mmio_read, NULL); - MMIO_DH(0xe6e1c, D_ALL, dpy_reg_mmio_read, NULL); + MMIO_DH(_MMIO(0xe651c), D_ALL, dpy_reg_mmio_read, NULL); + MMIO_DH(_MMIO(0xe661c), D_ALL, dpy_reg_mmio_read, NULL); + MMIO_DH(_MMIO(0xe671c), D_ALL, dpy_reg_mmio_read, NULL); + MMIO_DH(_MMIO(0xe681c), D_ALL, dpy_reg_mmio_read, NULL); + MMIO_DH(_MMIO(0xe6c04), D_ALL, dpy_reg_mmio_read, NULL); + MMIO_DH(_MMIO(0xe6e1c), D_ALL, dpy_reg_mmio_read, NULL); MMIO_RO(PCH_PORT_HOTPLUG, D_ALL, 0, PORTA_HOTPLUG_STATUS_MASK @@ -2030,11 +2071,11 @@ static int init_generic_mmio_info(struct intel_gvt *gvt) MMIO_D(SOUTH_CHICKEN1, D_ALL); MMIO_DH(SOUTH_CHICKEN2, D_ALL, NULL, south_chicken2_mmio_write); - MMIO_D(_TRANSA_CHICKEN1, D_ALL); - MMIO_D(_TRANSB_CHICKEN1, D_ALL); + MMIO_D(_MMIO(_TRANSA_CHICKEN1), D_ALL); + MMIO_D(_MMIO(_TRANSB_CHICKEN1), D_ALL); MMIO_D(SOUTH_DSPCLK_GATE_D, D_ALL); - MMIO_D(_TRANSA_CHICKEN2, D_ALL); - MMIO_D(_TRANSB_CHICKEN2, D_ALL); + MMIO_D(_MMIO(_TRANSA_CHICKEN2), D_ALL); + MMIO_D(_MMIO(_TRANSB_CHICKEN2), D_ALL); MMIO_D(ILK_DPFC_CB_BASE, D_ALL); MMIO_D(ILK_DPFC_CONTROL, D_ALL); @@ -2100,24 +2141,24 @@ static int init_generic_mmio_info(struct intel_gvt *gvt) MMIO_D(PREC_PAL_DATA(PIPE_C), D_ALL); MMIO_F(PREC_PAL_GC_MAX(PIPE_C, 0), 4 * 3, 0, 0, 0, D_ALL, NULL, NULL); - MMIO_D(0x60110, D_ALL); - MMIO_D(0x61110, D_ALL); - MMIO_F(0x70400, 0x40, 0, 0, 0, D_ALL, NULL, NULL); - MMIO_F(0x71400, 0x40, 0, 0, 0, D_ALL, NULL, NULL); - MMIO_F(0x72400, 0x40, 0, 0, 0, D_ALL, NULL, NULL); - MMIO_F(0x70440, 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL); - MMIO_F(0x71440, 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL); - MMIO_F(0x72440, 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL); - MMIO_F(0x7044c, 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL); - MMIO_F(0x7144c, 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL); - MMIO_F(0x7244c, 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL); + MMIO_D(_MMIO(0x60110), D_ALL); + MMIO_D(_MMIO(0x61110), D_ALL); + MMIO_F(_MMIO(0x70400), 0x40, 0, 0, 0, D_ALL, NULL, NULL); + MMIO_F(_MMIO(0x71400), 0x40, 0, 0, 0, D_ALL, NULL, NULL); + MMIO_F(_MMIO(0x72400), 0x40, 0, 0, 0, D_ALL, NULL, NULL); + MMIO_F(_MMIO(0x70440), 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL); + MMIO_F(_MMIO(0x71440), 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL); + MMIO_F(_MMIO(0x72440), 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL); + MMIO_F(_MMIO(0x7044c), 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL); + MMIO_F(_MMIO(0x7144c), 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL); + MMIO_F(_MMIO(0x7244c), 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL); MMIO_D(PIPE_WM_LINETIME(PIPE_A), D_ALL); MMIO_D(PIPE_WM_LINETIME(PIPE_B), D_ALL); MMIO_D(PIPE_WM_LINETIME(PIPE_C), D_ALL); MMIO_D(SPLL_CTL, D_ALL); - MMIO_D(_WRPLL_CTL1, D_ALL); - MMIO_D(_WRPLL_CTL2, D_ALL); + MMIO_D(_MMIO(_WRPLL_CTL1), D_ALL); + MMIO_D(_MMIO(_WRPLL_CTL2), D_ALL); MMIO_D(PORT_CLK_SEL(PORT_A), D_ALL); MMIO_D(PORT_CLK_SEL(PORT_B), D_ALL); MMIO_D(PORT_CLK_SEL(PORT_C), D_ALL); @@ -2128,15 +2169,15 @@ static int init_generic_mmio_info(struct intel_gvt *gvt) MMIO_D(TRANS_CLK_SEL(TRANSCODER_C), D_ALL); MMIO_D(HSW_NDE_RSTWRN_OPT, D_ALL); - MMIO_D(0x46508, D_ALL); + MMIO_D(_MMIO(0x46508), D_ALL); - MMIO_D(0x49080, D_ALL); - MMIO_D(0x49180, D_ALL); - MMIO_D(0x49280, D_ALL); + MMIO_D(_MMIO(0x49080), D_ALL); + MMIO_D(_MMIO(0x49180), D_ALL); + MMIO_D(_MMIO(0x49280), D_ALL); - MMIO_F(0x49090, 0x14, 0, 0, 0, D_ALL, NULL, NULL); - MMIO_F(0x49190, 0x14, 0, 0, 0, D_ALL, NULL, NULL); - MMIO_F(0x49290, 0x14, 0, 0, 0, D_ALL, NULL, NULL); + MMIO_F(_MMIO(0x49090), 0x14, 0, 0, 0, D_ALL, NULL, NULL); + MMIO_F(_MMIO(0x49190), 0x14, 0, 0, 0, D_ALL, NULL, NULL); + MMIO_F(_MMIO(0x49290), 0x14, 0, 0, 0, D_ALL, NULL, NULL); MMIO_D(GAMMA_MODE(PIPE_A), D_ALL); MMIO_D(GAMMA_MODE(PIPE_B), D_ALL); @@ -2156,7 +2197,7 @@ static int init_generic_mmio_info(struct intel_gvt *gvt) MMIO_DH(SBI_CTL_STAT, D_ALL, NULL, sbi_ctl_mmio_write); MMIO_D(PIXCLK_GATE, D_ALL); - MMIO_F(_DPA_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_ALL, NULL, + MMIO_F(_MMIO(_DPA_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_ALL, NULL, dp_aux_ch_ctl_mmio_write); MMIO_DH(DDI_BUF_CTL(PORT_A), D_ALL, NULL, ddi_buf_ctl_mmio_write); @@ -2177,24 +2218,24 @@ static int init_generic_mmio_info(struct intel_gvt *gvt) MMIO_DH(DP_TP_STATUS(PORT_D), D_ALL, NULL, dp_tp_status_mmio_write); MMIO_DH(DP_TP_STATUS(PORT_E), D_ALL, NULL, NULL); - MMIO_F(_DDI_BUF_TRANS_A, 0x50, 0, 0, 0, D_ALL, NULL, NULL); - MMIO_F(0x64e60, 0x50, 0, 0, 0, D_ALL, NULL, NULL); - MMIO_F(0x64eC0, 0x50, 0, 0, 0, D_ALL, NULL, NULL); - MMIO_F(0x64f20, 0x50, 0, 0, 0, D_ALL, NULL, NULL); - MMIO_F(0x64f80, 0x50, 0, 0, 0, D_ALL, NULL, NULL); + MMIO_F(_MMIO(_DDI_BUF_TRANS_A), 0x50, 0, 0, 0, D_ALL, NULL, NULL); + MMIO_F(_MMIO(0x64e60), 0x50, 0, 0, 0, D_ALL, NULL, NULL); + MMIO_F(_MMIO(0x64eC0), 0x50, 0, 0, 0, D_ALL, NULL, NULL); + MMIO_F(_MMIO(0x64f20), 0x50, 0, 0, 0, D_ALL, NULL, NULL); + MMIO_F(_MMIO(0x64f80), 0x50, 0, 0, 0, D_ALL, NULL, NULL); MMIO_D(HSW_AUD_CFG(PIPE_A), D_ALL); MMIO_D(HSW_AUD_PIN_ELD_CP_VLD, D_ALL); - MMIO_DH(_TRANS_DDI_FUNC_CTL_A, D_ALL, NULL, NULL); - MMIO_DH(_TRANS_DDI_FUNC_CTL_B, D_ALL, NULL, NULL); - MMIO_DH(_TRANS_DDI_FUNC_CTL_C, D_ALL, NULL, NULL); - MMIO_DH(_TRANS_DDI_FUNC_CTL_EDP, D_ALL, NULL, NULL); + MMIO_DH(_MMIO(_TRANS_DDI_FUNC_CTL_A), D_ALL, NULL, NULL); + MMIO_DH(_MMIO(_TRANS_DDI_FUNC_CTL_B), D_ALL, NULL, NULL); + MMIO_DH(_MMIO(_TRANS_DDI_FUNC_CTL_C), D_ALL, NULL, NULL); + MMIO_DH(_MMIO(_TRANS_DDI_FUNC_CTL_EDP), D_ALL, NULL, NULL); - MMIO_D(_TRANSA_MSA_MISC, D_ALL); - MMIO_D(_TRANSB_MSA_MISC, D_ALL); - MMIO_D(_TRANSC_MSA_MISC, D_ALL); - MMIO_D(_TRANS_EDP_MSA_MISC, D_ALL); + MMIO_D(_MMIO(_TRANSA_MSA_MISC), D_ALL); + MMIO_D(_MMIO(_TRANSB_MSA_MISC), D_ALL); + MMIO_D(_MMIO(_TRANSC_MSA_MISC), D_ALL); + MMIO_D(_MMIO(_TRANS_EDP_MSA_MISC), D_ALL); MMIO_DH(FORCEWAKE, D_ALL, NULL, NULL); MMIO_D(FORCEWAKE_ACK, D_ALL); @@ -2260,101 +2301,101 @@ static int init_generic_mmio_info(struct intel_gvt *gvt) MMIO_D(GEN6_UCGCTL1, D_ALL); MMIO_D(GEN6_UCGCTL2, D_ALL); - MMIO_F(0x4f000, 0x90, 0, 0, 0, D_ALL, NULL, NULL); + MMIO_F(_MMIO(0x4f000), 0x90, 0, 0, 0, D_ALL, NULL, NULL); MMIO_D(GEN6_PCODE_DATA, D_ALL); - MMIO_D(0x13812c, D_ALL); + MMIO_D(_MMIO(0x13812c), D_ALL); MMIO_DH(GEN7_ERR_INT, D_ALL, NULL, NULL); MMIO_D(HSW_EDRAM_CAP, D_ALL); MMIO_D(HSW_IDICR, D_ALL); MMIO_DH(GFX_FLSH_CNTL_GEN6, D_ALL, NULL, NULL); - MMIO_D(0x3c, D_ALL); - MMIO_D(0x860, D_ALL); + MMIO_D(_MMIO(0x3c), D_ALL); + MMIO_D(_MMIO(0x860), D_ALL); MMIO_D(ECOSKPD, D_ALL); - MMIO_D(0x121d0, D_ALL); + MMIO_D(_MMIO(0x121d0), D_ALL); MMIO_D(GEN6_BLITTER_ECOSKPD, D_ALL); - MMIO_D(0x41d0, D_ALL); + MMIO_D(_MMIO(0x41d0), D_ALL); MMIO_D(GAC_ECO_BITS, D_ALL); - MMIO_D(0x6200, D_ALL); - MMIO_D(0x6204, D_ALL); - MMIO_D(0x6208, D_ALL); - MMIO_D(0x7118, D_ALL); - MMIO_D(0x7180, D_ALL); - MMIO_D(0x7408, D_ALL); - MMIO_D(0x7c00, D_ALL); + MMIO_D(_MMIO(0x6200), D_ALL); + MMIO_D(_MMIO(0x6204), D_ALL); + MMIO_D(_MMIO(0x6208), D_ALL); + MMIO_D(_MMIO(0x7118), D_ALL); + MMIO_D(_MMIO(0x7180), D_ALL); + MMIO_D(_MMIO(0x7408), D_ALL); + MMIO_D(_MMIO(0x7c00), D_ALL); MMIO_DH(GEN6_MBCTL, D_ALL, NULL, mbctl_write); - MMIO_D(0x911c, D_ALL); - MMIO_D(0x9120, D_ALL); + MMIO_D(_MMIO(0x911c), D_ALL); + MMIO_D(_MMIO(0x9120), D_ALL); MMIO_DFH(GEN7_UCGCTL4, D_ALL, F_CMD_ACCESS, NULL, NULL); MMIO_D(GAB_CTL, D_ALL); - MMIO_D(0x48800, D_ALL); - MMIO_D(0xce044, D_ALL); - MMIO_D(0xe6500, D_ALL); - MMIO_D(0xe6504, D_ALL); - MMIO_D(0xe6600, D_ALL); - MMIO_D(0xe6604, D_ALL); - MMIO_D(0xe6700, D_ALL); - MMIO_D(0xe6704, D_ALL); - MMIO_D(0xe6800, D_ALL); - MMIO_D(0xe6804, D_ALL); + MMIO_D(_MMIO(0x48800), D_ALL); + MMIO_D(_MMIO(0xce044), D_ALL); + MMIO_D(_MMIO(0xe6500), D_ALL); + MMIO_D(_MMIO(0xe6504), D_ALL); + MMIO_D(_MMIO(0xe6600), D_ALL); + MMIO_D(_MMIO(0xe6604), D_ALL); + MMIO_D(_MMIO(0xe6700), D_ALL); + MMIO_D(_MMIO(0xe6704), D_ALL); + MMIO_D(_MMIO(0xe6800), D_ALL); + MMIO_D(_MMIO(0xe6804), D_ALL); MMIO_D(PCH_GMBUS4, D_ALL); MMIO_D(PCH_GMBUS5, D_ALL); - MMIO_D(0x902c, D_ALL); - MMIO_D(0xec008, D_ALL); - MMIO_D(0xec00c, D_ALL); - MMIO_D(0xec008 + 0x18, D_ALL); - MMIO_D(0xec00c + 0x18, D_ALL); - MMIO_D(0xec008 + 0x18 * 2, D_ALL); - MMIO_D(0xec00c + 0x18 * 2, D_ALL); - MMIO_D(0xec008 + 0x18 * 3, D_ALL); - MMIO_D(0xec00c + 0x18 * 3, D_ALL); - MMIO_D(0xec408, D_ALL); - MMIO_D(0xec40c, D_ALL); - MMIO_D(0xec408 + 0x18, D_ALL); - MMIO_D(0xec40c + 0x18, D_ALL); - MMIO_D(0xec408 + 0x18 * 2, D_ALL); - MMIO_D(0xec40c + 0x18 * 2, D_ALL); - MMIO_D(0xec408 + 0x18 * 3, D_ALL); - MMIO_D(0xec40c + 0x18 * 3, D_ALL); - MMIO_D(0xfc810, D_ALL); - MMIO_D(0xfc81c, D_ALL); - MMIO_D(0xfc828, D_ALL); - MMIO_D(0xfc834, D_ALL); - MMIO_D(0xfcc00, D_ALL); - MMIO_D(0xfcc0c, D_ALL); - MMIO_D(0xfcc18, D_ALL); - MMIO_D(0xfcc24, D_ALL); - MMIO_D(0xfd000, D_ALL); - MMIO_D(0xfd00c, D_ALL); - MMIO_D(0xfd018, D_ALL); - MMIO_D(0xfd024, D_ALL); - MMIO_D(0xfd034, D_ALL); + MMIO_D(_MMIO(0x902c), D_ALL); + MMIO_D(_MMIO(0xec008), D_ALL); + MMIO_D(_MMIO(0xec00c), D_ALL); + MMIO_D(_MMIO(0xec008 + 0x18), D_ALL); + MMIO_D(_MMIO(0xec00c + 0x18), D_ALL); + MMIO_D(_MMIO(0xec008 + 0x18 * 2), D_ALL); + MMIO_D(_MMIO(0xec00c + 0x18 * 2), D_ALL); + MMIO_D(_MMIO(0xec008 + 0x18 * 3), D_ALL); + MMIO_D(_MMIO(0xec00c + 0x18 * 3), D_ALL); + MMIO_D(_MMIO(0xec408), D_ALL); + MMIO_D(_MMIO(0xec40c), D_ALL); + MMIO_D(_MMIO(0xec408 + 0x18), D_ALL); + MMIO_D(_MMIO(0xec40c + 0x18), D_ALL); + MMIO_D(_MMIO(0xec408 + 0x18 * 2), D_ALL); + MMIO_D(_MMIO(0xec40c + 0x18 * 2), D_ALL); + MMIO_D(_MMIO(0xec408 + 0x18 * 3), D_ALL); + MMIO_D(_MMIO(0xec40c + 0x18 * 3), D_ALL); + MMIO_D(_MMIO(0xfc810), D_ALL); + MMIO_D(_MMIO(0xfc81c), D_ALL); + MMIO_D(_MMIO(0xfc828), D_ALL); + MMIO_D(_MMIO(0xfc834), D_ALL); + MMIO_D(_MMIO(0xfcc00), D_ALL); + MMIO_D(_MMIO(0xfcc0c), D_ALL); + MMIO_D(_MMIO(0xfcc18), D_ALL); + MMIO_D(_MMIO(0xfcc24), D_ALL); + MMIO_D(_MMIO(0xfd000), D_ALL); + MMIO_D(_MMIO(0xfd00c), D_ALL); + MMIO_D(_MMIO(0xfd018), D_ALL); + MMIO_D(_MMIO(0xfd024), D_ALL); + MMIO_D(_MMIO(0xfd034), D_ALL); MMIO_DH(FPGA_DBG, D_ALL, NULL, fpga_dbg_mmio_write); - MMIO_D(0x2054, D_ALL); - MMIO_D(0x12054, D_ALL); - MMIO_D(0x22054, D_ALL); - MMIO_D(0x1a054, D_ALL); - - MMIO_D(0x44070, D_ALL); - MMIO_DFH(0x215c, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x2178, D_ALL, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x217c, D_ALL, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x12178, D_ALL, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x1217c, D_ALL, F_CMD_ACCESS, NULL, NULL); - - MMIO_F(0x2290, 8, F_CMD_ACCESS, 0, 0, D_BDW_PLUS, NULL, NULL); - MMIO_D(0x2b00, D_BDW_PLUS); - MMIO_D(0x2360, D_BDW_PLUS); - MMIO_F(0x5200, 32, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); - MMIO_F(0x5240, 32, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); - MMIO_F(0x5280, 16, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); - - MMIO_DFH(0x1c17c, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x1c178, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); + MMIO_D(_MMIO(0x2054), D_ALL); + MMIO_D(_MMIO(0x12054), D_ALL); + MMIO_D(_MMIO(0x22054), D_ALL); + MMIO_D(_MMIO(0x1a054), D_ALL); + + MMIO_D(_MMIO(0x44070), D_ALL); + MMIO_DFH(_MMIO(0x215c), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x2178), D_ALL, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x217c), D_ALL, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x12178), D_ALL, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x1217c), D_ALL, F_CMD_ACCESS, NULL, NULL); + + MMIO_F(_MMIO(0x2290), 8, F_CMD_ACCESS, 0, 0, D_BDW_PLUS, NULL, NULL); + MMIO_D(_MMIO(0x2b00), D_BDW_PLUS); + MMIO_D(_MMIO(0x2360), D_BDW_PLUS); + MMIO_F(_MMIO(0x5200), 32, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); + MMIO_F(_MMIO(0x5240), 32, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); + MMIO_F(_MMIO(0x5280), 16, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); + + MMIO_DFH(_MMIO(0x1c17c), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x1c178), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); MMIO_DFH(BCS_SWCTRL, D_ALL, F_CMD_ACCESS, NULL, NULL); MMIO_F(HS_INVOCATION_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); @@ -2368,24 +2409,24 @@ static int init_generic_mmio_info(struct intel_gvt *gvt) MMIO_F(CL_PRIMITIVES_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); MMIO_F(PS_INVOCATION_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); MMIO_F(PS_DEPTH_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); - MMIO_DH(0x4260, D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler); - MMIO_DH(0x4264, D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler); - MMIO_DH(0x4268, D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler); - MMIO_DH(0x426c, D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler); - MMIO_DH(0x4270, D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler); - MMIO_DFH(0x4094, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); + MMIO_DH(_MMIO(0x4260), D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler); + MMIO_DH(_MMIO(0x4264), D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler); + MMIO_DH(_MMIO(0x4268), D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler); + MMIO_DH(_MMIO(0x426c), D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler); + MMIO_DH(_MMIO(0x4270), D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler); + MMIO_DFH(_MMIO(0x4094), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); MMIO_DFH(ARB_MODE, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); MMIO_RING_GM_RDR(RING_BBADDR, D_ALL, NULL, NULL); - MMIO_DFH(0x2220, D_ALL, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x12220, D_ALL, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x22220, D_ALL, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x2220), D_ALL, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x12220), D_ALL, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x22220), D_ALL, F_CMD_ACCESS, NULL, NULL); MMIO_RING_DFH(RING_SYNC_1, D_ALL, F_CMD_ACCESS, NULL, NULL); MMIO_RING_DFH(RING_SYNC_0, D_ALL, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x22178, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x1a178, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x1a17c, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x2217c, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x22178), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x1a178), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x1a17c), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x2217c), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); return 0; } @@ -2459,40 +2500,40 @@ static int init_broadwell_mmio_info(struct intel_gvt *gvt) MMIO_RING_DFH(RING_ACTHD_UDW, D_BDW_PLUS, F_CMD_ACCESS, mmio_read_from_hw, NULL); -#define RING_REG(base) (base + 0xd0) +#define RING_REG(base) _MMIO((base) + 0xd0) MMIO_RING_F(RING_REG, 4, F_RO, 0, ~_MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET), D_BDW_PLUS, NULL, ring_reset_ctl_write); #undef RING_REG -#define RING_REG(base) (base + 0x230) +#define RING_REG(base) _MMIO((base) + 0x230) MMIO_RING_DFH(RING_REG, D_BDW_PLUS, 0, NULL, elsp_mmio_write); #undef RING_REG -#define RING_REG(base) (base + 0x234) +#define RING_REG(base) _MMIO((base) + 0x234) MMIO_RING_F(RING_REG, 8, F_RO | F_CMD_ACCESS, 0, ~0, D_BDW_PLUS, NULL, NULL); #undef RING_REG -#define RING_REG(base) (base + 0x244) +#define RING_REG(base) _MMIO((base) + 0x244) MMIO_RING_DFH(RING_REG, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); #undef RING_REG -#define RING_REG(base) (base + 0x370) +#define RING_REG(base) _MMIO((base) + 0x370) MMIO_RING_F(RING_REG, 48, F_RO, 0, ~0, D_BDW_PLUS, NULL, NULL); #undef RING_REG -#define RING_REG(base) (base + 0x3a0) +#define RING_REG(base) _MMIO((base) + 0x3a0) MMIO_RING_DFH(RING_REG, D_BDW_PLUS, F_MODE_MASK, NULL, NULL); #undef RING_REG MMIO_D(PIPEMISC(PIPE_A), D_BDW_PLUS); MMIO_D(PIPEMISC(PIPE_B), D_BDW_PLUS); MMIO_D(PIPEMISC(PIPE_C), D_BDW_PLUS); - MMIO_D(0x1c1d0, D_BDW_PLUS); + MMIO_D(_MMIO(0x1c1d0), D_BDW_PLUS); MMIO_D(GEN6_MBCUNIT_SNPCR, D_BDW_PLUS); MMIO_D(GEN7_MISCCPCTL, D_BDW_PLUS); - MMIO_D(0x1c054, D_BDW_PLUS); + MMIO_D(_MMIO(0x1c054), D_BDW_PLUS); MMIO_DH(GEN6_PCODE_MAILBOX, D_BDW_PLUS, NULL, mailbox_write); @@ -2501,11 +2542,11 @@ static int init_broadwell_mmio_info(struct intel_gvt *gvt) MMIO_D(GAMTARBMODE, D_BDW_PLUS); -#define RING_REG(base) (base + 0x270) +#define RING_REG(base) _MMIO((base) + 0x270) MMIO_RING_F(RING_REG, 32, 0, 0, 0, D_BDW_PLUS, NULL, NULL); #undef RING_REG - MMIO_RING_GM_RDR(RING_HWS_PGA, D_BDW_PLUS, NULL, NULL); + MMIO_RING_GM_RDR(RING_HWS_PGA, D_BDW_PLUS, NULL, hws_pga_write); MMIO_DFH(HDC_CHICKEN0, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); @@ -2514,10 +2555,10 @@ static int init_broadwell_mmio_info(struct intel_gvt *gvt) MMIO_D(CHICKEN_PIPESL_1(PIPE_C), D_BDW_PLUS); MMIO_D(WM_MISC, D_BDW); - MMIO_D(BDW_EDP_PSR_BASE, D_BDW); + MMIO_D(_MMIO(BDW_EDP_PSR_BASE), D_BDW); - MMIO_D(0x66c00, D_BDW_PLUS); - MMIO_D(0x66c04, D_BDW_PLUS); + MMIO_D(_MMIO(0x66c00), D_BDW_PLUS); + MMIO_D(_MMIO(0x66c04), D_BDW_PLUS); MMIO_D(HSW_GTT_CACHE_EN, D_BDW_PLUS); @@ -2525,54 +2566,54 @@ static int init_broadwell_mmio_info(struct intel_gvt *gvt) MMIO_D(GEN8_EU_DISABLE1, D_BDW_PLUS); MMIO_D(GEN8_EU_DISABLE2, D_BDW_PLUS); - MMIO_D(0xfdc, D_BDW_PLUS); + MMIO_D(_MMIO(0xfdc), D_BDW_PLUS); MMIO_DFH(GEN8_ROW_CHICKEN, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); MMIO_DFH(GEN7_ROW_CHICKEN2, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); MMIO_DFH(GEN8_UCGCTL6, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0xb1f0, D_BDW, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0xb1c0, D_BDW, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0xb1f0), D_BDW, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0xb1c0), D_BDW, F_CMD_ACCESS, NULL, NULL); MMIO_DFH(GEN8_L3SQCREG4, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0xb100, D_BDW, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0xb10c, D_BDW, F_CMD_ACCESS, NULL, NULL); - MMIO_D(0xb110, D_BDW); + MMIO_DFH(_MMIO(0xb100), D_BDW, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0xb10c), D_BDW, F_CMD_ACCESS, NULL, NULL); + MMIO_D(_MMIO(0xb110), D_BDW); - MMIO_F(0x24d0, 48, F_CMD_ACCESS, 0, 0, D_BDW_PLUS, + MMIO_F(_MMIO(0x24d0), 48, F_CMD_ACCESS, 0, 0, D_BDW_PLUS, NULL, force_nonpriv_write); - MMIO_D(0x44484, D_BDW_PLUS); - MMIO_D(0x4448c, D_BDW_PLUS); + MMIO_D(_MMIO(0x44484), D_BDW_PLUS); + MMIO_D(_MMIO(0x4448c), D_BDW_PLUS); - MMIO_DFH(0x83a4, D_BDW, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x83a4), D_BDW, F_CMD_ACCESS, NULL, NULL); MMIO_D(GEN8_L3_LRA_1_GPGPU, D_BDW_PLUS); - MMIO_DFH(0x8430, D_BDW, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x8430), D_BDW, F_CMD_ACCESS, NULL, NULL); - MMIO_D(0x110000, D_BDW_PLUS); + MMIO_D(_MMIO(0x110000), D_BDW_PLUS); - MMIO_D(0x48400, D_BDW_PLUS); + MMIO_D(_MMIO(0x48400), D_BDW_PLUS); - MMIO_D(0x6e570, D_BDW_PLUS); - MMIO_D(0x65f10, D_BDW_PLUS); + MMIO_D(_MMIO(0x6e570), D_BDW_PLUS); + MMIO_D(_MMIO(0x65f10), D_BDW_PLUS); - MMIO_DFH(0xe194, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0xe188, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0xe194), D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0xe188), D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); MMIO_DFH(HALF_SLICE_CHICKEN2, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x2580, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); - - MMIO_DFH(0x2248, D_BDW, F_CMD_ACCESS, NULL, NULL); - - MMIO_DFH(0xe220, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0xe230, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0xe240, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0xe260, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0xe270, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0xe280, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0xe2a0, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0xe2b0, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0xe2c0, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x2580), D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); + + MMIO_DFH(_MMIO(0x2248), D_BDW, F_CMD_ACCESS, NULL, NULL); + + MMIO_DFH(_MMIO(0xe220), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0xe230), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0xe240), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0xe260), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0xe270), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0xe280), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0xe2a0), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0xe2b0), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0xe2c0), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); return 0; } @@ -2588,11 +2629,11 @@ static int init_skl_mmio_info(struct intel_gvt *gvt) MMIO_DH(FORCEWAKE_MEDIA_GEN9, D_SKL_PLUS, NULL, mul_force_wake_write); MMIO_DH(FORCEWAKE_ACK_MEDIA_GEN9, D_SKL_PLUS, NULL, NULL); - MMIO_F(_DPB_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_SKL_PLUS, NULL, + MMIO_F(_MMIO(_DPB_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_SKL_PLUS, NULL, dp_aux_ch_ctl_mmio_write); - MMIO_F(_DPC_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_SKL_PLUS, NULL, + MMIO_F(_MMIO(_DPC_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_SKL_PLUS, NULL, dp_aux_ch_ctl_mmio_write); - MMIO_F(_DPD_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_SKL_PLUS, NULL, + MMIO_F(_MMIO(_DPD_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_SKL_PLUS, NULL, dp_aux_ch_ctl_mmio_write); /* @@ -2603,26 +2644,26 @@ static int init_skl_mmio_info(struct intel_gvt *gvt) MMIO_DH(HSW_PWR_WELL_CTL_DRIVER(SKL_DISP_PW_MISC_IO), D_SKL_PLUS, NULL, skl_power_well_ctl_write); - MMIO_D(0xa210, D_SKL_PLUS); + MMIO_D(_MMIO(0xa210), D_SKL_PLUS); MMIO_D(GEN9_MEDIA_PG_IDLE_HYSTERESIS, D_SKL_PLUS); MMIO_D(GEN9_RENDER_PG_IDLE_HYSTERESIS, D_SKL_PLUS); MMIO_DFH(GEN9_GAMT_ECO_REG_RW_IA, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); - MMIO_DH(0x4ddc, D_SKL_PLUS, NULL, NULL); - MMIO_DH(0x42080, D_SKL_PLUS, NULL, NULL); - MMIO_D(0x45504, D_SKL_PLUS); - MMIO_D(0x45520, D_SKL_PLUS); - MMIO_D(0x46000, D_SKL_PLUS); - MMIO_DH(0x46010, D_SKL | D_KBL, NULL, skl_lcpll_write); - MMIO_DH(0x46014, D_SKL | D_KBL, NULL, skl_lcpll_write); - MMIO_D(0x6C040, D_SKL | D_KBL); - MMIO_D(0x6C048, D_SKL | D_KBL); - MMIO_D(0x6C050, D_SKL | D_KBL); - MMIO_D(0x6C044, D_SKL | D_KBL); - MMIO_D(0x6C04C, D_SKL | D_KBL); - MMIO_D(0x6C054, D_SKL | D_KBL); - MMIO_D(0x6c058, D_SKL | D_KBL); - MMIO_D(0x6c05c, D_SKL | D_KBL); - MMIO_DH(0X6c060, D_SKL | D_KBL, dpll_status_read, NULL); + MMIO_DH(_MMIO(0x4ddc), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(0x42080), D_SKL_PLUS, NULL, NULL); + MMIO_D(_MMIO(0x45504), D_SKL_PLUS); + MMIO_D(_MMIO(0x45520), D_SKL_PLUS); + MMIO_D(_MMIO(0x46000), D_SKL_PLUS); + MMIO_DH(_MMIO(0x46010), D_SKL | D_KBL, NULL, skl_lcpll_write); + MMIO_DH(_MMIO(0x46014), D_SKL | D_KBL, NULL, skl_lcpll_write); + MMIO_D(_MMIO(0x6C040), D_SKL | D_KBL); + MMIO_D(_MMIO(0x6C048), D_SKL | D_KBL); + MMIO_D(_MMIO(0x6C050), D_SKL | D_KBL); + MMIO_D(_MMIO(0x6C044), D_SKL | D_KBL); + MMIO_D(_MMIO(0x6C04C), D_SKL | D_KBL); + MMIO_D(_MMIO(0x6C054), D_SKL | D_KBL); + MMIO_D(_MMIO(0x6c058), D_SKL | D_KBL); + MMIO_D(_MMIO(0x6c05c), D_SKL | D_KBL); + MMIO_DH(_MMIO(0x6c060), D_SKL | D_KBL, dpll_status_read, NULL); MMIO_DH(SKL_PS_WIN_POS(PIPE_A, 0), D_SKL_PLUS, NULL, pf_write); MMIO_DH(SKL_PS_WIN_POS(PIPE_A, 1), D_SKL_PLUS, NULL, pf_write); @@ -2711,105 +2752,108 @@ static int init_skl_mmio_info(struct intel_gvt *gvt) MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_C, 2), D_SKL_PLUS, NULL, NULL); MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_C, 3), D_SKL_PLUS, NULL, NULL); - MMIO_DH(_REG_701C0(PIPE_A, 1), D_SKL_PLUS, NULL, NULL); - MMIO_DH(_REG_701C0(PIPE_A, 2), D_SKL_PLUS, NULL, NULL); - MMIO_DH(_REG_701C0(PIPE_A, 3), D_SKL_PLUS, NULL, NULL); - MMIO_DH(_REG_701C0(PIPE_A, 4), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(_REG_701C0(PIPE_A, 1)), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(_REG_701C0(PIPE_A, 2)), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(_REG_701C0(PIPE_A, 3)), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(_REG_701C0(PIPE_A, 4)), D_SKL_PLUS, NULL, NULL); - MMIO_DH(_REG_701C0(PIPE_B, 1), D_SKL_PLUS, NULL, NULL); - MMIO_DH(_REG_701C0(PIPE_B, 2), D_SKL_PLUS, NULL, NULL); - MMIO_DH(_REG_701C0(PIPE_B, 3), D_SKL_PLUS, NULL, NULL); - MMIO_DH(_REG_701C0(PIPE_B, 4), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(_REG_701C0(PIPE_B, 1)), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(_REG_701C0(PIPE_B, 2)), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(_REG_701C0(PIPE_B, 3)), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(_REG_701C0(PIPE_B, 4)), D_SKL_PLUS, NULL, NULL); - MMIO_DH(_REG_701C0(PIPE_C, 1), D_SKL_PLUS, NULL, NULL); - MMIO_DH(_REG_701C0(PIPE_C, 2), D_SKL_PLUS, NULL, NULL); - MMIO_DH(_REG_701C0(PIPE_C, 3), D_SKL_PLUS, NULL, NULL); - MMIO_DH(_REG_701C0(PIPE_C, 4), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(_REG_701C0(PIPE_C, 1)), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(_REG_701C0(PIPE_C, 2)), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(_REG_701C0(PIPE_C, 3)), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(_REG_701C0(PIPE_C, 4)), D_SKL_PLUS, NULL, NULL); - MMIO_DH(_REG_701C4(PIPE_A, 1), D_SKL_PLUS, NULL, NULL); - MMIO_DH(_REG_701C4(PIPE_A, 2), D_SKL_PLUS, NULL, NULL); - MMIO_DH(_REG_701C4(PIPE_A, 3), D_SKL_PLUS, NULL, NULL); - MMIO_DH(_REG_701C4(PIPE_A, 4), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(_REG_701C4(PIPE_A, 1)), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(_REG_701C4(PIPE_A, 2)), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(_REG_701C4(PIPE_A, 3)), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(_REG_701C4(PIPE_A, 4)), D_SKL_PLUS, NULL, NULL); - MMIO_DH(_REG_701C4(PIPE_B, 1), D_SKL_PLUS, NULL, NULL); - MMIO_DH(_REG_701C4(PIPE_B, 2), D_SKL_PLUS, NULL, NULL); - MMIO_DH(_REG_701C4(PIPE_B, 3), D_SKL_PLUS, NULL, NULL); - MMIO_DH(_REG_701C4(PIPE_B, 4), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(_REG_701C4(PIPE_B, 1)), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(_REG_701C4(PIPE_B, 2)), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(_REG_701C4(PIPE_B, 3)), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(_REG_701C4(PIPE_B, 4)), D_SKL_PLUS, NULL, NULL); - MMIO_DH(_REG_701C4(PIPE_C, 1), D_SKL_PLUS, NULL, NULL); - MMIO_DH(_REG_701C4(PIPE_C, 2), D_SKL_PLUS, NULL, NULL); - MMIO_DH(_REG_701C4(PIPE_C, 3), D_SKL_PLUS, NULL, NULL); - MMIO_DH(_REG_701C4(PIPE_C, 4), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(_REG_701C4(PIPE_C, 1)), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(_REG_701C4(PIPE_C, 2)), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(_REG_701C4(PIPE_C, 3)), D_SKL_PLUS, NULL, NULL); + MMIO_DH(_MMIO(_REG_701C4(PIPE_C, 4)), D_SKL_PLUS, NULL, NULL); - MMIO_D(0x70380, D_SKL_PLUS); - MMIO_D(0x71380, D_SKL_PLUS); - MMIO_D(0x72380, D_SKL_PLUS); - MMIO_D(0x7039c, D_SKL_PLUS); + MMIO_D(_MMIO(0x70380), D_SKL_PLUS); + MMIO_D(_MMIO(0x71380), D_SKL_PLUS); + MMIO_D(_MMIO(0x72380), D_SKL_PLUS); + MMIO_D(_MMIO(0x7039c), D_SKL_PLUS); - MMIO_D(0x8f074, D_SKL | D_KBL); - MMIO_D(0x8f004, D_SKL | D_KBL); - MMIO_D(0x8f034, D_SKL | D_KBL); + MMIO_D(_MMIO(0x8f074), D_SKL | D_KBL); + MMIO_D(_MMIO(0x8f004), D_SKL | D_KBL); + MMIO_D(_MMIO(0x8f034), D_SKL | D_KBL); - MMIO_D(0xb11c, D_SKL | D_KBL); + MMIO_D(_MMIO(0xb11c), D_SKL | D_KBL); - MMIO_D(0x51000, D_SKL | D_KBL); - MMIO_D(0x6c00c, D_SKL_PLUS); + MMIO_D(_MMIO(0x51000), D_SKL | D_KBL); + MMIO_D(_MMIO(0x6c00c), D_SKL_PLUS); - MMIO_F(0xc800, 0x7f8, F_CMD_ACCESS, 0, 0, D_SKL | D_KBL, NULL, NULL); - MMIO_F(0xb020, 0x80, F_CMD_ACCESS, 0, 0, D_SKL | D_KBL, NULL, NULL); + MMIO_F(_MMIO(0xc800), 0x7f8, F_CMD_ACCESS, 0, 0, D_SKL | D_KBL, NULL, NULL); + MMIO_F(_MMIO(0xb020), 0x80, F_CMD_ACCESS, 0, 0, D_SKL | D_KBL, NULL, NULL); - MMIO_D(0xd08, D_SKL_PLUS); - MMIO_DFH(0x20e0, D_SKL_PLUS, F_MODE_MASK, NULL, NULL); - MMIO_DFH(0x20ec, D_SKL_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); + MMIO_D(_MMIO(0xd08), D_SKL_PLUS); + MMIO_DFH(_MMIO(0x20e0), D_SKL_PLUS, F_MODE_MASK, NULL, NULL); + MMIO_DFH(_MMIO(0x20ec), D_SKL_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); /* TRTT */ - MMIO_DFH(0x4de0, D_SKL | D_KBL, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x4de4, D_SKL | D_KBL, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x4de8, D_SKL | D_KBL, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x4dec, D_SKL | D_KBL, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x4df0, D_SKL | D_KBL, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(0x4df4, D_SKL | D_KBL, F_CMD_ACCESS, NULL, gen9_trtte_write); - MMIO_DH(0x4dfc, D_SKL | D_KBL, NULL, gen9_trtt_chicken_write); + MMIO_DFH(_MMIO(0x4de0), D_SKL | D_KBL, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x4de4), D_SKL | D_KBL, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x4de8), D_SKL | D_KBL, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x4dec), D_SKL | D_KBL, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x4df0), D_SKL | D_KBL, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x4df4), D_SKL | D_KBL, F_CMD_ACCESS, NULL, gen9_trtte_write); + MMIO_DH(_MMIO(0x4dfc), D_SKL | D_KBL, NULL, gen9_trtt_chicken_write); - MMIO_D(0x45008, D_SKL | D_KBL); + MMIO_D(_MMIO(0x45008), D_SKL | D_KBL); - MMIO_D(0x46430, D_SKL | D_KBL); + MMIO_D(_MMIO(0x46430), D_SKL | D_KBL); - MMIO_D(0x46520, D_SKL | D_KBL); + MMIO_D(_MMIO(0x46520), D_SKL | D_KBL); - MMIO_D(0xc403c, D_SKL | D_KBL); - MMIO_D(0xb004, D_SKL_PLUS); + MMIO_D(_MMIO(0xc403c), D_SKL | D_KBL); + MMIO_D(_MMIO(0xb004), D_SKL_PLUS); MMIO_DH(DMA_CTRL, D_SKL_PLUS, NULL, dma_ctrl_write); - MMIO_D(0x65900, D_SKL_PLUS); - MMIO_D(0x1082c0, D_SKL | D_KBL); - MMIO_D(0x4068, D_SKL | D_KBL); - MMIO_D(0x67054, D_SKL | D_KBL); - MMIO_D(0x6e560, D_SKL | D_KBL); - MMIO_D(0x6e554, D_SKL | D_KBL); - MMIO_D(0x2b20, D_SKL | D_KBL); - MMIO_D(0x65f00, D_SKL | D_KBL); - MMIO_D(0x65f08, D_SKL | D_KBL); - MMIO_D(0x320f0, D_SKL | D_KBL); - - MMIO_D(0x70034, D_SKL_PLUS); - MMIO_D(0x71034, D_SKL_PLUS); - MMIO_D(0x72034, D_SKL_PLUS); - - MMIO_D(_PLANE_KEYVAL_1(PIPE_A), D_SKL_PLUS); - MMIO_D(_PLANE_KEYVAL_1(PIPE_B), D_SKL_PLUS); - MMIO_D(_PLANE_KEYVAL_1(PIPE_C), D_SKL_PLUS); - MMIO_D(_PLANE_KEYMSK_1(PIPE_A), D_SKL_PLUS); - MMIO_D(_PLANE_KEYMSK_1(PIPE_B), D_SKL_PLUS); - MMIO_D(_PLANE_KEYMSK_1(PIPE_C), D_SKL_PLUS); - - MMIO_D(0x44500, D_SKL_PLUS); + MMIO_D(_MMIO(0x65900), D_SKL_PLUS); + MMIO_D(_MMIO(0x1082c0), D_SKL | D_KBL); + MMIO_D(_MMIO(0x4068), D_SKL | D_KBL); + MMIO_D(_MMIO(0x67054), D_SKL | D_KBL); + MMIO_D(_MMIO(0x6e560), D_SKL | D_KBL); + MMIO_D(_MMIO(0x6e554), D_SKL | D_KBL); + MMIO_D(_MMIO(0x2b20), D_SKL | D_KBL); + MMIO_D(_MMIO(0x65f00), D_SKL | D_KBL); + MMIO_D(_MMIO(0x65f08), D_SKL | D_KBL); + MMIO_D(_MMIO(0x320f0), D_SKL | D_KBL); + + MMIO_D(_MMIO(0x70034), D_SKL_PLUS); + MMIO_D(_MMIO(0x71034), D_SKL_PLUS); + MMIO_D(_MMIO(0x72034), D_SKL_PLUS); + + MMIO_D(_MMIO(_PLANE_KEYVAL_1(PIPE_A)), D_SKL_PLUS); + MMIO_D(_MMIO(_PLANE_KEYVAL_1(PIPE_B)), D_SKL_PLUS); + MMIO_D(_MMIO(_PLANE_KEYVAL_1(PIPE_C)), D_SKL_PLUS); + MMIO_D(_MMIO(_PLANE_KEYMAX_1(PIPE_A)), D_SKL_PLUS); + MMIO_D(_MMIO(_PLANE_KEYMAX_1(PIPE_B)), D_SKL_PLUS); + MMIO_D(_MMIO(_PLANE_KEYMAX_1(PIPE_C)), D_SKL_PLUS); + MMIO_D(_MMIO(_PLANE_KEYMSK_1(PIPE_A)), D_SKL_PLUS); + MMIO_D(_MMIO(_PLANE_KEYMSK_1(PIPE_B)), D_SKL_PLUS); + MMIO_D(_MMIO(_PLANE_KEYMSK_1(PIPE_C)), D_SKL_PLUS); + + MMIO_D(_MMIO(0x44500), D_SKL_PLUS); MMIO_DFH(GEN9_CSFE_CHICKEN1_RCS, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); MMIO_DFH(GEN8_HDC_CHICKEN1, D_SKL | D_KBL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); - MMIO_D(0x4ab8, D_KBL); - MMIO_D(0x2248, D_SKL_PLUS | D_KBL); + MMIO_D(_MMIO(0x4ab8), D_KBL); + MMIO_D(_MMIO(0x2248), D_SKL_PLUS | D_KBL); return 0; } @@ -2825,8 +2869,8 @@ static struct gvt_mmio_block *find_mmio_block(struct intel_gvt *gvt, for (i = 0; i < num; i++, block++) { if (!(device & block->device)) continue; - if (offset >= INTEL_GVT_MMIO_OFFSET(block->offset) && - offset < INTEL_GVT_MMIO_OFFSET(block->offset) + block->size) + if (offset >= i915_mmio_reg_offset(block->offset) && + offset < i915_mmio_reg_offset(block->offset) + block->size) return block; } return NULL; @@ -2906,14 +2950,46 @@ int intel_gvt_setup_mmio_info(struct intel_gvt *gvt) gvt->mmio.mmio_block = mmio_blocks; gvt->mmio.num_mmio_block = ARRAY_SIZE(mmio_blocks); - gvt_dbg_mmio("traced %u virtual mmio registers\n", - gvt->mmio.num_tracked_mmio); return 0; err: intel_gvt_clean_mmio_info(gvt); return ret; } +/** + * intel_gvt_for_each_tracked_mmio - iterate each tracked mmio + * @gvt: a GVT device + * @handler: the handler + * @data: private data given to handler + * + * Returns: + * Zero on success, negative error code if failed. + */ +int intel_gvt_for_each_tracked_mmio(struct intel_gvt *gvt, + int (*handler)(struct intel_gvt *gvt, u32 offset, void *data), + void *data) +{ + struct gvt_mmio_block *block = gvt->mmio.mmio_block; + struct intel_gvt_mmio_info *e; + int i, j, ret; + + hash_for_each(gvt->mmio.mmio_info_table, i, e, node) { + ret = handler(gvt, e->offset, data); + if (ret) + return ret; + } + + for (i = 0; i < gvt->mmio.num_mmio_block; i++, block++) { + for (j = 0; j < block->size; j += 4) { + ret = handler(gvt, + i915_mmio_reg_offset(block->offset) + j, + data); + if (ret) + return ret; + } + } + return 0; +} /** * intel_vgpu_default_mmio_read - default MMIO read handler diff --git a/drivers/gpu/drm/i915/gvt/hypercall.h b/drivers/gpu/drm/i915/gvt/hypercall.h index df7f33abd393..f8e77e166246 100644 --- a/drivers/gpu/drm/i915/gvt/hypercall.h +++ b/drivers/gpu/drm/i915/gvt/hypercall.h @@ -55,6 +55,10 @@ struct intel_gvt_mpt { unsigned long mfn, unsigned int nr, bool map); int (*set_trap_area)(unsigned long handle, u64 start, u64 end, bool map); + int (*set_opregion)(void *vgpu); + int (*get_vfio_device)(void *vgpu); + void (*put_vfio_device)(void *vgpu); + bool (*is_valid_gfn)(unsigned long handle, unsigned long gfn); }; extern struct intel_gvt_mpt xengt_mpt; diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c index 96060920a6fe..021f722e2481 100644 --- a/drivers/gpu/drm/i915/gvt/kvmgt.c +++ b/drivers/gpu/drm/i915/gvt/kvmgt.c @@ -53,11 +53,23 @@ static const struct intel_gvt_ops *intel_gvt_ops; #define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT) #define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1) +#define OPREGION_SIGNATURE "IntelGraphicsMem" + +struct vfio_region; +struct intel_vgpu_regops { + size_t (*rw)(struct intel_vgpu *vgpu, char *buf, + size_t count, loff_t *ppos, bool iswrite); + void (*release)(struct intel_vgpu *vgpu, + struct vfio_region *region); +}; + struct vfio_region { u32 type; u32 subtype; size_t size; u32 flags; + const struct intel_vgpu_regops *ops; + void *data; }; struct kvmgt_pgfn { @@ -248,120 +260,6 @@ static void gvt_cache_destroy(struct intel_vgpu *vgpu) } } -static struct intel_vgpu_type *intel_gvt_find_vgpu_type(struct intel_gvt *gvt, - const char *name) -{ - int i; - struct intel_vgpu_type *t; - const char *driver_name = dev_driver_string( - &gvt->dev_priv->drm.pdev->dev); - - for (i = 0; i < gvt->num_types; i++) { - t = &gvt->types[i]; - if (!strncmp(t->name, name + strlen(driver_name) + 1, - sizeof(t->name))) - return t; - } - - return NULL; -} - -static ssize_t available_instances_show(struct kobject *kobj, - struct device *dev, char *buf) -{ - struct intel_vgpu_type *type; - unsigned int num = 0; - void *gvt = kdev_to_i915(dev)->gvt; - - type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj)); - if (!type) - num = 0; - else - num = type->avail_instance; - - return sprintf(buf, "%u\n", num); -} - -static ssize_t device_api_show(struct kobject *kobj, struct device *dev, - char *buf) -{ - return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING); -} - -static ssize_t description_show(struct kobject *kobj, struct device *dev, - char *buf) -{ - struct intel_vgpu_type *type; - void *gvt = kdev_to_i915(dev)->gvt; - - type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj)); - if (!type) - return 0; - - return sprintf(buf, "low_gm_size: %dMB\nhigh_gm_size: %dMB\n" - "fence: %d\nresolution: %s\n" - "weight: %d\n", - BYTES_TO_MB(type->low_gm_size), - BYTES_TO_MB(type->high_gm_size), - type->fence, vgpu_edid_str(type->resolution), - type->weight); -} - -static MDEV_TYPE_ATTR_RO(available_instances); -static MDEV_TYPE_ATTR_RO(device_api); -static MDEV_TYPE_ATTR_RO(description); - -static struct attribute *type_attrs[] = { - &mdev_type_attr_available_instances.attr, - &mdev_type_attr_device_api.attr, - &mdev_type_attr_description.attr, - NULL, -}; - -static struct attribute_group *intel_vgpu_type_groups[] = { - [0 ... NR_MAX_INTEL_VGPU_TYPES - 1] = NULL, -}; - -static bool intel_gvt_init_vgpu_type_groups(struct intel_gvt *gvt) -{ - int i, j; - struct intel_vgpu_type *type; - struct attribute_group *group; - - for (i = 0; i < gvt->num_types; i++) { - type = &gvt->types[i]; - - group = kzalloc(sizeof(struct attribute_group), GFP_KERNEL); - if (WARN_ON(!group)) - goto unwind; - - group->name = type->name; - group->attrs = type_attrs; - intel_vgpu_type_groups[i] = group; - } - - return true; - -unwind: - for (j = 0; j < i; j++) { - group = intel_vgpu_type_groups[j]; - kfree(group); - } - - return false; -} - -static void intel_gvt_cleanup_vgpu_type_groups(struct intel_gvt *gvt) -{ - int i; - struct attribute_group *group; - - for (i = 0; i < gvt->num_types; i++) { - group = intel_vgpu_type_groups[i]; - kfree(group); - } -} - static void kvmgt_protect_table_init(struct kvmgt_guest_info *info) { hash_init(info->ptable); @@ -430,6 +328,108 @@ static void kvmgt_protect_table_del(struct kvmgt_guest_info *info, } } +static size_t intel_vgpu_reg_rw_opregion(struct intel_vgpu *vgpu, char *buf, + size_t count, loff_t *ppos, bool iswrite) +{ + unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) - + VFIO_PCI_NUM_REGIONS; + void *base = vgpu->vdev.region[i].data; + loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK; + + if (pos >= vgpu->vdev.region[i].size || iswrite) { + gvt_vgpu_err("invalid op or offset for Intel vgpu OpRegion\n"); + return -EINVAL; + } + count = min(count, (size_t)(vgpu->vdev.region[i].size - pos)); + memcpy(buf, base + pos, count); + + return count; +} + +static void intel_vgpu_reg_release_opregion(struct intel_vgpu *vgpu, + struct vfio_region *region) +{ +} + +static const struct intel_vgpu_regops intel_vgpu_regops_opregion = { + .rw = intel_vgpu_reg_rw_opregion, + .release = intel_vgpu_reg_release_opregion, +}; + +static int intel_vgpu_register_reg(struct intel_vgpu *vgpu, + unsigned int type, unsigned int subtype, + const struct intel_vgpu_regops *ops, + size_t size, u32 flags, void *data) +{ + struct vfio_region *region; + + region = krealloc(vgpu->vdev.region, + (vgpu->vdev.num_regions + 1) * sizeof(*region), + GFP_KERNEL); + if (!region) + return -ENOMEM; + + vgpu->vdev.region = region; + vgpu->vdev.region[vgpu->vdev.num_regions].type = type; + vgpu->vdev.region[vgpu->vdev.num_regions].subtype = subtype; + vgpu->vdev.region[vgpu->vdev.num_regions].ops = ops; + vgpu->vdev.region[vgpu->vdev.num_regions].size = size; + vgpu->vdev.region[vgpu->vdev.num_regions].flags = flags; + vgpu->vdev.region[vgpu->vdev.num_regions].data = data; + vgpu->vdev.num_regions++; + return 0; +} + +static int kvmgt_get_vfio_device(void *p_vgpu) +{ + struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu; + + vgpu->vdev.vfio_device = vfio_device_get_from_dev( + mdev_dev(vgpu->vdev.mdev)); + if (!vgpu->vdev.vfio_device) { + gvt_vgpu_err("failed to get vfio device\n"); + return -ENODEV; + } + return 0; +} + + +static int kvmgt_set_opregion(void *p_vgpu) +{ + struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu; + void *base; + int ret; + + /* Each vgpu has its own opregion, although VFIO would create another + * one later. This one is used to expose opregion to VFIO. And the + * other one created by VFIO later, is used by guest actually. + */ + base = vgpu_opregion(vgpu)->va; + if (!base) + return -ENOMEM; + + if (memcmp(base, OPREGION_SIGNATURE, 16)) { + memunmap(base); + return -EINVAL; + } + + ret = intel_vgpu_register_reg(vgpu, + PCI_VENDOR_ID_INTEL | VFIO_REGION_TYPE_PCI_VENDOR_TYPE, + VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION, + &intel_vgpu_regops_opregion, OPREGION_SIZE, + VFIO_REGION_INFO_FLAG_READ, base); + + return ret; +} + +static void kvmgt_put_vfio_device(void *vgpu) +{ + if (WARN_ON(!((struct intel_vgpu *)vgpu)->vdev.vfio_device)) + return; + + vfio_device_put(((struct intel_vgpu *)vgpu)->vdev.vfio_device); +} + static int intel_vgpu_create(struct kobject *kobj, struct mdev_device *mdev) { struct intel_vgpu *vgpu = NULL; @@ -441,7 +441,7 @@ static int intel_vgpu_create(struct kobject *kobj, struct mdev_device *mdev) pdev = mdev_parent_dev(mdev); gvt = kdev_to_i915(pdev)->gvt; - type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj)); + type = intel_gvt_ops->gvt_find_vgpu_type(gvt, kobject_name(kobj)); if (!type) { gvt_vgpu_err("failed to find type %s to create\n", kobject_name(kobj)); @@ -651,6 +651,39 @@ static int intel_vgpu_bar_rw(struct intel_vgpu *vgpu, int bar, uint64_t off, return ret; } +static inline bool intel_vgpu_in_aperture(struct intel_vgpu *vgpu, uint64_t off) +{ + return off >= vgpu_aperture_offset(vgpu) && + off < vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu); +} + +static int intel_vgpu_aperture_rw(struct intel_vgpu *vgpu, uint64_t off, + void *buf, unsigned long count, bool is_write) +{ + void *aperture_va; + + if (!intel_vgpu_in_aperture(vgpu, off) || + !intel_vgpu_in_aperture(vgpu, off + count)) { + gvt_vgpu_err("Invalid aperture offset %llu\n", off); + return -EINVAL; + } + + aperture_va = io_mapping_map_wc(&vgpu->gvt->dev_priv->ggtt.iomap, + ALIGN_DOWN(off, PAGE_SIZE), + count + offset_in_page(off)); + if (!aperture_va) + return -EIO; + + if (is_write) + memcpy(aperture_va + offset_in_page(off), buf, count); + else + memcpy(buf, aperture_va + offset_in_page(off), count); + + io_mapping_unmap(aperture_va); + + return 0; +} + static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf, size_t count, loff_t *ppos, bool is_write) { @@ -660,7 +693,7 @@ static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf, int ret = -EINVAL; - if (index >= VFIO_PCI_NUM_REGIONS) { + if (index >= VFIO_PCI_NUM_REGIONS + vgpu->vdev.num_regions) { gvt_vgpu_err("invalid index: %u\n", index); return -EINVAL; } @@ -679,8 +712,7 @@ static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf, buf, count, is_write); break; case VFIO_PCI_BAR2_REGION_INDEX: - ret = intel_vgpu_bar_rw(vgpu, PCI_BASE_ADDRESS_2, pos, - buf, count, is_write); + ret = intel_vgpu_aperture_rw(vgpu, pos, buf, count, is_write); break; case VFIO_PCI_BAR1_REGION_INDEX: case VFIO_PCI_BAR3_REGION_INDEX: @@ -688,13 +720,38 @@ static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf, case VFIO_PCI_BAR5_REGION_INDEX: case VFIO_PCI_VGA_REGION_INDEX: case VFIO_PCI_ROM_REGION_INDEX: + break; default: - gvt_vgpu_err("unsupported region: %u\n", index); + if (index >= VFIO_PCI_NUM_REGIONS + vgpu->vdev.num_regions) + return -EINVAL; + + index -= VFIO_PCI_NUM_REGIONS; + return vgpu->vdev.region[index].ops->rw(vgpu, buf, count, + ppos, is_write); } return ret == 0 ? count : ret; } +static bool gtt_entry(struct mdev_device *mdev, loff_t *ppos) +{ + struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); + unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); + struct intel_gvt *gvt = vgpu->gvt; + int offset; + + /* Only allow MMIO GGTT entry access */ + if (index != PCI_BASE_ADDRESS_0) + return false; + + offset = (u64)(*ppos & VFIO_PCI_OFFSET_MASK) - + intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_0); + + return (offset >= gvt->device_info.gtt_start_offset && + offset < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt)) ? + true : false; +} + static ssize_t intel_vgpu_read(struct mdev_device *mdev, char __user *buf, size_t count, loff_t *ppos) { @@ -704,7 +761,21 @@ static ssize_t intel_vgpu_read(struct mdev_device *mdev, char __user *buf, while (count) { size_t filled; - if (count >= 4 && !(*ppos % 4)) { + /* Only support GGTT entry 8 bytes read */ + if (count >= 8 && !(*ppos % 8) && + gtt_entry(mdev, ppos)) { + u64 val; + + ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), + ppos, false); + if (ret <= 0) + goto read_err; + + if (copy_to_user(buf, &val, sizeof(val))) + goto read_err; + + filled = 8; + } else if (count >= 4 && !(*ppos % 4)) { u32 val; ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), @@ -764,7 +835,21 @@ static ssize_t intel_vgpu_write(struct mdev_device *mdev, while (count) { size_t filled; - if (count >= 4 && !(*ppos % 4)) { + /* Only support GGTT entry 8 bytes write */ + if (count >= 8 && !(*ppos % 8) && + gtt_entry(mdev, ppos)) { + u64 val; + + if (copy_from_user(&val, buf, sizeof(val))) + goto write_err; + + ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), + ppos, true); + if (ret <= 0) + goto write_err; + + filled = 8; + } else if (count >= 4 && !(*ppos % 4)) { u32 val; if (copy_from_user(&val, buf, sizeof(val))) @@ -952,7 +1037,8 @@ static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd, info.flags = VFIO_DEVICE_FLAGS_PCI; info.flags |= VFIO_DEVICE_FLAGS_RESET; - info.num_regions = VFIO_PCI_NUM_REGIONS; + info.num_regions = VFIO_PCI_NUM_REGIONS + + vgpu->vdev.num_regions; info.num_irqs = VFIO_PCI_NUM_IRQS; return copy_to_user((void __user *)arg, &info, minsz) ? @@ -1012,6 +1098,8 @@ static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd, if (!sparse) return -ENOMEM; + sparse->header.id = VFIO_REGION_INFO_CAP_SPARSE_MMAP; + sparse->header.version = 1; sparse->nr_areas = nr_areas; cap_type_id = VFIO_REGION_INFO_CAP_SPARSE_MMAP; sparse->areas[0].offset = @@ -1022,18 +1110,24 @@ static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd, case VFIO_PCI_BAR3_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); info.size = 0; - info.flags = 0; + gvt_dbg_core("get region info bar:%d\n", info.index); break; case VFIO_PCI_ROM_REGION_INDEX: case VFIO_PCI_VGA_REGION_INDEX: + info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); + info.size = 0; + info.flags = 0; + gvt_dbg_core("get region info index:%d\n", info.index); break; default: { - struct vfio_region_info_cap_type cap_type; + struct vfio_region_info_cap_type cap_type = { + .header.id = VFIO_REGION_INFO_CAP_TYPE, + .header.version = 1 }; if (info.index >= VFIO_PCI_NUM_REGIONS + vgpu->vdev.num_regions) @@ -1050,8 +1144,8 @@ static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd, cap_type.subtype = vgpu->vdev.region[i].subtype; ret = vfio_info_add_capability(&caps, - VFIO_REGION_INFO_CAP_TYPE, - &cap_type); + &cap_type.header, + sizeof(cap_type)); if (ret) return ret; } @@ -1061,8 +1155,9 @@ static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd, switch (cap_type_id) { case VFIO_REGION_INFO_CAP_SPARSE_MMAP: ret = vfio_info_add_capability(&caps, - VFIO_REGION_INFO_CAP_SPARSE_MMAP, - sparse); + &sparse->header, sizeof(*sparse) + + (sparse->nr_areas * + sizeof(*sparse->areas))); kfree(sparse); if (ret) return ret; @@ -1073,6 +1168,7 @@ static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd, } if (caps.size) { + info.flags |= VFIO_REGION_INFO_FLAG_CAPS; if (info.argsz < sizeof(info) + caps.size) { info.argsz = sizeof(info) + caps.size; info.cap_offset = 0; @@ -1159,6 +1255,33 @@ static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd, } else if (cmd == VFIO_DEVICE_RESET) { intel_gvt_ops->vgpu_reset(vgpu); return 0; + } else if (cmd == VFIO_DEVICE_QUERY_GFX_PLANE) { + struct vfio_device_gfx_plane_info dmabuf; + int ret = 0; + + minsz = offsetofend(struct vfio_device_gfx_plane_info, + dmabuf_id); + if (copy_from_user(&dmabuf, (void __user *)arg, minsz)) + return -EFAULT; + if (dmabuf.argsz < minsz) + return -EINVAL; + + ret = intel_gvt_ops->vgpu_query_plane(vgpu, &dmabuf); + if (ret != 0) + return ret; + + return copy_to_user((void __user *)arg, &dmabuf, minsz) ? + -EFAULT : 0; + } else if (cmd == VFIO_DEVICE_GET_GFX_DMABUF) { + __u32 dmabuf_id; + __s32 dmabuf_fd; + + if (get_user(dmabuf_id, (__u32 __user *)arg)) + return -EFAULT; + + dmabuf_fd = intel_gvt_ops->vgpu_get_dmabuf(vgpu, dmabuf_id); + return dmabuf_fd; + } return 0; @@ -1188,7 +1311,7 @@ hw_id_show(struct device *dev, struct device_attribute *attr, struct intel_vgpu *vgpu = (struct intel_vgpu *) mdev_get_drvdata(mdev); return sprintf(buf, "%u\n", - vgpu->shadow_ctx->hw_id); + vgpu->submission.shadow_ctx->hw_id); } return sprintf(buf, "\n"); } @@ -1212,8 +1335,7 @@ static const struct attribute_group *intel_vgpu_groups[] = { NULL, }; -static const struct mdev_parent_ops intel_vgpu_ops = { - .supported_type_groups = intel_vgpu_type_groups, +static struct mdev_parent_ops intel_vgpu_ops = { .mdev_attr_groups = intel_vgpu_groups, .create = intel_vgpu_create, .remove = intel_vgpu_remove, @@ -1229,17 +1351,20 @@ static const struct mdev_parent_ops intel_vgpu_ops = { static int kvmgt_host_init(struct device *dev, void *gvt, const void *ops) { - if (!intel_gvt_init_vgpu_type_groups(gvt)) - return -EFAULT; + struct attribute **kvm_type_attrs; + struct attribute_group **kvm_vgpu_type_groups; intel_gvt_ops = ops; + if (!intel_gvt_ops->get_gvt_attrs(&kvm_type_attrs, + &kvm_vgpu_type_groups)) + return -EFAULT; + intel_vgpu_ops.supported_type_groups = kvm_vgpu_type_groups; return mdev_register_device(dev, &intel_vgpu_ops); } static void kvmgt_host_exit(struct device *dev, void *gvt) { - intel_gvt_cleanup_vgpu_type_groups(gvt); mdev_unregister_device(dev); } @@ -1319,8 +1444,8 @@ static void kvmgt_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, struct kvmgt_guest_info, track_node); if (kvmgt_gfn_is_write_protected(info, gpa_to_gfn(gpa))) - intel_gvt_ops->emulate_mmio_write(info->vgpu, gpa, - (void *)val, len); + intel_gvt_ops->write_protect_handler(info->vgpu, gpa, + (void *)val, len); } static void kvmgt_page_track_flush_slot(struct kvm *kvm, @@ -1398,6 +1523,9 @@ static int kvmgt_guest_init(struct mdev_device *mdev) kvmgt_protect_table_init(info); gvt_cache_init(vgpu); + mutex_init(&vgpu->dmabuf_lock); + init_completion(&vgpu->vblank_done); + info->track_node.track_write = kvmgt_page_track_write; info->track_node.track_flush_slot = kvmgt_page_track_flush_slot; kvm_page_track_register_notifier(kvm, &info->track_node); @@ -1526,6 +1654,21 @@ static unsigned long kvmgt_virt_to_pfn(void *addr) return PFN_DOWN(__pa(addr)); } +static bool kvmgt_is_valid_gfn(unsigned long handle, unsigned long gfn) +{ + struct kvmgt_guest_info *info; + struct kvm *kvm; + + if (!handle_valid(handle)) + return false; + + info = (struct kvmgt_guest_info *)handle; + kvm = info->kvm; + + return kvm_is_visible_gfn(kvm, gfn); + +} + struct intel_gvt_mpt kvmgt_mpt = { .host_init = kvmgt_host_init, .host_exit = kvmgt_host_exit, @@ -1538,6 +1681,10 @@ struct intel_gvt_mpt kvmgt_mpt = { .read_gpa = kvmgt_read_gpa, .write_gpa = kvmgt_write_gpa, .gfn_to_mfn = kvmgt_gfn_to_pfn, + .set_opregion = kvmgt_set_opregion, + .get_vfio_device = kvmgt_get_vfio_device, + .put_vfio_device = kvmgt_put_vfio_device, + .is_valid_gfn = kvmgt_is_valid_gfn, }; EXPORT_SYMBOL_GPL(kvmgt_mpt); diff --git a/drivers/gpu/drm/i915/gvt/mmio.c b/drivers/gpu/drm/i915/gvt/mmio.c index 1e1310f50289..5c869e3fdf3b 100644 --- a/drivers/gpu/drm/i915/gvt/mmio.c +++ b/drivers/gpu/drm/i915/gvt/mmio.c @@ -56,38 +56,6 @@ int intel_vgpu_gpa_to_mmio_offset(struct intel_vgpu *vgpu, u64 gpa) (reg >= gvt->device_info.gtt_start_offset \ && reg < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt)) -static bool vgpu_gpa_is_aperture(struct intel_vgpu *vgpu, uint64_t gpa) -{ - u64 aperture_gpa = intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_2); - u64 aperture_sz = vgpu_aperture_sz(vgpu); - - return gpa >= aperture_gpa && gpa < aperture_gpa + aperture_sz; -} - -static int vgpu_aperture_rw(struct intel_vgpu *vgpu, uint64_t gpa, - void *pdata, unsigned int size, bool is_read) -{ - u64 aperture_gpa = intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_2); - u64 offset = gpa - aperture_gpa; - - if (!vgpu_gpa_is_aperture(vgpu, gpa + size - 1)) { - gvt_vgpu_err("Aperture rw out of range, offset %llx, size %d\n", - offset, size); - return -EINVAL; - } - - if (!vgpu->gm.aperture_va) { - gvt_vgpu_err("BAR is not enabled\n"); - return -ENXIO; - } - - if (is_read) - memcpy(pdata, vgpu->gm.aperture_va + offset, size); - else - memcpy(vgpu->gm.aperture_va + offset, pdata, size); - return 0; -} - static void failsafe_emulate_mmio_rw(struct intel_vgpu *vgpu, uint64_t pa, void *p_data, unsigned int bytes, bool read) { @@ -117,25 +85,6 @@ static void failsafe_emulate_mmio_rw(struct intel_vgpu *vgpu, uint64_t pa, else memcpy(pt, p_data, bytes); - } else if (atomic_read(&vgpu->gtt.n_write_protected_guest_page)) { - struct intel_vgpu_guest_page *gp; - - /* Since we enter the failsafe mode early during guest boot, - * guest may not have chance to set up its ppgtt table, so - * there should not be any wp pages for guest. Keep the wp - * related code here in case we need to handle it in furture. - */ - gp = intel_vgpu_find_guest_page(vgpu, pa >> PAGE_SHIFT); - if (gp) { - /* remove write protection to prevent furture traps */ - intel_vgpu_clean_guest_page(vgpu, gp); - if (read) - intel_gvt_hypervisor_read_gpa(vgpu, pa, - p_data, bytes); - else - intel_gvt_hypervisor_write_gpa(vgpu, pa, - p_data, bytes); - } } mutex_unlock(&gvt->lock); } @@ -157,37 +106,12 @@ int intel_vgpu_emulate_mmio_read(struct intel_vgpu *vgpu, uint64_t pa, unsigned int offset = 0; int ret = -EINVAL; - if (vgpu->failsafe) { failsafe_emulate_mmio_rw(vgpu, pa, p_data, bytes, true); return 0; } mutex_lock(&gvt->lock); - if (vgpu_gpa_is_aperture(vgpu, pa)) { - ret = vgpu_aperture_rw(vgpu, pa, p_data, bytes, true); - mutex_unlock(&gvt->lock); - return ret; - } - - if (atomic_read(&vgpu->gtt.n_write_protected_guest_page)) { - struct intel_vgpu_guest_page *gp; - - gp = intel_vgpu_find_guest_page(vgpu, pa >> PAGE_SHIFT); - if (gp) { - ret = intel_gvt_hypervisor_read_gpa(vgpu, pa, - p_data, bytes); - if (ret) { - gvt_vgpu_err("guest page read error %d, " - "gfn 0x%lx, pa 0x%llx, var 0x%x, len %d\n", - ret, gp->gfn, pa, *(u32 *)p_data, - bytes); - } - mutex_unlock(&gvt->lock); - return ret; - } - } - offset = intel_vgpu_gpa_to_mmio_offset(vgpu, pa); if (WARN_ON(bytes > 8)) @@ -205,14 +129,12 @@ int intel_vgpu_emulate_mmio_read(struct intel_vgpu *vgpu, uint64_t pa, p_data, bytes); if (ret) goto err; - mutex_unlock(&gvt->lock); - return ret; + goto out; } if (WARN_ON_ONCE(!reg_is_mmio(gvt, offset))) { ret = intel_gvt_hypervisor_read_gpa(vgpu, pa, p_data, bytes); - mutex_unlock(&gvt->lock); - return ret; + goto out; } if (WARN_ON(!reg_is_mmio(gvt, offset + bytes - 1))) @@ -228,11 +150,13 @@ int intel_vgpu_emulate_mmio_read(struct intel_vgpu *vgpu, uint64_t pa, goto err; intel_gvt_mmio_set_accessed(gvt, offset); - mutex_unlock(&gvt->lock); - return 0; + ret = 0; + goto out; + err: gvt_vgpu_err("fail to emulate MMIO read %08x len %d\n", offset, bytes); +out: mutex_unlock(&gvt->lock); return ret; } @@ -261,30 +185,6 @@ int intel_vgpu_emulate_mmio_write(struct intel_vgpu *vgpu, uint64_t pa, mutex_lock(&gvt->lock); - if (vgpu_gpa_is_aperture(vgpu, pa)) { - ret = vgpu_aperture_rw(vgpu, pa, p_data, bytes, false); - mutex_unlock(&gvt->lock); - return ret; - } - - if (atomic_read(&vgpu->gtt.n_write_protected_guest_page)) { - struct intel_vgpu_guest_page *gp; - - gp = intel_vgpu_find_guest_page(vgpu, pa >> PAGE_SHIFT); - if (gp) { - ret = gp->handler(gp, pa, p_data, bytes); - if (ret) { - gvt_err("guest page write error %d, " - "gfn 0x%lx, pa 0x%llx, " - "var 0x%x, len %d\n", - ret, gp->gfn, pa, - *(u32 *)p_data, bytes); - } - mutex_unlock(&gvt->lock); - return ret; - } - } - offset = intel_vgpu_gpa_to_mmio_offset(vgpu, pa); if (WARN_ON(bytes > 8)) @@ -302,14 +202,12 @@ int intel_vgpu_emulate_mmio_write(struct intel_vgpu *vgpu, uint64_t pa, p_data, bytes); if (ret) goto err; - mutex_unlock(&gvt->lock); - return ret; + goto out; } if (WARN_ON_ONCE(!reg_is_mmio(gvt, offset))) { ret = intel_gvt_hypervisor_write_gpa(vgpu, pa, p_data, bytes); - mutex_unlock(&gvt->lock); - return ret; + goto out; } ret = intel_vgpu_mmio_reg_rw(vgpu, offset, p_data, bytes, false); @@ -317,11 +215,12 @@ int intel_vgpu_emulate_mmio_write(struct intel_vgpu *vgpu, uint64_t pa, goto err; intel_gvt_mmio_set_accessed(gvt, offset); - mutex_unlock(&gvt->lock); - return 0; + ret = 0; + goto out; err: gvt_vgpu_err("fail to emulate MMIO write %08x len %d\n", offset, bytes); +out: mutex_unlock(&gvt->lock); return ret; } @@ -342,10 +241,10 @@ void intel_vgpu_reset_mmio(struct intel_vgpu *vgpu, bool dmlr) memcpy(vgpu->mmio.vreg, mmio, info->mmio_size); memcpy(vgpu->mmio.sreg, mmio, info->mmio_size); - vgpu_vreg(vgpu, GEN6_GT_THREAD_STATUS_REG) = 0; + vgpu_vreg_t(vgpu, GEN6_GT_THREAD_STATUS_REG) = 0; /* set the bit 0:2(Core C-State ) to C0 */ - vgpu_vreg(vgpu, GEN6_GT_CORE_STATUS) = 0; + vgpu_vreg_t(vgpu, GEN6_GT_CORE_STATUS) = 0; vgpu->mmio.disable_warn_untrack = false; } else { diff --git a/drivers/gpu/drm/i915/gvt/mmio.h b/drivers/gpu/drm/i915/gvt/mmio.h index dbc04ad2c7a1..71b620875943 100644 --- a/drivers/gpu/drm/i915/gvt/mmio.h +++ b/drivers/gpu/drm/i915/gvt/mmio.h @@ -72,12 +72,9 @@ bool intel_gvt_match_device(struct intel_gvt *gvt, unsigned long device); int intel_gvt_setup_mmio_info(struct intel_gvt *gvt); void intel_gvt_clean_mmio_info(struct intel_gvt *gvt); - -#define INTEL_GVT_MMIO_OFFSET(reg) ({ \ - typeof(reg) __reg = reg; \ - u32 *offset = (u32 *)&__reg; \ - *offset; \ -}) +int intel_gvt_for_each_tracked_mmio(struct intel_gvt *gvt, + int (*handler)(struct intel_gvt *gvt, u32 offset, void *data), + void *data); int intel_vgpu_init_mmio(struct intel_vgpu *vgpu); void intel_vgpu_reset_mmio(struct intel_vgpu *vgpu, bool dmlr); diff --git a/drivers/gpu/drm/i915/gvt/mmio_context.c b/drivers/gpu/drm/i915/gvt/mmio_context.c new file mode 100644 index 000000000000..256f1bb522b7 --- /dev/null +++ b/drivers/gpu/drm/i915/gvt/mmio_context.c @@ -0,0 +1,413 @@ +/* + * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Authors: + * Eddie Dong <eddie.dong@intel.com> + * Kevin Tian <kevin.tian@intel.com> + * + * Contributors: + * Zhi Wang <zhi.a.wang@intel.com> + * Changbin Du <changbin.du@intel.com> + * Zhenyu Wang <zhenyuw@linux.intel.com> + * Tina Zhang <tina.zhang@intel.com> + * Bing Niu <bing.niu@intel.com> + * + */ + +#include "i915_drv.h" +#include "gvt.h" +#include "trace.h" + +/** + * Defined in Intel Open Source PRM. + * Ref: https://01.org/linuxgraphics/documentation/hardware-specification-prms + */ +#define TRVATTL3PTRDW(i) _MMIO(0x4de0 + (i)*4) +#define TRNULLDETCT _MMIO(0x4de8) +#define TRINVTILEDETCT _MMIO(0x4dec) +#define TRVADR _MMIO(0x4df0) +#define TRTTE _MMIO(0x4df4) +#define RING_EXCC(base) _MMIO((base) + 0x28) +#define RING_GFX_MODE(base) _MMIO((base) + 0x29c) +#define VF_GUARDBAND _MMIO(0x83a4) + +/* Raw offset is appened to each line for convenience. */ +static struct engine_mmio gen8_engine_mmio_list[] __cacheline_aligned = { + {RCS, GFX_MODE_GEN7, 0xffff, false}, /* 0x229c */ + {RCS, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */ + {RCS, HWSTAM, 0x0, false}, /* 0x2098 */ + {RCS, INSTPM, 0xffff, true}, /* 0x20c0 */ + {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 0), 0, false}, /* 0x24d0 */ + {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 1), 0, false}, /* 0x24d4 */ + {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 2), 0, false}, /* 0x24d8 */ + {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 3), 0, false}, /* 0x24dc */ + {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 4), 0, false}, /* 0x24e0 */ + {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 5), 0, false}, /* 0x24e4 */ + {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 6), 0, false}, /* 0x24e8 */ + {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 7), 0, false}, /* 0x24ec */ + {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 8), 0, false}, /* 0x24f0 */ + {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 9), 0, false}, /* 0x24f4 */ + {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 10), 0, false}, /* 0x24f8 */ + {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 11), 0, false}, /* 0x24fc */ + {RCS, CACHE_MODE_1, 0xffff, true}, /* 0x7004 */ + {RCS, GEN7_GT_MODE, 0xffff, true}, /* 0x7008 */ + {RCS, CACHE_MODE_0_GEN7, 0xffff, true}, /* 0x7000 */ + {RCS, GEN7_COMMON_SLICE_CHICKEN1, 0xffff, true}, /* 0x7010 */ + {RCS, HDC_CHICKEN0, 0xffff, true}, /* 0x7300 */ + {RCS, VF_GUARDBAND, 0xffff, true}, /* 0x83a4 */ + + {BCS, RING_GFX_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2229c */ + {BCS, RING_MI_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2209c */ + {BCS, RING_INSTPM(BLT_RING_BASE), 0xffff, false}, /* 0x220c0 */ + {BCS, RING_HWSTAM(BLT_RING_BASE), 0x0, false}, /* 0x22098 */ + {BCS, RING_EXCC(BLT_RING_BASE), 0x0, false}, /* 0x22028 */ + {RCS, INVALID_MMIO_REG, 0, false } /* Terminated */ +}; + +static struct engine_mmio gen9_engine_mmio_list[] __cacheline_aligned = { + {RCS, GFX_MODE_GEN7, 0xffff, false}, /* 0x229c */ + {RCS, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */ + {RCS, HWSTAM, 0x0, false}, /* 0x2098 */ + {RCS, INSTPM, 0xffff, true}, /* 0x20c0 */ + {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 0), 0, false}, /* 0x24d0 */ + {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 1), 0, false}, /* 0x24d4 */ + {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 2), 0, false}, /* 0x24d8 */ + {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 3), 0, false}, /* 0x24dc */ + {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 4), 0, false}, /* 0x24e0 */ + {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 5), 0, false}, /* 0x24e4 */ + {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 6), 0, false}, /* 0x24e8 */ + {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 7), 0, false}, /* 0x24ec */ + {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 8), 0, false}, /* 0x24f0 */ + {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 9), 0, false}, /* 0x24f4 */ + {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 10), 0, false}, /* 0x24f8 */ + {RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 11), 0, false}, /* 0x24fc */ + {RCS, CACHE_MODE_1, 0xffff, true}, /* 0x7004 */ + {RCS, GEN7_GT_MODE, 0xffff, true}, /* 0x7008 */ + {RCS, CACHE_MODE_0_GEN7, 0xffff, true}, /* 0x7000 */ + {RCS, GEN7_COMMON_SLICE_CHICKEN1, 0xffff, true}, /* 0x7010 */ + {RCS, HDC_CHICKEN0, 0xffff, true}, /* 0x7300 */ + {RCS, VF_GUARDBAND, 0xffff, true}, /* 0x83a4 */ + + {RCS, GEN8_PRIVATE_PAT_LO, 0, false}, /* 0x40e0 */ + {RCS, GEN8_PRIVATE_PAT_HI, 0, false}, /* 0x40e4 */ + {RCS, GEN8_CS_CHICKEN1, 0xffff, true}, /* 0x2580 */ + {RCS, COMMON_SLICE_CHICKEN2, 0xffff, true}, /* 0x7014 */ + {RCS, GEN9_CS_DEBUG_MODE1, 0xffff, false}, /* 0x20ec */ + {RCS, GEN8_L3SQCREG4, 0, false}, /* 0xb118 */ + {RCS, GEN7_HALF_SLICE_CHICKEN1, 0xffff, true}, /* 0xe100 */ + {RCS, HALF_SLICE_CHICKEN2, 0xffff, true}, /* 0xe180 */ + {RCS, HALF_SLICE_CHICKEN3, 0xffff, true}, /* 0xe184 */ + {RCS, GEN9_HALF_SLICE_CHICKEN5, 0xffff, true}, /* 0xe188 */ + {RCS, GEN9_HALF_SLICE_CHICKEN7, 0xffff, true}, /* 0xe194 */ + {RCS, GEN8_ROW_CHICKEN, 0xffff, true}, /* 0xe4f0 */ + {RCS, TRVATTL3PTRDW(0), 0, false}, /* 0x4de0 */ + {RCS, TRVATTL3PTRDW(1), 0, false}, /* 0x4de4 */ + {RCS, TRNULLDETCT, 0, false}, /* 0x4de8 */ + {RCS, TRINVTILEDETCT, 0, false}, /* 0x4dec */ + {RCS, TRVADR, 0, false}, /* 0x4df0 */ + {RCS, TRTTE, 0, false}, /* 0x4df4 */ + + {BCS, RING_GFX_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2229c */ + {BCS, RING_MI_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2209c */ + {BCS, RING_INSTPM(BLT_RING_BASE), 0xffff, false}, /* 0x220c0 */ + {BCS, RING_HWSTAM(BLT_RING_BASE), 0x0, false}, /* 0x22098 */ + {BCS, RING_EXCC(BLT_RING_BASE), 0x0, false}, /* 0x22028 */ + + {VCS2, RING_EXCC(GEN8_BSD2_RING_BASE), 0xffff, false}, /* 0x1c028 */ + + {VECS, RING_EXCC(VEBOX_RING_BASE), 0xffff, false}, /* 0x1a028 */ + + {RCS, GEN8_HDC_CHICKEN1, 0xffff, true}, /* 0x7304 */ + {RCS, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */ + {RCS, GEN7_UCGCTL4, 0x0, false}, /* 0x940c */ + {RCS, GAMT_CHKN_BIT_REG, 0x0, false}, /* 0x4ab8 */ + + {RCS, GEN9_GAMT_ECO_REG_RW_IA, 0x0, false}, /* 0x4ab0 */ + {RCS, GEN9_CSFE_CHICKEN1_RCS, 0x0, false}, /* 0x20d4 */ + + {RCS, GEN8_GARBCNTL, 0x0, false}, /* 0xb004 */ + {RCS, GEN7_FF_THREAD_MODE, 0x0, false}, /* 0x20a0 */ + {RCS, FF_SLICE_CS_CHICKEN2, 0xffff, false}, /* 0x20e4 */ + {RCS, INVALID_MMIO_REG, 0, false } /* Terminated */ +}; + +static struct { + bool initialized; + u32 control_table[I915_NUM_ENGINES][64]; + u32 l3cc_table[32]; +} gen9_render_mocs; + +static void load_render_mocs(struct drm_i915_private *dev_priv) +{ + i915_reg_t offset; + u32 regs[] = { + [RCS] = 0xc800, + [VCS] = 0xc900, + [VCS2] = 0xca00, + [BCS] = 0xcc00, + [VECS] = 0xcb00, + }; + int ring_id, i; + + for (ring_id = 0; ring_id < ARRAY_SIZE(regs); ring_id++) { + offset.reg = regs[ring_id]; + for (i = 0; i < 64; i++) { + gen9_render_mocs.control_table[ring_id][i] = + I915_READ_FW(offset); + offset.reg += 4; + } + } + + offset.reg = 0xb020; + for (i = 0; i < 32; i++) { + gen9_render_mocs.l3cc_table[i] = + I915_READ_FW(offset); + offset.reg += 4; + } + gen9_render_mocs.initialized = true; +} + +static void handle_tlb_pending_event(struct intel_vgpu *vgpu, int ring_id) +{ + struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; + struct intel_vgpu_submission *s = &vgpu->submission; + enum forcewake_domains fw; + i915_reg_t reg; + u32 regs[] = { + [RCS] = 0x4260, + [VCS] = 0x4264, + [VCS2] = 0x4268, + [BCS] = 0x426c, + [VECS] = 0x4270, + }; + + if (WARN_ON(ring_id >= ARRAY_SIZE(regs))) + return; + + if (!test_and_clear_bit(ring_id, (void *)s->tlb_handle_pending)) + return; + + reg = _MMIO(regs[ring_id]); + + /* WaForceWakeRenderDuringMmioTLBInvalidate:skl + * we need to put a forcewake when invalidating RCS TLB caches, + * otherwise device can go to RC6 state and interrupt invalidation + * process + */ + fw = intel_uncore_forcewake_for_reg(dev_priv, reg, + FW_REG_READ | FW_REG_WRITE); + if (ring_id == RCS && (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))) + fw |= FORCEWAKE_RENDER; + + intel_uncore_forcewake_get(dev_priv, fw); + + I915_WRITE_FW(reg, 0x1); + + if (wait_for_atomic((I915_READ_FW(reg) == 0), 50)) + gvt_vgpu_err("timeout in invalidate ring (%d) tlb\n", ring_id); + else + vgpu_vreg_t(vgpu, reg) = 0; + + intel_uncore_forcewake_put(dev_priv, fw); + + gvt_dbg_core("invalidate TLB for ring %d\n", ring_id); +} + +static void switch_mocs(struct intel_vgpu *pre, struct intel_vgpu *next, + int ring_id) +{ + struct drm_i915_private *dev_priv; + i915_reg_t offset, l3_offset; + u32 old_v, new_v; + + u32 regs[] = { + [RCS] = 0xc800, + [VCS] = 0xc900, + [VCS2] = 0xca00, + [BCS] = 0xcc00, + [VECS] = 0xcb00, + }; + int i; + + dev_priv = pre ? pre->gvt->dev_priv : next->gvt->dev_priv; + if (WARN_ON(ring_id >= ARRAY_SIZE(regs))) + return; + + if (!pre && !gen9_render_mocs.initialized) + load_render_mocs(dev_priv); + + offset.reg = regs[ring_id]; + for (i = 0; i < 64; i++) { + if (pre) + old_v = vgpu_vreg_t(pre, offset); + else + old_v = gen9_render_mocs.control_table[ring_id][i]; + if (next) + new_v = vgpu_vreg_t(next, offset); + else + new_v = gen9_render_mocs.control_table[ring_id][i]; + + if (old_v != new_v) + I915_WRITE_FW(offset, new_v); + + offset.reg += 4; + } + + if (ring_id == RCS) { + l3_offset.reg = 0xb020; + for (i = 0; i < 32; i++) { + if (pre) + old_v = vgpu_vreg_t(pre, l3_offset); + else + old_v = gen9_render_mocs.l3cc_table[i]; + if (next) + new_v = vgpu_vreg_t(next, l3_offset); + else + new_v = gen9_render_mocs.l3cc_table[i]; + + if (old_v != new_v) + I915_WRITE_FW(l3_offset, new_v); + + l3_offset.reg += 4; + } + } +} + +#define CTX_CONTEXT_CONTROL_VAL 0x03 + +/* Switch ring mmio values (context). */ +static void switch_mmio(struct intel_vgpu *pre, + struct intel_vgpu *next, + int ring_id) +{ + struct drm_i915_private *dev_priv; + struct intel_vgpu_submission *s; + u32 *reg_state, ctx_ctrl; + u32 inhibit_mask = + _MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT); + struct engine_mmio *mmio; + u32 old_v, new_v; + + dev_priv = pre ? pre->gvt->dev_priv : next->gvt->dev_priv; + if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) + switch_mocs(pre, next, ring_id); + + for (mmio = dev_priv->gvt->engine_mmio_list; + i915_mmio_reg_valid(mmio->reg); mmio++) { + if (mmio->ring_id != ring_id) + continue; + // save + if (pre) { + vgpu_vreg_t(pre, mmio->reg) = I915_READ_FW(mmio->reg); + if (mmio->mask) + vgpu_vreg_t(pre, mmio->reg) &= + ~(mmio->mask << 16); + old_v = vgpu_vreg_t(pre, mmio->reg); + } else + old_v = mmio->value = I915_READ_FW(mmio->reg); + + // restore + if (next) { + s = &next->submission; + reg_state = + s->shadow_ctx->engine[ring_id].lrc_reg_state; + ctx_ctrl = reg_state[CTX_CONTEXT_CONTROL_VAL]; + /* + * if it is an inhibit context, load in_context mmio + * into HW by mmio write. If it is not, skip this mmio + * write. + */ + if (mmio->in_context && + (ctx_ctrl & inhibit_mask) != inhibit_mask) + continue; + + if (mmio->mask) + new_v = vgpu_vreg_t(next, mmio->reg) | + (mmio->mask << 16); + else + new_v = vgpu_vreg_t(next, mmio->reg); + } else { + if (mmio->in_context) + continue; + if (mmio->mask) + new_v = mmio->value | (mmio->mask << 16); + else + new_v = mmio->value; + } + + I915_WRITE_FW(mmio->reg, new_v); + + trace_render_mmio(pre ? pre->id : 0, + next ? next->id : 0, + "switch", + i915_mmio_reg_offset(mmio->reg), + old_v, new_v); + } + + if (next) + handle_tlb_pending_event(next, ring_id); +} + +/** + * intel_gvt_switch_render_mmio - switch mmio context of specific engine + * @pre: the last vGPU that own the engine + * @next: the vGPU to switch to + * @ring_id: specify the engine + * + * If pre is null indicates that host own the engine. If next is null + * indicates that we are switching to host workload. + */ +void intel_gvt_switch_mmio(struct intel_vgpu *pre, + struct intel_vgpu *next, int ring_id) +{ + struct drm_i915_private *dev_priv; + + if (WARN_ON(!pre && !next)) + return; + + gvt_dbg_render("switch ring %d from %s to %s\n", ring_id, + pre ? "vGPU" : "host", next ? "vGPU" : "HOST"); + + dev_priv = pre ? pre->gvt->dev_priv : next->gvt->dev_priv; + + /** + * We are using raw mmio access wrapper to improve the + * performace for batch mmio read/write, so we need + * handle forcewake mannually. + */ + intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); + switch_mmio(pre, next, ring_id); + intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); +} + +/** + * intel_gvt_init_engine_mmio_context - Initiate the engine mmio list + * @gvt: GVT device + * + */ +void intel_gvt_init_engine_mmio_context(struct intel_gvt *gvt) +{ + if (IS_SKYLAKE(gvt->dev_priv) || IS_KABYLAKE(gvt->dev_priv)) + gvt->engine_mmio_list = gen9_engine_mmio_list; + else + gvt->engine_mmio_list = gen8_engine_mmio_list; +} diff --git a/drivers/gpu/drm/i915/gvt/render.h b/drivers/gpu/drm/i915/gvt/mmio_context.h index 91db1d39d28f..ca2c6a745673 100644 --- a/drivers/gpu/drm/i915/gvt/render.h +++ b/drivers/gpu/drm/i915/gvt/mmio_context.h @@ -36,8 +36,17 @@ #ifndef __GVT_RENDER_H__ #define __GVT_RENDER_H__ +struct engine_mmio { + int ring_id; + i915_reg_t reg; + u32 mask; + bool in_context; + u32 value; +}; + void intel_gvt_switch_mmio(struct intel_vgpu *pre, struct intel_vgpu *next, int ring_id); +void intel_gvt_init_engine_mmio_context(struct intel_gvt *gvt); #endif diff --git a/drivers/gpu/drm/i915/gvt/mpt.h b/drivers/gpu/drm/i915/gvt/mpt.h index f0e5487e6688..81aff4eacbfe 100644 --- a/drivers/gpu/drm/i915/gvt/mpt.h +++ b/drivers/gpu/drm/i915/gvt/mpt.h @@ -154,51 +154,53 @@ static inline unsigned long intel_gvt_hypervisor_virt_to_mfn(void *p) } /** - * intel_gvt_hypervisor_set_wp_page - set a guest page to write-protected + * intel_gvt_hypervisor_enable - set a guest page to write-protected * @vgpu: a vGPU - * @p: intel_vgpu_guest_page + * @t: page track data structure * * Returns: * Zero on success, negative error code if failed. */ -static inline int intel_gvt_hypervisor_set_wp_page(struct intel_vgpu *vgpu, - struct intel_vgpu_guest_page *p) +static inline int intel_gvt_hypervisor_enable_page_track( + struct intel_vgpu *vgpu, + struct intel_vgpu_page_track *t) { int ret; - if (p->writeprotection) + if (t->tracked) return 0; - ret = intel_gvt_host.mpt->set_wp_page(vgpu->handle, p->gfn); + ret = intel_gvt_host.mpt->set_wp_page(vgpu->handle, t->gfn); if (ret) return ret; - p->writeprotection = true; - atomic_inc(&vgpu->gtt.n_write_protected_guest_page); + t->tracked = true; + atomic_inc(&vgpu->gtt.n_tracked_guest_page); return 0; } /** - * intel_gvt_hypervisor_unset_wp_page - remove the write-protection of a + * intel_gvt_hypervisor_disable_page_track - remove the write-protection of a * guest page * @vgpu: a vGPU - * @p: intel_vgpu_guest_page + * @t: page track data structure * * Returns: * Zero on success, negative error code if failed. */ -static inline int intel_gvt_hypervisor_unset_wp_page(struct intel_vgpu *vgpu, - struct intel_vgpu_guest_page *p) +static inline int intel_gvt_hypervisor_disable_page_track( + struct intel_vgpu *vgpu, + struct intel_vgpu_page_track *t) { int ret; - if (!p->writeprotection) + if (!t->tracked) return 0; - ret = intel_gvt_host.mpt->unset_wp_page(vgpu->handle, p->gfn); + ret = intel_gvt_host.mpt->unset_wp_page(vgpu->handle, t->gfn); if (ret) return ret; - p->writeprotection = false; - atomic_dec(&vgpu->gtt.n_write_protected_guest_page); + t->tracked = false; + atomic_dec(&vgpu->gtt.n_tracked_guest_page); return 0; } @@ -292,4 +294,66 @@ static inline int intel_gvt_hypervisor_set_trap_area( return intel_gvt_host.mpt->set_trap_area(vgpu->handle, start, end, map); } +/** + * intel_gvt_hypervisor_set_opregion - Set opregion for guest + * @vgpu: a vGPU + * + * Returns: + * Zero on success, negative error code if failed. + */ +static inline int intel_gvt_hypervisor_set_opregion(struct intel_vgpu *vgpu) +{ + if (!intel_gvt_host.mpt->set_opregion) + return 0; + + return intel_gvt_host.mpt->set_opregion(vgpu); +} + +/** + * intel_gvt_hypervisor_get_vfio_device - increase vfio device ref count + * @vgpu: a vGPU + * + * Returns: + * Zero on success, negative error code if failed. + */ +static inline int intel_gvt_hypervisor_get_vfio_device(struct intel_vgpu *vgpu) +{ + if (!intel_gvt_host.mpt->get_vfio_device) + return 0; + + return intel_gvt_host.mpt->get_vfio_device(vgpu); +} + +/** + * intel_gvt_hypervisor_put_vfio_device - decrease vfio device ref count + * @vgpu: a vGPU + * + * Returns: + * Zero on success, negative error code if failed. + */ +static inline void intel_gvt_hypervisor_put_vfio_device(struct intel_vgpu *vgpu) +{ + if (!intel_gvt_host.mpt->put_vfio_device) + return; + + intel_gvt_host.mpt->put_vfio_device(vgpu); +} + +/** + * intel_gvt_hypervisor_is_valid_gfn - check if a visible gfn + * @vgpu: a vGPU + * @gfn: guest PFN + * + * Returns: + * true on valid gfn, false on not. + */ +static inline bool intel_gvt_hypervisor_is_valid_gfn( + struct intel_vgpu *vgpu, unsigned long gfn) +{ + if (!intel_gvt_host.mpt->is_valid_gfn) + return true; + + return intel_gvt_host.mpt->is_valid_gfn(vgpu->handle, gfn); +} + #endif /* _GVT_MPT_H_ */ diff --git a/drivers/gpu/drm/i915/gvt/opregion.c b/drivers/gpu/drm/i915/gvt/opregion.c index 311799136d7f..fa75a2eead90 100644 --- a/drivers/gpu/drm/i915/gvt/opregion.c +++ b/drivers/gpu/drm/i915/gvt/opregion.c @@ -25,36 +25,237 @@ #include "i915_drv.h" #include "gvt.h" -static int init_vgpu_opregion(struct intel_vgpu *vgpu, u32 gpa) +/* + * Note: Only for GVT-g virtual VBT generation, other usage must + * not do like this. + */ +#define _INTEL_BIOS_PRIVATE +#include "intel_vbt_defs.h" + +#define OPREGION_SIGNATURE "IntelGraphicsMem" +#define MBOX_VBT (1<<3) + +/* device handle */ +#define DEVICE_TYPE_CRT 0x01 +#define DEVICE_TYPE_EFP1 0x04 +#define DEVICE_TYPE_EFP2 0x40 +#define DEVICE_TYPE_EFP3 0x20 +#define DEVICE_TYPE_EFP4 0x10 + +#define DEV_SIZE 38 + +struct opregion_header { + u8 signature[16]; + u32 size; + u32 opregion_ver; + u8 bios_ver[32]; + u8 vbios_ver[16]; + u8 driver_ver[16]; + u32 mboxes; + u32 driver_model; + u32 pcon; + u8 dver[32]; + u8 rsvd[124]; +} __packed; + +struct bdb_data_header { + u8 id; + u16 size; /* data size */ +} __packed; + +struct efp_child_device_config { + u16 handle; + u16 device_type; + u16 device_class; + u8 i2c_speed; + u8 dp_onboard_redriver; /* 158 */ + u8 dp_ondock_redriver; /* 158 */ + u8 hdmi_level_shifter_value:4; /* 169 */ + u8 hdmi_max_data_rate:4; /* 204 */ + u16 dtd_buf_ptr; /* 161 */ + u8 edidless_efp:1; /* 161 */ + u8 compression_enable:1; /* 198 */ + u8 compression_method:1; /* 198 */ + u8 ganged_edp:1; /* 202 */ + u8 skip0:4; + u8 compression_structure_index:4; /* 198 */ + u8 skip1:4; + u8 slave_port; /* 202 */ + u8 skip2; + u8 dvo_port; + u8 i2c_pin; /* for add-in card */ + u8 slave_addr; /* for add-in card */ + u8 ddc_pin; + u16 edid_ptr; + u8 dvo_config; + u8 efp_docked_port:1; /* 158 */ + u8 lane_reversal:1; /* 184 */ + u8 onboard_lspcon:1; /* 192 */ + u8 iboost_enable:1; /* 196 */ + u8 hpd_invert:1; /* BXT 196 */ + u8 slip3:3; + u8 hdmi_compat:1; + u8 dp_compat:1; + u8 tmds_compat:1; + u8 skip4:5; + u8 aux_channel; + u8 dongle_detect; + u8 pipe_cap:2; + u8 sdvo_stall:1; /* 158 */ + u8 hpd_status:2; + u8 integrated_encoder:1; + u8 skip5:2; + u8 dvo_wiring; + u8 mipi_bridge_type; /* 171 */ + u16 device_class_ext; + u8 dvo_function; + u8 dp_usb_type_c:1; /* 195 */ + u8 skip6:7; + u8 dp_usb_type_c_2x_gpio_index; /* 195 */ + u16 dp_usb_type_c_2x_gpio_pin; /* 195 */ + u8 iboost_dp:4; /* 196 */ + u8 iboost_hdmi:4; /* 196 */ +} __packed; + +struct vbt { + /* header->bdb_offset point to bdb_header offset */ + struct vbt_header header; + struct bdb_header bdb_header; + + struct bdb_data_header general_features_header; + struct bdb_general_features general_features; + + struct bdb_data_header general_definitions_header; + struct bdb_general_definitions general_definitions; + + struct efp_child_device_config child0; + struct efp_child_device_config child1; + struct efp_child_device_config child2; + struct efp_child_device_config child3; + + struct bdb_data_header driver_features_header; + struct bdb_driver_features driver_features; +}; + +static void virt_vbt_generation(struct vbt *v) { - u8 *buf; - int i; + int num_child; + + memset(v, 0, sizeof(struct vbt)); + + v->header.signature[0] = '$'; + v->header.signature[1] = 'V'; + v->header.signature[2] = 'B'; + v->header.signature[3] = 'T'; + + /* there's features depending on version! */ + v->header.version = 155; + v->header.header_size = sizeof(v->header); + v->header.vbt_size = sizeof(struct vbt) - sizeof(v->header); + v->header.bdb_offset = offsetof(struct vbt, bdb_header); + + strcpy(&v->bdb_header.signature[0], "BIOS_DATA_BLOCK"); + v->bdb_header.version = 186; /* child_dev_size = 38 */ + v->bdb_header.header_size = sizeof(v->bdb_header); + + v->bdb_header.bdb_size = sizeof(struct vbt) - sizeof(struct vbt_header) + - sizeof(struct bdb_header); + + /* general features */ + v->general_features_header.id = BDB_GENERAL_FEATURES; + v->general_features_header.size = sizeof(struct bdb_general_features); + v->general_features.int_crt_support = 0; + v->general_features.int_tv_support = 0; + + /* child device */ + num_child = 4; /* each port has one child */ + v->general_definitions_header.id = BDB_GENERAL_DEFINITIONS; + /* size will include child devices */ + v->general_definitions_header.size = + sizeof(struct bdb_general_definitions) + num_child * DEV_SIZE; + v->general_definitions.child_dev_size = DEV_SIZE; + + /* portA */ + v->child0.handle = DEVICE_TYPE_EFP1; + v->child0.device_type = DEVICE_TYPE_DP; + v->child0.dvo_port = DVO_PORT_DPA; + v->child0.aux_channel = DP_AUX_A; + v->child0.dp_compat = true; + v->child0.integrated_encoder = true; + + /* portB */ + v->child1.handle = DEVICE_TYPE_EFP2; + v->child1.device_type = DEVICE_TYPE_DP; + v->child1.dvo_port = DVO_PORT_DPB; + v->child1.aux_channel = DP_AUX_B; + v->child1.dp_compat = true; + v->child1.integrated_encoder = true; + + /* portC */ + v->child2.handle = DEVICE_TYPE_EFP3; + v->child2.device_type = DEVICE_TYPE_DP; + v->child2.dvo_port = DVO_PORT_DPC; + v->child2.aux_channel = DP_AUX_C; + v->child2.dp_compat = true; + v->child2.integrated_encoder = true; + + /* portD */ + v->child3.handle = DEVICE_TYPE_EFP4; + v->child3.device_type = DEVICE_TYPE_DP; + v->child3.dvo_port = DVO_PORT_DPD; + v->child3.aux_channel = DP_AUX_D; + v->child3.dp_compat = true; + v->child3.integrated_encoder = true; + + /* driver features */ + v->driver_features_header.id = BDB_DRIVER_FEATURES; + v->driver_features_header.size = sizeof(struct bdb_driver_features); + v->driver_features.lvds_config = BDB_DRIVER_FEATURE_NO_LVDS; +} - if (WARN((vgpu_opregion(vgpu)->va), - "vgpu%d: opregion has been initialized already.\n", - vgpu->id)) - return -EINVAL; +/** + * intel_vgpu_init_opregion - initialize the stuff used to emulate opregion + * @vgpu: a vGPU + * @gpa: guest physical address of opregion + * + * Returns: + * Zero on success, negative error code if failed. + */ +int intel_vgpu_init_opregion(struct intel_vgpu *vgpu) +{ + u8 *buf; + struct opregion_header *header; + struct vbt v; + const char opregion_signature[16] = OPREGION_SIGNATURE; + gvt_dbg_core("init vgpu%d opregion\n", vgpu->id); vgpu_opregion(vgpu)->va = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, get_order(INTEL_GVT_OPREGION_SIZE)); - - if (!vgpu_opregion(vgpu)->va) + if (!vgpu_opregion(vgpu)->va) { + gvt_err("fail to get memory for vgpu virt opregion\n"); return -ENOMEM; + } - memcpy(vgpu_opregion(vgpu)->va, vgpu->gvt->opregion.opregion_va, - INTEL_GVT_OPREGION_SIZE); - - for (i = 0; i < INTEL_GVT_OPREGION_PAGES; i++) - vgpu_opregion(vgpu)->gfn[i] = (gpa >> PAGE_SHIFT) + i; + /* emulated opregion with VBT mailbox only */ + buf = (u8 *)vgpu_opregion(vgpu)->va; + header = (struct opregion_header *)buf; + memcpy(header->signature, opregion_signature, + sizeof(opregion_signature)); + header->size = 0x8; + header->opregion_ver = 0x02000000; + header->mboxes = MBOX_VBT; /* for unknown reason, the value in LID field is incorrect * which block the windows guest, so workaround it by force * setting it to "OPEN" */ - buf = (u8 *)vgpu_opregion(vgpu)->va; buf[INTEL_GVT_OPREGION_CLID] = 0x3; + /* emulated vbt from virt vbt generation */ + virt_vbt_generation(&v); + memcpy(buf + INTEL_GVT_OPREGION_VBT_OFFSET, &v, sizeof(struct vbt)); + return 0; } @@ -79,93 +280,80 @@ static int map_vgpu_opregion(struct intel_vgpu *vgpu, bool map) return ret; } } - return 0; -} -/** - * intel_vgpu_clean_opregion - clean the stuff used to emulate opregion - * @vgpu: a vGPU - * - */ -void intel_vgpu_clean_opregion(struct intel_vgpu *vgpu) -{ - gvt_dbg_core("vgpu%d: clean vgpu opregion\n", vgpu->id); - - if (!vgpu_opregion(vgpu)->va) - return; - - if (intel_gvt_host.hypervisor_type == INTEL_GVT_HYPERVISOR_XEN) { - map_vgpu_opregion(vgpu, false); - free_pages((unsigned long)vgpu_opregion(vgpu)->va, - get_order(INTEL_GVT_OPREGION_SIZE)); + vgpu_opregion(vgpu)->mapped = map; - vgpu_opregion(vgpu)->va = NULL; - } + return 0; } /** - * intel_vgpu_init_opregion - initialize the stuff used to emulate opregion + * intel_vgpu_opregion_base_write_handler - Opregion base register write handler + * * @vgpu: a vGPU * @gpa: guest physical address of opregion * * Returns: * Zero on success, negative error code if failed. */ -int intel_vgpu_init_opregion(struct intel_vgpu *vgpu, u32 gpa) +int intel_vgpu_opregion_base_write_handler(struct intel_vgpu *vgpu, u32 gpa) { - int ret; - gvt_dbg_core("vgpu%d: init vgpu opregion\n", vgpu->id); + int i, ret = 0; - if (intel_gvt_host.hypervisor_type == INTEL_GVT_HYPERVISOR_XEN) { - gvt_dbg_core("emulate opregion from kernel\n"); + gvt_dbg_core("emulate opregion from kernel\n"); - ret = init_vgpu_opregion(vgpu, gpa); - if (ret) - return ret; + switch (intel_gvt_host.hypervisor_type) { + case INTEL_GVT_HYPERVISOR_KVM: + for (i = 0; i < INTEL_GVT_OPREGION_PAGES; i++) + vgpu_opregion(vgpu)->gfn[i] = (gpa >> PAGE_SHIFT) + i; + break; + case INTEL_GVT_HYPERVISOR_XEN: + /** + * Wins guest on Xengt will write this register twice: xen + * hvmloader and windows graphic driver. + */ + if (vgpu_opregion(vgpu)->mapped) + map_vgpu_opregion(vgpu, false); + + for (i = 0; i < INTEL_GVT_OPREGION_PAGES; i++) + vgpu_opregion(vgpu)->gfn[i] = (gpa >> PAGE_SHIFT) + i; ret = map_vgpu_opregion(vgpu, true); - if (ret) - return ret; + break; + default: + ret = -EINVAL; + gvt_vgpu_err("not supported hypervisor\n"); } - return 0; -} - -/** - * intel_gvt_clean_opregion - clean host opergion related stuffs - * @gvt: a GVT device - * - */ -void intel_gvt_clean_opregion(struct intel_gvt *gvt) -{ - memunmap(gvt->opregion.opregion_va); - gvt->opregion.opregion_va = NULL; + return ret; } /** - * intel_gvt_init_opregion - initialize host opergion related stuffs - * @gvt: a GVT device + * intel_vgpu_clean_opregion - clean the stuff used to emulate opregion + * @vgpu: a vGPU * - * Returns: - * Zero on success, negative error code if failed. */ -int intel_gvt_init_opregion(struct intel_gvt *gvt) +void intel_vgpu_clean_opregion(struct intel_vgpu *vgpu) { - gvt_dbg_core("init host opregion\n"); + gvt_dbg_core("vgpu%d: clean vgpu opregion\n", vgpu->id); - pci_read_config_dword(gvt->dev_priv->drm.pdev, INTEL_GVT_PCI_OPREGION, - &gvt->opregion.opregion_pa); + if (!vgpu_opregion(vgpu)->va) + return; - gvt->opregion.opregion_va = memremap(gvt->opregion.opregion_pa, - INTEL_GVT_OPREGION_SIZE, MEMREMAP_WB); - if (!gvt->opregion.opregion_va) { - gvt_err("fail to map host opregion\n"); - return -EFAULT; + if (intel_gvt_host.hypervisor_type == INTEL_GVT_HYPERVISOR_XEN) { + if (vgpu_opregion(vgpu)->mapped) + map_vgpu_opregion(vgpu, false); + } else if (intel_gvt_host.hypervisor_type == INTEL_GVT_HYPERVISOR_KVM) { + /* Guest opregion is released by VFIO */ } - return 0; + free_pages((unsigned long)vgpu_opregion(vgpu)->va, + get_order(INTEL_GVT_OPREGION_SIZE)); + + vgpu_opregion(vgpu)->va = NULL; + } + #define GVT_OPREGION_FUNC(scic) \ ({ \ u32 __ret; \ @@ -281,11 +469,45 @@ static bool querying_capabilities(u32 scic) */ int intel_vgpu_emulate_opregion_request(struct intel_vgpu *vgpu, u32 swsci) { - u32 *scic, *parm; + u32 scic, parm; u32 func, subfunc; + u64 scic_pa = 0, parm_pa = 0; + int ret; + + switch (intel_gvt_host.hypervisor_type) { + case INTEL_GVT_HYPERVISOR_XEN: + scic = *((u32 *)vgpu_opregion(vgpu)->va + + INTEL_GVT_OPREGION_SCIC); + parm = *((u32 *)vgpu_opregion(vgpu)->va + + INTEL_GVT_OPREGION_PARM); + break; + case INTEL_GVT_HYPERVISOR_KVM: + scic_pa = (vgpu_opregion(vgpu)->gfn[0] << PAGE_SHIFT) + + INTEL_GVT_OPREGION_SCIC; + parm_pa = (vgpu_opregion(vgpu)->gfn[0] << PAGE_SHIFT) + + INTEL_GVT_OPREGION_PARM; + + ret = intel_gvt_hypervisor_read_gpa(vgpu, scic_pa, + &scic, sizeof(scic)); + if (ret) { + gvt_vgpu_err("guest opregion read error %d, gpa 0x%llx, len %lu\n", + ret, scic_pa, sizeof(scic)); + return ret; + } - scic = vgpu_opregion(vgpu)->va + INTEL_GVT_OPREGION_SCIC; - parm = vgpu_opregion(vgpu)->va + INTEL_GVT_OPREGION_PARM; + ret = intel_gvt_hypervisor_read_gpa(vgpu, parm_pa, + &parm, sizeof(parm)); + if (ret) { + gvt_vgpu_err("guest opregion read error %d, gpa 0x%llx, len %lu\n", + ret, scic_pa, sizeof(scic)); + return ret; + } + + break; + default: + gvt_vgpu_err("not supported hypervisor\n"); + return -EINVAL; + } if (!(swsci & SWSCI_SCI_SELECT)) { gvt_vgpu_err("requesting SMI service\n"); @@ -298,9 +520,9 @@ int intel_vgpu_emulate_opregion_request(struct intel_vgpu *vgpu, u32 swsci) return 0; } - func = GVT_OPREGION_FUNC(*scic); - subfunc = GVT_OPREGION_SUBFUNC(*scic); - if (!querying_capabilities(*scic)) { + func = GVT_OPREGION_FUNC(scic); + subfunc = GVT_OPREGION_SUBFUNC(scic); + if (!querying_capabilities(scic)) { gvt_vgpu_err("requesting runtime service: func \"%s\"," " subfunc \"%s\"\n", opregion_func_name(func), @@ -309,11 +531,43 @@ int intel_vgpu_emulate_opregion_request(struct intel_vgpu *vgpu, u32 swsci) * emulate exit status of function call, '0' means * "failure, generic, unsupported or unknown cause" */ - *scic &= ~OPREGION_SCIC_EXIT_MASK; - return 0; + scic &= ~OPREGION_SCIC_EXIT_MASK; + goto out; + } + + scic = 0; + parm = 0; + +out: + switch (intel_gvt_host.hypervisor_type) { + case INTEL_GVT_HYPERVISOR_XEN: + *((u32 *)vgpu_opregion(vgpu)->va + + INTEL_GVT_OPREGION_SCIC) = scic; + *((u32 *)vgpu_opregion(vgpu)->va + + INTEL_GVT_OPREGION_PARM) = parm; + break; + case INTEL_GVT_HYPERVISOR_KVM: + ret = intel_gvt_hypervisor_write_gpa(vgpu, scic_pa, + &scic, sizeof(scic)); + if (ret) { + gvt_vgpu_err("guest opregion write error %d, gpa 0x%llx, len %lu\n", + ret, scic_pa, sizeof(scic)); + return ret; + } + + ret = intel_gvt_hypervisor_write_gpa(vgpu, parm_pa, + &parm, sizeof(parm)); + if (ret) { + gvt_vgpu_err("guest opregion write error %d, gpa 0x%llx, len %lu\n", + ret, scic_pa, sizeof(scic)); + return ret; + } + + break; + default: + gvt_vgpu_err("not supported hypervisor\n"); + return -EINVAL; } - *scic = 0; - *parm = 0; return 0; } diff --git a/drivers/gpu/drm/i915/gvt/reg.h b/drivers/gpu/drm/i915/gvt/reg.h index 7d01c77a0f7a..d4f7ce6dc1d7 100644 --- a/drivers/gpu/drm/i915/gvt/reg.h +++ b/drivers/gpu/drm/i915/gvt/reg.h @@ -51,6 +51,9 @@ #define INTEL_GVT_OPREGION_PAGES 2 #define INTEL_GVT_OPREGION_SIZE (INTEL_GVT_OPREGION_PAGES * PAGE_SIZE) +#define INTEL_GVT_OPREGION_VBT_OFFSET 0x400 +#define INTEL_GVT_OPREGION_VBT_SIZE \ + (INTEL_GVT_OPREGION_SIZE - INTEL_GVT_OPREGION_VBT_OFFSET) #define VGT_SPRSTRIDE(pipe) _PIPE(pipe, _SPRA_STRIDE, _PLANE_STRIDE_2_B) @@ -71,6 +74,7 @@ #define RB_HEAD_OFF_MASK ((1U << 21) - (1U << 2)) #define RB_TAIL_OFF_MASK ((1U << 21) - (1U << 3)) #define RB_TAIL_SIZE_MASK ((1U << 21) - (1U << 12)) -#define _RING_CTL_BUF_SIZE(ctl) (((ctl) & RB_TAIL_SIZE_MASK) + GTT_PAGE_SIZE) +#define _RING_CTL_BUF_SIZE(ctl) (((ctl) & RB_TAIL_SIZE_MASK) + \ + I915_GTT_PAGE_SIZE) #endif diff --git a/drivers/gpu/drm/i915/gvt/render.c b/drivers/gpu/drm/i915/gvt/render.c deleted file mode 100644 index 6d066cf35478..000000000000 --- a/drivers/gpu/drm/i915/gvt/render.c +++ /dev/null @@ -1,405 +0,0 @@ -/* - * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - * Authors: - * Eddie Dong <eddie.dong@intel.com> - * Kevin Tian <kevin.tian@intel.com> - * - * Contributors: - * Zhi Wang <zhi.a.wang@intel.com> - * Changbin Du <changbin.du@intel.com> - * Zhenyu Wang <zhenyuw@linux.intel.com> - * Tina Zhang <tina.zhang@intel.com> - * Bing Niu <bing.niu@intel.com> - * - */ - -#include "i915_drv.h" -#include "gvt.h" -#include "trace.h" - -struct render_mmio { - int ring_id; - i915_reg_t reg; - u32 mask; - bool in_context; - u32 value; -}; - -static struct render_mmio gen8_render_mmio_list[] __cacheline_aligned = { - {RCS, _MMIO(0x229c), 0xffff, false}, - {RCS, _MMIO(0x2248), 0x0, false}, - {RCS, _MMIO(0x2098), 0x0, false}, - {RCS, _MMIO(0x20c0), 0xffff, true}, - {RCS, _MMIO(0x24d0), 0, false}, - {RCS, _MMIO(0x24d4), 0, false}, - {RCS, _MMIO(0x24d8), 0, false}, - {RCS, _MMIO(0x24dc), 0, false}, - {RCS, _MMIO(0x24e0), 0, false}, - {RCS, _MMIO(0x24e4), 0, false}, - {RCS, _MMIO(0x24e8), 0, false}, - {RCS, _MMIO(0x24ec), 0, false}, - {RCS, _MMIO(0x24f0), 0, false}, - {RCS, _MMIO(0x24f4), 0, false}, - {RCS, _MMIO(0x24f8), 0, false}, - {RCS, _MMIO(0x24fc), 0, false}, - {RCS, _MMIO(0x7004), 0xffff, true}, - {RCS, _MMIO(0x7008), 0xffff, true}, - {RCS, _MMIO(0x7000), 0xffff, true}, - {RCS, _MMIO(0x7010), 0xffff, true}, - {RCS, _MMIO(0x7300), 0xffff, true}, - {RCS, _MMIO(0x83a4), 0xffff, true}, - - {BCS, _MMIO(0x2229c), 0xffff, false}, - {BCS, _MMIO(0x2209c), 0xffff, false}, - {BCS, _MMIO(0x220c0), 0xffff, false}, - {BCS, _MMIO(0x22098), 0x0, false}, - {BCS, _MMIO(0x22028), 0x0, false}, -}; - -static struct render_mmio gen9_render_mmio_list[] __cacheline_aligned = { - {RCS, _MMIO(0x229c), 0xffff, false}, - {RCS, _MMIO(0x2248), 0x0, false}, - {RCS, _MMIO(0x2098), 0x0, false}, - {RCS, _MMIO(0x20c0), 0xffff, true}, - {RCS, _MMIO(0x24d0), 0, false}, - {RCS, _MMIO(0x24d4), 0, false}, - {RCS, _MMIO(0x24d8), 0, false}, - {RCS, _MMIO(0x24dc), 0, false}, - {RCS, _MMIO(0x24e0), 0, false}, - {RCS, _MMIO(0x24e4), 0, false}, - {RCS, _MMIO(0x24e8), 0, false}, - {RCS, _MMIO(0x24ec), 0, false}, - {RCS, _MMIO(0x24f0), 0, false}, - {RCS, _MMIO(0x24f4), 0, false}, - {RCS, _MMIO(0x24f8), 0, false}, - {RCS, _MMIO(0x24fc), 0, false}, - {RCS, _MMIO(0x7004), 0xffff, true}, - {RCS, _MMIO(0x7008), 0xffff, true}, - {RCS, _MMIO(0x7000), 0xffff, true}, - {RCS, _MMIO(0x7010), 0xffff, true}, - {RCS, _MMIO(0x7300), 0xffff, true}, - {RCS, _MMIO(0x83a4), 0xffff, true}, - - {RCS, _MMIO(0x40e0), 0, false}, - {RCS, _MMIO(0x40e4), 0, false}, - {RCS, _MMIO(0x2580), 0xffff, true}, - {RCS, _MMIO(0x7014), 0xffff, true}, - {RCS, _MMIO(0x20ec), 0xffff, false}, - {RCS, _MMIO(0xb118), 0, false}, - {RCS, _MMIO(0xe100), 0xffff, true}, - {RCS, _MMIO(0xe180), 0xffff, true}, - {RCS, _MMIO(0xe184), 0xffff, true}, - {RCS, _MMIO(0xe188), 0xffff, true}, - {RCS, _MMIO(0xe194), 0xffff, true}, - {RCS, _MMIO(0x4de0), 0, false}, - {RCS, _MMIO(0x4de4), 0, false}, - {RCS, _MMIO(0x4de8), 0, false}, - {RCS, _MMIO(0x4dec), 0, false}, - {RCS, _MMIO(0x4df0), 0, false}, - {RCS, _MMIO(0x4df4), 0, false}, - - {BCS, _MMIO(0x2229c), 0xffff, false}, - {BCS, _MMIO(0x2209c), 0xffff, false}, - {BCS, _MMIO(0x220c0), 0xffff, false}, - {BCS, _MMIO(0x22098), 0x0, false}, - {BCS, _MMIO(0x22028), 0x0, false}, - - {VCS2, _MMIO(0x1c028), 0xffff, false}, - - {VECS, _MMIO(0x1a028), 0xffff, false}, - - {RCS, _MMIO(0x7304), 0xffff, true}, - {RCS, _MMIO(0x2248), 0x0, false}, - {RCS, _MMIO(0x940c), 0x0, false}, - {RCS, _MMIO(0x4ab8), 0x0, false}, - - {RCS, _MMIO(0x4ab0), 0x0, false}, - {RCS, _MMIO(0x20d4), 0x0, false}, - - {RCS, _MMIO(0xb004), 0x0, false}, - {RCS, _MMIO(0x20a0), 0x0, false}, - {RCS, _MMIO(0x20e4), 0xffff, false}, -}; - -static u32 gen9_render_mocs[I915_NUM_ENGINES][64]; -static u32 gen9_render_mocs_L3[32]; - -static void handle_tlb_pending_event(struct intel_vgpu *vgpu, int ring_id) -{ - struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; - enum forcewake_domains fw; - i915_reg_t reg; - u32 regs[] = { - [RCS] = 0x4260, - [VCS] = 0x4264, - [VCS2] = 0x4268, - [BCS] = 0x426c, - [VECS] = 0x4270, - }; - - if (WARN_ON(ring_id >= ARRAY_SIZE(regs))) - return; - - if (!test_and_clear_bit(ring_id, (void *)vgpu->tlb_handle_pending)) - return; - - reg = _MMIO(regs[ring_id]); - - /* WaForceWakeRenderDuringMmioTLBInvalidate:skl - * we need to put a forcewake when invalidating RCS TLB caches, - * otherwise device can go to RC6 state and interrupt invalidation - * process - */ - fw = intel_uncore_forcewake_for_reg(dev_priv, reg, - FW_REG_READ | FW_REG_WRITE); - if (ring_id == RCS && (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))) - fw |= FORCEWAKE_RENDER; - - intel_uncore_forcewake_get(dev_priv, fw); - - I915_WRITE_FW(reg, 0x1); - - if (wait_for_atomic((I915_READ_FW(reg) == 0), 50)) - gvt_vgpu_err("timeout in invalidate ring (%d) tlb\n", ring_id); - else - vgpu_vreg(vgpu, regs[ring_id]) = 0; - - intel_uncore_forcewake_put(dev_priv, fw); - - gvt_dbg_core("invalidate TLB for ring %d\n", ring_id); -} - -static void load_mocs(struct intel_vgpu *vgpu, int ring_id) -{ - struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; - i915_reg_t offset, l3_offset; - u32 regs[] = { - [RCS] = 0xc800, - [VCS] = 0xc900, - [VCS2] = 0xca00, - [BCS] = 0xcc00, - [VECS] = 0xcb00, - }; - int i; - - if (WARN_ON(ring_id >= ARRAY_SIZE(regs))) - return; - - offset.reg = regs[ring_id]; - for (i = 0; i < 64; i++) { - gen9_render_mocs[ring_id][i] = I915_READ_FW(offset); - I915_WRITE(offset, vgpu_vreg(vgpu, offset)); - offset.reg += 4; - } - - if (ring_id == RCS) { - l3_offset.reg = 0xb020; - for (i = 0; i < 32; i++) { - gen9_render_mocs_L3[i] = I915_READ_FW(l3_offset); - I915_WRITE_FW(l3_offset, vgpu_vreg(vgpu, l3_offset)); - l3_offset.reg += 4; - } - } -} - -static void restore_mocs(struct intel_vgpu *vgpu, int ring_id) -{ - struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; - i915_reg_t offset, l3_offset; - u32 regs[] = { - [RCS] = 0xc800, - [VCS] = 0xc900, - [VCS2] = 0xca00, - [BCS] = 0xcc00, - [VECS] = 0xcb00, - }; - int i; - - if (WARN_ON(ring_id >= ARRAY_SIZE(regs))) - return; - - offset.reg = regs[ring_id]; - for (i = 0; i < 64; i++) { - vgpu_vreg(vgpu, offset) = I915_READ_FW(offset); - I915_WRITE_FW(offset, gen9_render_mocs[ring_id][i]); - offset.reg += 4; - } - - if (ring_id == RCS) { - l3_offset.reg = 0xb020; - for (i = 0; i < 32; i++) { - vgpu_vreg(vgpu, l3_offset) = I915_READ_FW(l3_offset); - I915_WRITE_FW(l3_offset, gen9_render_mocs_L3[i]); - l3_offset.reg += 4; - } - } -} - -#define CTX_CONTEXT_CONTROL_VAL 0x03 - -/* Switch ring mmio values (context) from host to a vgpu. */ -static void switch_mmio_to_vgpu(struct intel_vgpu *vgpu, int ring_id) -{ - struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; - struct render_mmio *mmio; - u32 v; - int i, array_size; - u32 *reg_state = vgpu->shadow_ctx->engine[ring_id].lrc_reg_state; - u32 ctx_ctrl = reg_state[CTX_CONTEXT_CONTROL_VAL]; - u32 inhibit_mask = - _MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT); - i915_reg_t last_reg = _MMIO(0); - - if (IS_SKYLAKE(vgpu->gvt->dev_priv) - || IS_KABYLAKE(vgpu->gvt->dev_priv)) { - mmio = gen9_render_mmio_list; - array_size = ARRAY_SIZE(gen9_render_mmio_list); - load_mocs(vgpu, ring_id); - } else { - mmio = gen8_render_mmio_list; - array_size = ARRAY_SIZE(gen8_render_mmio_list); - } - - for (i = 0; i < array_size; i++, mmio++) { - if (mmio->ring_id != ring_id) - continue; - - mmio->value = I915_READ_FW(mmio->reg); - - /* - * if it is an inhibit context, load in_context mmio - * into HW by mmio write. If it is not, skip this mmio - * write. - */ - if (mmio->in_context && - ((ctx_ctrl & inhibit_mask) != inhibit_mask) && - i915_modparams.enable_execlists) - continue; - - if (mmio->mask) - v = vgpu_vreg(vgpu, mmio->reg) | (mmio->mask << 16); - else - v = vgpu_vreg(vgpu, mmio->reg); - - I915_WRITE_FW(mmio->reg, v); - last_reg = mmio->reg; - - trace_render_mmio(vgpu->id, "load", - i915_mmio_reg_offset(mmio->reg), - mmio->value, v); - } - - /* Make sure the swiched MMIOs has taken effect. */ - if (likely(INTEL_GVT_MMIO_OFFSET(last_reg))) - I915_READ_FW(last_reg); - - handle_tlb_pending_event(vgpu, ring_id); -} - -/* Switch ring mmio values (context) from vgpu to host. */ -static void switch_mmio_to_host(struct intel_vgpu *vgpu, int ring_id) -{ - struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; - struct render_mmio *mmio; - i915_reg_t last_reg = _MMIO(0); - u32 v; - int i, array_size; - - if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { - mmio = gen9_render_mmio_list; - array_size = ARRAY_SIZE(gen9_render_mmio_list); - restore_mocs(vgpu, ring_id); - } else { - mmio = gen8_render_mmio_list; - array_size = ARRAY_SIZE(gen8_render_mmio_list); - } - - for (i = 0; i < array_size; i++, mmio++) { - if (mmio->ring_id != ring_id) - continue; - - vgpu_vreg(vgpu, mmio->reg) = I915_READ_FW(mmio->reg); - - if (mmio->mask) { - vgpu_vreg(vgpu, mmio->reg) &= ~(mmio->mask << 16); - v = mmio->value | (mmio->mask << 16); - } else - v = mmio->value; - - if (mmio->in_context) - continue; - - I915_WRITE_FW(mmio->reg, v); - last_reg = mmio->reg; - - trace_render_mmio(vgpu->id, "restore", - i915_mmio_reg_offset(mmio->reg), - mmio->value, v); - } - - /* Make sure the swiched MMIOs has taken effect. */ - if (likely(INTEL_GVT_MMIO_OFFSET(last_reg))) - I915_READ_FW(last_reg); -} - -/** - * intel_gvt_switch_render_mmio - switch mmio context of specific engine - * @pre: the last vGPU that own the engine - * @next: the vGPU to switch to - * @ring_id: specify the engine - * - * If pre is null indicates that host own the engine. If next is null - * indicates that we are switching to host workload. - */ -void intel_gvt_switch_mmio(struct intel_vgpu *pre, - struct intel_vgpu *next, int ring_id) -{ - struct drm_i915_private *dev_priv; - - if (WARN_ON(!pre && !next)) - return; - - gvt_dbg_render("switch ring %d from %s to %s\n", ring_id, - pre ? "vGPU" : "host", next ? "vGPU" : "HOST"); - - dev_priv = pre ? pre->gvt->dev_priv : next->gvt->dev_priv; - - /** - * We are using raw mmio access wrapper to improve the - * performace for batch mmio read/write, so we need - * handle forcewake mannually. - */ - intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); - - /** - * TODO: Optimize for vGPU to vGPU switch by merging - * switch_mmio_to_host() and switch_mmio_to_vgpu(). - */ - if (pre) - switch_mmio_to_host(pre, ring_id); - - if (next) - switch_mmio_to_vgpu(next, ring_id); - - intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); -} diff --git a/drivers/gpu/drm/i915/gvt/sched_policy.c b/drivers/gpu/drm/i915/gvt/sched_policy.c index 03532dfc0cd5..cc1ce361cd76 100644 --- a/drivers/gpu/drm/i915/gvt/sched_policy.c +++ b/drivers/gpu/drm/i915/gvt/sched_policy.c @@ -50,6 +50,7 @@ static bool vgpu_has_pending_workload(struct intel_vgpu *vgpu) struct vgpu_sched_data { struct list_head lru_list; struct intel_vgpu *vgpu; + bool active; ktime_t sched_in_time; ktime_t sched_out_time; @@ -308,8 +309,15 @@ static int tbs_sched_init_vgpu(struct intel_vgpu *vgpu) static void tbs_sched_clean_vgpu(struct intel_vgpu *vgpu) { + struct intel_gvt *gvt = vgpu->gvt; + struct gvt_sched_data *sched_data = gvt->scheduler.sched_data; + kfree(vgpu->sched_data); vgpu->sched_data = NULL; + + /* this vgpu id has been removed */ + if (idr_is_empty(&gvt->vgpu_idr)) + hrtimer_cancel(&sched_data->timer); } static void tbs_sched_start_schedule(struct intel_vgpu *vgpu) @@ -325,6 +333,7 @@ static void tbs_sched_start_schedule(struct intel_vgpu *vgpu) if (!hrtimer_active(&sched_data->timer)) hrtimer_start(&sched_data->timer, ktime_add_ns(ktime_get(), sched_data->period), HRTIMER_MODE_ABS); + vgpu_data->active = true; } static void tbs_sched_stop_schedule(struct intel_vgpu *vgpu) @@ -332,6 +341,7 @@ static void tbs_sched_stop_schedule(struct intel_vgpu *vgpu) struct vgpu_sched_data *vgpu_data = vgpu->sched_data; list_del_init(&vgpu_data->lru_list); + vgpu_data->active = false; } static struct intel_gvt_sched_policy_ops tbs_schedule_ops = { @@ -367,9 +377,17 @@ void intel_vgpu_clean_sched_policy(struct intel_vgpu *vgpu) void intel_vgpu_start_schedule(struct intel_vgpu *vgpu) { - gvt_dbg_core("vgpu%d: start schedule\n", vgpu->id); + struct vgpu_sched_data *vgpu_data = vgpu->sched_data; - vgpu->gvt->scheduler.sched_ops->start_schedule(vgpu); + if (!vgpu_data->active) { + gvt_dbg_core("vgpu%d: start schedule\n", vgpu->id); + vgpu->gvt->scheduler.sched_ops->start_schedule(vgpu); + } +} + +void intel_gvt_kick_schedule(struct intel_gvt *gvt) +{ + intel_gvt_request_service(gvt, INTEL_GVT_REQUEST_EVENT_SCHED); } void intel_vgpu_stop_schedule(struct intel_vgpu *vgpu) @@ -377,6 +395,10 @@ void intel_vgpu_stop_schedule(struct intel_vgpu *vgpu) struct intel_gvt_workload_scheduler *scheduler = &vgpu->gvt->scheduler; int ring_id; + struct vgpu_sched_data *vgpu_data = vgpu->sched_data; + + if (!vgpu_data->active) + return; gvt_dbg_core("vgpu%d: stop schedule\n", vgpu->id); diff --git a/drivers/gpu/drm/i915/gvt/sched_policy.h b/drivers/gpu/drm/i915/gvt/sched_policy.h index ba00a5f7455f..7b59e3e88b8b 100644 --- a/drivers/gpu/drm/i915/gvt/sched_policy.h +++ b/drivers/gpu/drm/i915/gvt/sched_policy.h @@ -57,4 +57,6 @@ void intel_vgpu_start_schedule(struct intel_vgpu *vgpu); void intel_vgpu_stop_schedule(struct intel_vgpu *vgpu); +void intel_gvt_kick_schedule(struct intel_gvt *gvt); + #endif diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c index 69f8f0d155b9..b55b3580ca1d 100644 --- a/drivers/gpu/drm/i915/gvt/scheduler.c +++ b/drivers/gpu/drm/i915/gvt/scheduler.c @@ -57,7 +57,7 @@ static int populate_shadow_context(struct intel_vgpu_workload *workload) struct intel_vgpu *vgpu = workload->vgpu; struct intel_gvt *gvt = vgpu->gvt; int ring_id = workload->ring_id; - struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx; + struct i915_gem_context *shadow_ctx = vgpu->submission.shadow_ctx; struct drm_i915_gem_object *ctx_obj = shadow_ctx->engine[ring_id].state->obj; struct execlist_ring_context *shadow_ring_context; @@ -81,16 +81,16 @@ static int populate_shadow_context(struct intel_vgpu_workload *workload) while (i < context_page_num) { context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, (u32)((workload->ctx_desc.lrca + i) << - GTT_PAGE_SHIFT)); + I915_GTT_PAGE_SHIFT)); if (context_gpa == INTEL_GVT_INVALID_ADDR) { gvt_vgpu_err("Invalid guest context descriptor\n"); - return -EINVAL; + return -EFAULT; } page = i915_gem_object_get_page(ctx_obj, LRC_HEADER_PAGES + i); dst = kmap(page); intel_gvt_hypervisor_read_gpa(vgpu, context_gpa, dst, - GTT_PAGE_SIZE); + I915_GTT_PAGE_SIZE); kunmap(page); i++; } @@ -120,7 +120,7 @@ static int populate_shadow_context(struct intel_vgpu_workload *workload) sizeof(*shadow_ring_context), (void *)shadow_ring_context + sizeof(*shadow_ring_context), - GTT_PAGE_SIZE - sizeof(*shadow_ring_context)); + I915_GTT_PAGE_SIZE - sizeof(*shadow_ring_context)); kunmap(page); return 0; @@ -248,7 +248,7 @@ static int copy_workload_to_ring_buffer(struct intel_vgpu_workload *workload) return 0; } -void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) +static void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) { if (!wa_ctx->indirect_ctx.obj) return; @@ -267,11 +267,12 @@ void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) */ int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload) { + struct intel_vgpu *vgpu = workload->vgpu; + struct intel_vgpu_submission *s = &vgpu->submission; + struct i915_gem_context *shadow_ctx = s->shadow_ctx; + struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; int ring_id = workload->ring_id; - struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx; - struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv; struct intel_engine_cs *engine = dev_priv->engine[ring_id]; - struct intel_vgpu *vgpu = workload->vgpu; struct intel_ring *ring; int ret; @@ -284,7 +285,7 @@ int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload) shadow_ctx->desc_template |= workload->ctx_desc.addressing_mode << GEN8_CTX_ADDRESSING_MODE_SHIFT; - if (!test_and_set_bit(ring_id, vgpu->shadow_ctx_desc_updated)) + if (!test_and_set_bit(ring_id, s->shadow_ctx_desc_updated)) shadow_context_descriptor_update(shadow_ctx, dev_priv->engine[ring_id]); @@ -327,14 +328,15 @@ err_scan: return ret; } -int intel_gvt_generate_request(struct intel_vgpu_workload *workload) +static int intel_gvt_generate_request(struct intel_vgpu_workload *workload) { int ring_id = workload->ring_id; struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv; struct intel_engine_cs *engine = dev_priv->engine[ring_id]; struct drm_i915_gem_request *rq; struct intel_vgpu *vgpu = workload->vgpu; - struct i915_gem_context *shadow_ctx = vgpu->shadow_ctx; + struct intel_vgpu_submission *s = &vgpu->submission; + struct i915_gem_context *shadow_ctx = s->shadow_ctx; int ret; rq = i915_gem_request_alloc(dev_priv->engine[ring_id], shadow_ctx); @@ -358,11 +360,203 @@ err_unpin: return ret; } +static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload); + +static int prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload) +{ + struct intel_gvt *gvt = workload->vgpu->gvt; + const int gmadr_bytes = gvt->device_info.gmadr_bytes_in_cmd; + struct intel_vgpu_shadow_bb *bb; + int ret; + + list_for_each_entry(bb, &workload->shadow_bb, list) { + bb->vma = i915_gem_object_ggtt_pin(bb->obj, NULL, 0, 0, 0); + if (IS_ERR(bb->vma)) { + ret = PTR_ERR(bb->vma); + goto err; + } + + /* relocate shadow batch buffer */ + bb->bb_start_cmd_va[1] = i915_ggtt_offset(bb->vma); + if (gmadr_bytes == 8) + bb->bb_start_cmd_va[2] = 0; + + /* No one is going to touch shadow bb from now on. */ + if (bb->clflush & CLFLUSH_AFTER) { + drm_clflush_virt_range(bb->va, bb->obj->base.size); + bb->clflush &= ~CLFLUSH_AFTER; + } + + ret = i915_gem_object_set_to_gtt_domain(bb->obj, false); + if (ret) + goto err; + + i915_gem_obj_finish_shmem_access(bb->obj); + bb->accessing = false; + + i915_vma_move_to_active(bb->vma, workload->req, 0); + } + return 0; +err: + release_shadow_batch_buffer(workload); + return ret; +} + +static int update_wa_ctx_2_shadow_ctx(struct intel_shadow_wa_ctx *wa_ctx) +{ + struct intel_vgpu_workload *workload = container_of(wa_ctx, + struct intel_vgpu_workload, + wa_ctx); + int ring_id = workload->ring_id; + struct intel_vgpu_submission *s = &workload->vgpu->submission; + struct i915_gem_context *shadow_ctx = s->shadow_ctx; + struct drm_i915_gem_object *ctx_obj = + shadow_ctx->engine[ring_id].state->obj; + struct execlist_ring_context *shadow_ring_context; + struct page *page; + + page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN); + shadow_ring_context = kmap_atomic(page); + + shadow_ring_context->bb_per_ctx_ptr.val = + (shadow_ring_context->bb_per_ctx_ptr.val & + (~PER_CTX_ADDR_MASK)) | wa_ctx->per_ctx.shadow_gma; + shadow_ring_context->rcs_indirect_ctx.val = + (shadow_ring_context->rcs_indirect_ctx.val & + (~INDIRECT_CTX_ADDR_MASK)) | wa_ctx->indirect_ctx.shadow_gma; + + kunmap_atomic(shadow_ring_context); + return 0; +} + +static int prepare_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) +{ + struct i915_vma *vma; + unsigned char *per_ctx_va = + (unsigned char *)wa_ctx->indirect_ctx.shadow_va + + wa_ctx->indirect_ctx.size; + + if (wa_ctx->indirect_ctx.size == 0) + return 0; + + vma = i915_gem_object_ggtt_pin(wa_ctx->indirect_ctx.obj, NULL, + 0, CACHELINE_BYTES, 0); + if (IS_ERR(vma)) + return PTR_ERR(vma); + + /* FIXME: we are not tracking our pinned VMA leaving it + * up to the core to fix up the stray pin_count upon + * free. + */ + + wa_ctx->indirect_ctx.shadow_gma = i915_ggtt_offset(vma); + + wa_ctx->per_ctx.shadow_gma = *((unsigned int *)per_ctx_va + 1); + memset(per_ctx_va, 0, CACHELINE_BYTES); + + update_wa_ctx_2_shadow_ctx(wa_ctx); + return 0; +} + +static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload) +{ + struct intel_vgpu *vgpu = workload->vgpu; + struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; + struct intel_vgpu_shadow_bb *bb, *pos; + + if (list_empty(&workload->shadow_bb)) + return; + + bb = list_first_entry(&workload->shadow_bb, + struct intel_vgpu_shadow_bb, list); + + mutex_lock(&dev_priv->drm.struct_mutex); + + list_for_each_entry_safe(bb, pos, &workload->shadow_bb, list) { + if (bb->obj) { + if (bb->accessing) + i915_gem_obj_finish_shmem_access(bb->obj); + + if (bb->va && !IS_ERR(bb->va)) + i915_gem_object_unpin_map(bb->obj); + + if (bb->vma && !IS_ERR(bb->vma)) { + i915_vma_unpin(bb->vma); + i915_vma_close(bb->vma); + } + __i915_gem_object_release_unless_active(bb->obj); + } + list_del(&bb->list); + kfree(bb); + } + + mutex_unlock(&dev_priv->drm.struct_mutex); +} + +static int prepare_workload(struct intel_vgpu_workload *workload) +{ + struct intel_vgpu *vgpu = workload->vgpu; + int ret = 0; + + ret = intel_vgpu_pin_mm(workload->shadow_mm); + if (ret) { + gvt_vgpu_err("fail to vgpu pin mm\n"); + return ret; + } + + ret = intel_vgpu_sync_oos_pages(workload->vgpu); + if (ret) { + gvt_vgpu_err("fail to vgpu sync oos pages\n"); + goto err_unpin_mm; + } + + ret = intel_vgpu_flush_post_shadow(workload->vgpu); + if (ret) { + gvt_vgpu_err("fail to flush post shadow\n"); + goto err_unpin_mm; + } + + ret = intel_gvt_generate_request(workload); + if (ret) { + gvt_vgpu_err("fail to generate request\n"); + goto err_unpin_mm; + } + + ret = prepare_shadow_batch_buffer(workload); + if (ret) { + gvt_vgpu_err("fail to prepare_shadow_batch_buffer\n"); + goto err_unpin_mm; + } + + ret = prepare_shadow_wa_ctx(&workload->wa_ctx); + if (ret) { + gvt_vgpu_err("fail to prepare_shadow_wa_ctx\n"); + goto err_shadow_batch; + } + + if (workload->prepare) { + ret = workload->prepare(workload); + if (ret) + goto err_shadow_wa_ctx; + } + + return 0; +err_shadow_wa_ctx: + release_shadow_wa_ctx(&workload->wa_ctx); +err_shadow_batch: + release_shadow_batch_buffer(workload); +err_unpin_mm: + intel_vgpu_unpin_mm(workload->shadow_mm); + return ret; +} + static int dispatch_workload(struct intel_vgpu_workload *workload) { + struct intel_vgpu *vgpu = workload->vgpu; + struct intel_vgpu_submission *s = &vgpu->submission; + struct i915_gem_context *shadow_ctx = s->shadow_ctx; + struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; int ring_id = workload->ring_id; - struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx; - struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv; struct intel_engine_cs *engine = dev_priv->engine[ring_id]; int ret = 0; @@ -375,12 +569,10 @@ static int dispatch_workload(struct intel_vgpu_workload *workload) if (ret) goto out; - if (workload->prepare) { - ret = workload->prepare(workload); - if (ret) { - engine->context_unpin(engine, shadow_ctx); - goto out; - } + ret = prepare_workload(workload); + if (ret) { + engine->context_unpin(engine, shadow_ctx); + goto out; } out: @@ -448,7 +640,7 @@ static struct intel_vgpu_workload *pick_next_workload( gvt_dbg_sched("ring id %d pick new workload %p\n", ring_id, workload); - atomic_inc(&workload->vgpu->running_workload_num); + atomic_inc(&workload->vgpu->submission.running_workload_num); out: mutex_unlock(&gvt->lock); return workload; @@ -458,8 +650,9 @@ static void update_guest_context(struct intel_vgpu_workload *workload) { struct intel_vgpu *vgpu = workload->vgpu; struct intel_gvt *gvt = vgpu->gvt; + struct intel_vgpu_submission *s = &vgpu->submission; + struct i915_gem_context *shadow_ctx = s->shadow_ctx; int ring_id = workload->ring_id; - struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx; struct drm_i915_gem_object *ctx_obj = shadow_ctx->engine[ring_id].state->obj; struct execlist_ring_context *shadow_ring_context; @@ -483,7 +676,7 @@ static void update_guest_context(struct intel_vgpu_workload *workload) while (i < context_page_num) { context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, (u32)((workload->ctx_desc.lrca + i) << - GTT_PAGE_SHIFT)); + I915_GTT_PAGE_SHIFT)); if (context_gpa == INTEL_GVT_INVALID_ADDR) { gvt_vgpu_err("invalid guest context descriptor\n"); return; @@ -492,7 +685,7 @@ static void update_guest_context(struct intel_vgpu_workload *workload) page = i915_gem_object_get_page(ctx_obj, LRC_HEADER_PAGES + i); src = kmap(page); intel_gvt_hypervisor_write_gpa(vgpu, context_gpa, src, - GTT_PAGE_SIZE); + I915_GTT_PAGE_SIZE); kunmap(page); i++; } @@ -517,23 +710,41 @@ static void update_guest_context(struct intel_vgpu_workload *workload) sizeof(*shadow_ring_context), (void *)shadow_ring_context + sizeof(*shadow_ring_context), - GTT_PAGE_SIZE - sizeof(*shadow_ring_context)); + I915_GTT_PAGE_SIZE - sizeof(*shadow_ring_context)); kunmap(page); } +static void clean_workloads(struct intel_vgpu *vgpu, unsigned long engine_mask) +{ + struct intel_vgpu_submission *s = &vgpu->submission; + struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; + struct intel_engine_cs *engine; + struct intel_vgpu_workload *pos, *n; + unsigned int tmp; + + /* free the unsubmited workloads in the queues. */ + for_each_engine_masked(engine, dev_priv, engine_mask, tmp) { + list_for_each_entry_safe(pos, n, + &s->workload_q_head[engine->id], list) { + list_del_init(&pos->list); + intel_vgpu_destroy_workload(pos); + } + clear_bit(engine->id, s->shadow_ctx_desc_updated); + } +} + static void complete_current_workload(struct intel_gvt *gvt, int ring_id) { struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; - struct intel_vgpu_workload *workload; - struct intel_vgpu *vgpu; + struct intel_vgpu_workload *workload = + scheduler->current_workload[ring_id]; + struct intel_vgpu *vgpu = workload->vgpu; + struct intel_vgpu_submission *s = &vgpu->submission; int event; mutex_lock(&gvt->lock); - workload = scheduler->current_workload[ring_id]; - vgpu = workload->vgpu; - /* For the workload w/ request, needs to wait for the context * switch to make sure request is completed. * For the workload w/o request, directly complete the workload. @@ -570,7 +781,7 @@ static void complete_current_workload(struct intel_gvt *gvt, int ring_id) } mutex_lock(&dev_priv->drm.struct_mutex); /* unpin shadow ctx as the shadow_ctx update is done */ - engine->context_unpin(engine, workload->vgpu->shadow_ctx); + engine->context_unpin(engine, s->shadow_ctx); mutex_unlock(&dev_priv->drm.struct_mutex); } @@ -580,9 +791,32 @@ static void complete_current_workload(struct intel_gvt *gvt, int ring_id) scheduler->current_workload[ring_id] = NULL; list_del_init(&workload->list); + + if (!workload->status) { + release_shadow_batch_buffer(workload); + release_shadow_wa_ctx(&workload->wa_ctx); + } + + if (workload->status || (vgpu->resetting_eng & ENGINE_MASK(ring_id))) { + /* if workload->status is not successful means HW GPU + * has occurred GPU hang or something wrong with i915/GVT, + * and GVT won't inject context switch interrupt to guest. + * So this error is a vGPU hang actually to the guest. + * According to this we should emunlate a vGPU hang. If + * there are pending workloads which are already submitted + * from guest, we should clean them up like HW GPU does. + * + * if it is in middle of engine resetting, the pending + * workloads won't be submitted to HW GPU and will be + * cleaned up during the resetting process later, so doing + * the workload clean up here doesn't have any impact. + **/ + clean_workloads(vgpu, ENGINE_MASK(ring_id)); + } + workload->complete(workload); - atomic_dec(&vgpu->running_workload_num); + atomic_dec(&s->running_workload_num); wake_up(&scheduler->workload_complete_wq); if (gvt->scheduler.need_reschedule) @@ -665,20 +899,23 @@ complete: FORCEWAKE_ALL); intel_runtime_pm_put(gvt->dev_priv); + if (ret && (vgpu_is_vm_unhealthy(ret))) + enter_failsafe_mode(vgpu, GVT_FAILSAFE_GUEST_ERR); } return 0; } void intel_gvt_wait_vgpu_idle(struct intel_vgpu *vgpu) { + struct intel_vgpu_submission *s = &vgpu->submission; struct intel_gvt *gvt = vgpu->gvt; struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; - if (atomic_read(&vgpu->running_workload_num)) { + if (atomic_read(&s->running_workload_num)) { gvt_dbg_sched("wait vgpu idle\n"); wait_event(scheduler->workload_complete_wq, - !atomic_read(&vgpu->running_workload_num)); + !atomic_read(&s->running_workload_num)); } } @@ -743,26 +980,370 @@ err: return ret; } -void intel_vgpu_clean_gvt_context(struct intel_vgpu *vgpu) +/** + * intel_vgpu_clean_submission - free submission-related resource for vGPU + * @vgpu: a vGPU + * + * This function is called when a vGPU is being destroyed. + * + */ +void intel_vgpu_clean_submission(struct intel_vgpu *vgpu) { - i915_gem_context_put(vgpu->shadow_ctx); + struct intel_vgpu_submission *s = &vgpu->submission; + + intel_vgpu_select_submission_ops(vgpu, ALL_ENGINES, 0); + i915_gem_context_put(s->shadow_ctx); + kmem_cache_destroy(s->workloads); } -int intel_vgpu_init_gvt_context(struct intel_vgpu *vgpu) + +/** + * intel_vgpu_reset_submission - reset submission-related resource for vGPU + * @vgpu: a vGPU + * @engine_mask: engines expected to be reset + * + * This function is called when a vGPU is being destroyed. + * + */ +void intel_vgpu_reset_submission(struct intel_vgpu *vgpu, + unsigned long engine_mask) { - atomic_set(&vgpu->running_workload_num, 0); + struct intel_vgpu_submission *s = &vgpu->submission; - vgpu->shadow_ctx = i915_gem_context_create_gvt( + if (!s->active) + return; + + clean_workloads(vgpu, engine_mask); + s->ops->reset(vgpu, engine_mask); +} + +/** + * intel_vgpu_setup_submission - setup submission-related resource for vGPU + * @vgpu: a vGPU + * + * This function is called when a vGPU is being created. + * + * Returns: + * Zero on success, negative error code if failed. + * + */ +int intel_vgpu_setup_submission(struct intel_vgpu *vgpu) +{ + struct intel_vgpu_submission *s = &vgpu->submission; + enum intel_engine_id i; + struct intel_engine_cs *engine; + int ret; + + s->shadow_ctx = i915_gem_context_create_gvt( &vgpu->gvt->dev_priv->drm); - if (IS_ERR(vgpu->shadow_ctx)) - return PTR_ERR(vgpu->shadow_ctx); + if (IS_ERR(s->shadow_ctx)) + return PTR_ERR(s->shadow_ctx); + + if (HAS_LOGICAL_RING_PREEMPTION(vgpu->gvt->dev_priv)) + s->shadow_ctx->priority = INT_MAX; + + bitmap_zero(s->shadow_ctx_desc_updated, I915_NUM_ENGINES); - if (INTEL_INFO(vgpu->gvt->dev_priv)->has_logical_ring_preemption) - vgpu->shadow_ctx->priority = INT_MAX; + s->workloads = kmem_cache_create("gvt-g_vgpu_workload", + sizeof(struct intel_vgpu_workload), 0, + SLAB_HWCACHE_ALIGN, + NULL); - vgpu->shadow_ctx->engine[RCS].initialised = true; + if (!s->workloads) { + ret = -ENOMEM; + goto out_shadow_ctx; + } + + for_each_engine(engine, vgpu->gvt->dev_priv, i) + INIT_LIST_HEAD(&s->workload_q_head[i]); - bitmap_zero(vgpu->shadow_ctx_desc_updated, I915_NUM_ENGINES); + atomic_set(&s->running_workload_num, 0); + bitmap_zero(s->tlb_handle_pending, I915_NUM_ENGINES); return 0; + +out_shadow_ctx: + i915_gem_context_put(s->shadow_ctx); + return ret; +} + +/** + * intel_vgpu_select_submission_ops - select virtual submission interface + * @vgpu: a vGPU + * @interface: expected vGPU virtual submission interface + * + * This function is called when guest configures submission interface. + * + * Returns: + * Zero on success, negative error code if failed. + * + */ +int intel_vgpu_select_submission_ops(struct intel_vgpu *vgpu, + unsigned long engine_mask, + unsigned int interface) +{ + struct intel_vgpu_submission *s = &vgpu->submission; + const struct intel_vgpu_submission_ops *ops[] = { + [INTEL_VGPU_EXECLIST_SUBMISSION] = + &intel_vgpu_execlist_submission_ops, + }; + int ret; + + if (WARN_ON(interface >= ARRAY_SIZE(ops))) + return -EINVAL; + + if (WARN_ON(interface == 0 && engine_mask != ALL_ENGINES)) + return -EINVAL; + + if (s->active) + s->ops->clean(vgpu, engine_mask); + + if (interface == 0) { + s->ops = NULL; + s->virtual_submission_interface = 0; + s->active = false; + gvt_dbg_core("vgpu%d: remove submission ops\n", vgpu->id); + return 0; + } + + ret = ops[interface]->init(vgpu, engine_mask); + if (ret) + return ret; + + s->ops = ops[interface]; + s->virtual_submission_interface = interface; + s->active = true; + + gvt_dbg_core("vgpu%d: activate ops [ %s ]\n", + vgpu->id, s->ops->name); + + return 0; +} + +/** + * intel_vgpu_destroy_workload - destroy a vGPU workload + * @vgpu: a vGPU + * + * This function is called when destroy a vGPU workload. + * + */ +void intel_vgpu_destroy_workload(struct intel_vgpu_workload *workload) +{ + struct intel_vgpu_submission *s = &workload->vgpu->submission; + + if (workload->shadow_mm) + intel_gvt_mm_unreference(workload->shadow_mm); + + kmem_cache_free(s->workloads, workload); +} + +static struct intel_vgpu_workload * +alloc_workload(struct intel_vgpu *vgpu) +{ + struct intel_vgpu_submission *s = &vgpu->submission; + struct intel_vgpu_workload *workload; + + workload = kmem_cache_zalloc(s->workloads, GFP_KERNEL); + if (!workload) + return ERR_PTR(-ENOMEM); + + INIT_LIST_HEAD(&workload->list); + INIT_LIST_HEAD(&workload->shadow_bb); + + init_waitqueue_head(&workload->shadow_ctx_status_wq); + atomic_set(&workload->shadow_ctx_active, 0); + + workload->status = -EINPROGRESS; + workload->shadowed = false; + workload->vgpu = vgpu; + + return workload; +} + +#define RING_CTX_OFF(x) \ + offsetof(struct execlist_ring_context, x) + +static void read_guest_pdps(struct intel_vgpu *vgpu, + u64 ring_context_gpa, u32 pdp[8]) +{ + u64 gpa; + int i; + + gpa = ring_context_gpa + RING_CTX_OFF(pdp3_UDW.val); + + for (i = 0; i < 8; i++) + intel_gvt_hypervisor_read_gpa(vgpu, + gpa + i * 8, &pdp[7 - i], 4); +} + +static int prepare_mm(struct intel_vgpu_workload *workload) +{ + struct execlist_ctx_descriptor_format *desc = &workload->ctx_desc; + struct intel_vgpu_mm *mm; + struct intel_vgpu *vgpu = workload->vgpu; + int page_table_level; + u32 pdp[8]; + + if (desc->addressing_mode == 1) { /* legacy 32-bit */ + page_table_level = 3; + } else if (desc->addressing_mode == 3) { /* legacy 64 bit */ + page_table_level = 4; + } else { + gvt_vgpu_err("Advanced Context mode(SVM) is not supported!\n"); + return -EINVAL; + } + + read_guest_pdps(workload->vgpu, workload->ring_context_gpa, pdp); + + mm = intel_vgpu_find_ppgtt_mm(workload->vgpu, page_table_level, pdp); + if (mm) { + intel_gvt_mm_reference(mm); + } else { + + mm = intel_vgpu_create_mm(workload->vgpu, INTEL_GVT_MM_PPGTT, + pdp, page_table_level, 0); + if (IS_ERR(mm)) { + gvt_vgpu_err("fail to create mm object.\n"); + return PTR_ERR(mm); + } + } + workload->shadow_mm = mm; + return 0; +} + +#define same_context(a, b) (((a)->context_id == (b)->context_id) && \ + ((a)->lrca == (b)->lrca)) + +#define get_last_workload(q) \ + (list_empty(q) ? NULL : container_of(q->prev, \ + struct intel_vgpu_workload, list)) +/** + * intel_vgpu_create_workload - create a vGPU workload + * @vgpu: a vGPU + * @desc: a guest context descriptor + * + * This function is called when creating a vGPU workload. + * + * Returns: + * struct intel_vgpu_workload * on success, negative error code in + * pointer if failed. + * + */ +struct intel_vgpu_workload * +intel_vgpu_create_workload(struct intel_vgpu *vgpu, int ring_id, + struct execlist_ctx_descriptor_format *desc) +{ + struct intel_vgpu_submission *s = &vgpu->submission; + struct list_head *q = workload_q_head(vgpu, ring_id); + struct intel_vgpu_workload *last_workload = get_last_workload(q); + struct intel_vgpu_workload *workload = NULL; + struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; + u64 ring_context_gpa; + u32 head, tail, start, ctl, ctx_ctl, per_ctx, indirect_ctx; + int ret; + + ring_context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, + (u32)((desc->lrca + 1) << I915_GTT_PAGE_SHIFT)); + if (ring_context_gpa == INTEL_GVT_INVALID_ADDR) { + gvt_vgpu_err("invalid guest context LRCA: %x\n", desc->lrca); + return ERR_PTR(-EINVAL); + } + + intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + + RING_CTX_OFF(ring_header.val), &head, 4); + + intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + + RING_CTX_OFF(ring_tail.val), &tail, 4); + + head &= RB_HEAD_OFF_MASK; + tail &= RB_TAIL_OFF_MASK; + + if (last_workload && same_context(&last_workload->ctx_desc, desc)) { + gvt_dbg_el("ring id %d cur workload == last\n", ring_id); + gvt_dbg_el("ctx head %x real head %lx\n", head, + last_workload->rb_tail); + /* + * cannot use guest context head pointer here, + * as it might not be updated at this time + */ + head = last_workload->rb_tail; + } + + gvt_dbg_el("ring id %d begin a new workload\n", ring_id); + + /* record some ring buffer register values for scan and shadow */ + intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + + RING_CTX_OFF(rb_start.val), &start, 4); + intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + + RING_CTX_OFF(rb_ctrl.val), &ctl, 4); + intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + + RING_CTX_OFF(ctx_ctrl.val), &ctx_ctl, 4); + + workload = alloc_workload(vgpu); + if (IS_ERR(workload)) + return workload; + + workload->ring_id = ring_id; + workload->ctx_desc = *desc; + workload->ring_context_gpa = ring_context_gpa; + workload->rb_head = head; + workload->rb_tail = tail; + workload->rb_start = start; + workload->rb_ctl = ctl; + + if (ring_id == RCS) { + intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + + RING_CTX_OFF(bb_per_ctx_ptr.val), &per_ctx, 4); + intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + + RING_CTX_OFF(rcs_indirect_ctx.val), &indirect_ctx, 4); + + workload->wa_ctx.indirect_ctx.guest_gma = + indirect_ctx & INDIRECT_CTX_ADDR_MASK; + workload->wa_ctx.indirect_ctx.size = + (indirect_ctx & INDIRECT_CTX_SIZE_MASK) * + CACHELINE_BYTES; + workload->wa_ctx.per_ctx.guest_gma = + per_ctx & PER_CTX_ADDR_MASK; + workload->wa_ctx.per_ctx.valid = per_ctx & 1; + } + + gvt_dbg_el("workload %p ring id %d head %x tail %x start %x ctl %x\n", + workload, ring_id, head, tail, start, ctl); + + ret = prepare_mm(workload); + if (ret) { + kmem_cache_free(s->workloads, workload); + return ERR_PTR(ret); + } + + /* Only scan and shadow the first workload in the queue + * as there is only one pre-allocated buf-obj for shadow. + */ + if (list_empty(workload_q_head(vgpu, ring_id))) { + intel_runtime_pm_get(dev_priv); + mutex_lock(&dev_priv->drm.struct_mutex); + ret = intel_gvt_scan_and_shadow_workload(workload); + mutex_unlock(&dev_priv->drm.struct_mutex); + intel_runtime_pm_put(dev_priv); + } + + if (ret && (vgpu_is_vm_unhealthy(ret))) { + enter_failsafe_mode(vgpu, GVT_FAILSAFE_GUEST_ERR); + intel_vgpu_destroy_workload(workload); + return ERR_PTR(ret); + } + + return workload; +} + +/** + * intel_vgpu_queue_workload - Qeue a vGPU workload + * @workload: the workload to queue in + */ +void intel_vgpu_queue_workload(struct intel_vgpu_workload *workload) +{ + list_add_tail(&workload->list, + workload_q_head(workload->vgpu, workload->ring_id)); + intel_gvt_kick_schedule(workload->vgpu->gvt); + wake_up(&workload->vgpu->gvt->scheduler.waitq[workload->ring_id]); } diff --git a/drivers/gpu/drm/i915/gvt/scheduler.h b/drivers/gpu/drm/i915/gvt/scheduler.h index b9f872204d7e..ff175a98b19e 100644 --- a/drivers/gpu/drm/i915/gvt/scheduler.h +++ b/drivers/gpu/drm/i915/gvt/scheduler.h @@ -112,24 +112,20 @@ struct intel_vgpu_workload { struct intel_shadow_wa_ctx wa_ctx; }; -/* Intel shadow batch buffer is a i915 gem object */ -struct intel_shadow_bb_entry { +struct intel_vgpu_shadow_bb { struct list_head list; struct drm_i915_gem_object *obj; + struct i915_vma *vma; void *va; - unsigned long len; u32 *bb_start_cmd_va; + unsigned int clflush; + bool accessing; }; #define workload_q_head(vgpu, ring_id) \ - (&(vgpu->workload_q_head[ring_id])) + (&(vgpu->submission.workload_q_head[ring_id])) -#define queue_workload(workload) do { \ - list_add_tail(&workload->list, \ - workload_q_head(workload->vgpu, workload->ring_id)); \ - wake_up(&workload->vgpu->gvt-> \ - scheduler.waitq[workload->ring_id]); \ -} while (0) +void intel_vgpu_queue_workload(struct intel_vgpu_workload *workload); int intel_gvt_init_workload_scheduler(struct intel_gvt *gvt); @@ -137,12 +133,24 @@ void intel_gvt_clean_workload_scheduler(struct intel_gvt *gvt); void intel_gvt_wait_vgpu_idle(struct intel_vgpu *vgpu); -int intel_vgpu_init_gvt_context(struct intel_vgpu *vgpu); +int intel_vgpu_setup_submission(struct intel_vgpu *vgpu); -void intel_vgpu_clean_gvt_context(struct intel_vgpu *vgpu); +void intel_vgpu_reset_submission(struct intel_vgpu *vgpu, + unsigned long engine_mask); -void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx); +void intel_vgpu_clean_submission(struct intel_vgpu *vgpu); -int intel_gvt_generate_request(struct intel_vgpu_workload *workload); +int intel_vgpu_select_submission_ops(struct intel_vgpu *vgpu, + unsigned long engine_mask, + unsigned int interface); + +extern const struct intel_vgpu_submission_ops +intel_vgpu_execlist_submission_ops; + +struct intel_vgpu_workload * +intel_vgpu_create_workload(struct intel_vgpu *vgpu, int ring_id, + struct execlist_ctx_descriptor_format *desc); + +void intel_vgpu_destroy_workload(struct intel_vgpu_workload *workload); #endif diff --git a/drivers/gpu/drm/i915/gvt/trace.h b/drivers/gpu/drm/i915/gvt/trace.h index 8c150381d9a4..736bd2bc5127 100644 --- a/drivers/gpu/drm/i915/gvt/trace.h +++ b/drivers/gpu/drm/i915/gvt/trace.h @@ -330,13 +330,14 @@ TRACE_EVENT(inject_msi, ); TRACE_EVENT(render_mmio, - TP_PROTO(int id, char *action, unsigned int reg, + TP_PROTO(int old_id, int new_id, char *action, unsigned int reg, unsigned int old_val, unsigned int new_val), - TP_ARGS(id, action, reg, new_val, old_val), + TP_ARGS(old_id, new_id, action, reg, old_val, new_val), TP_STRUCT__entry( - __field(int, id) + __field(int, old_id) + __field(int, new_id) __array(char, buf, GVT_TEMP_STR_LEN) __field(unsigned int, reg) __field(unsigned int, old_val) @@ -344,15 +345,17 @@ TRACE_EVENT(render_mmio, ), TP_fast_assign( - __entry->id = id; + __entry->old_id = old_id; + __entry->new_id = new_id; snprintf(__entry->buf, GVT_TEMP_STR_LEN, "%s", action); __entry->reg = reg; __entry->old_val = old_val; __entry->new_val = new_val; ), - TP_printk("VM%u %s reg %x, old %08x new %08x\n", - __entry->id, __entry->buf, __entry->reg, + TP_printk("VM%u -> VM%u %s reg %x, old %08x new %08x\n", + __entry->old_id, __entry->new_id, + __entry->buf, __entry->reg, __entry->old_val, __entry->new_val) ); diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c index 02c61a1ad56a..b87b19d8443c 100644 --- a/drivers/gpu/drm/i915/gvt/vgpu.c +++ b/drivers/gpu/drm/i915/gvt/vgpu.c @@ -38,22 +38,25 @@ void populate_pvinfo_page(struct intel_vgpu *vgpu) { /* setup the ballooning information */ - vgpu_vreg64(vgpu, vgtif_reg(magic)) = VGT_MAGIC; - vgpu_vreg(vgpu, vgtif_reg(version_major)) = 1; - vgpu_vreg(vgpu, vgtif_reg(version_minor)) = 0; - vgpu_vreg(vgpu, vgtif_reg(display_ready)) = 0; - vgpu_vreg(vgpu, vgtif_reg(vgt_id)) = vgpu->id; - vgpu_vreg(vgpu, vgtif_reg(vgt_caps)) = VGT_CAPS_FULL_48BIT_PPGTT; - vgpu_vreg(vgpu, vgtif_reg(avail_rs.mappable_gmadr.base)) = + vgpu_vreg64_t(vgpu, vgtif_reg(magic)) = VGT_MAGIC; + vgpu_vreg_t(vgpu, vgtif_reg(version_major)) = 1; + vgpu_vreg_t(vgpu, vgtif_reg(version_minor)) = 0; + vgpu_vreg_t(vgpu, vgtif_reg(display_ready)) = 0; + vgpu_vreg_t(vgpu, vgtif_reg(vgt_id)) = vgpu->id; + + vgpu_vreg_t(vgpu, vgtif_reg(vgt_caps)) = VGT_CAPS_FULL_48BIT_PPGTT; + vgpu_vreg_t(vgpu, vgtif_reg(vgt_caps)) |= VGT_CAPS_HWSP_EMULATION; + + vgpu_vreg_t(vgpu, vgtif_reg(avail_rs.mappable_gmadr.base)) = vgpu_aperture_gmadr_base(vgpu); - vgpu_vreg(vgpu, vgtif_reg(avail_rs.mappable_gmadr.size)) = + vgpu_vreg_t(vgpu, vgtif_reg(avail_rs.mappable_gmadr.size)) = vgpu_aperture_sz(vgpu); - vgpu_vreg(vgpu, vgtif_reg(avail_rs.nonmappable_gmadr.base)) = + vgpu_vreg_t(vgpu, vgtif_reg(avail_rs.nonmappable_gmadr.base)) = vgpu_hidden_gmadr_base(vgpu); - vgpu_vreg(vgpu, vgtif_reg(avail_rs.nonmappable_gmadr.size)) = + vgpu_vreg_t(vgpu, vgtif_reg(avail_rs.nonmappable_gmadr.size)) = vgpu_hidden_sz(vgpu); - vgpu_vreg(vgpu, vgtif_reg(avail_rs.fence_num)) = vgpu_fence_sz(vgpu); + vgpu_vreg_t(vgpu, vgtif_reg(avail_rs.fence_num)) = vgpu_fence_sz(vgpu); gvt_dbg_core("Populate PVINFO PAGE for vGPU %d\n", vgpu->id); gvt_dbg_core("aperture base [GMADR] 0x%llx size 0x%llx\n", @@ -226,13 +229,14 @@ void intel_gvt_deactivate_vgpu(struct intel_vgpu *vgpu) vgpu->active = false; - if (atomic_read(&vgpu->running_workload_num)) { + if (atomic_read(&vgpu->submission.running_workload_num)) { mutex_unlock(&gvt->lock); intel_gvt_wait_vgpu_idle(vgpu); mutex_lock(&gvt->lock); } intel_vgpu_stop_schedule(vgpu); + intel_vgpu_dmabuf_cleanup(vgpu); mutex_unlock(&gvt->lock); } @@ -252,16 +256,19 @@ void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu) WARN(vgpu->active, "vGPU is still active!\n"); + intel_gvt_debugfs_remove_vgpu(vgpu); idr_remove(&gvt->vgpu_idr, vgpu->id); + if (idr_is_empty(&gvt->vgpu_idr)) + intel_gvt_clean_irq(gvt); intel_vgpu_clean_sched_policy(vgpu); - intel_vgpu_clean_gvt_context(vgpu); - intel_vgpu_clean_execlist(vgpu); + intel_vgpu_clean_submission(vgpu); intel_vgpu_clean_display(vgpu); intel_vgpu_clean_opregion(vgpu); intel_vgpu_clean_gtt(vgpu); intel_gvt_hypervisor_detach_vgpu(vgpu); intel_vgpu_free_resource(vgpu); intel_vgpu_clean_mmio(vgpu); + intel_vgpu_dmabuf_cleanup(vgpu); vfree(vgpu); intel_gvt_update_vgpu_types(gvt); @@ -293,7 +300,7 @@ struct intel_vgpu *intel_gvt_create_idle_vgpu(struct intel_gvt *gvt) vgpu->gvt = gvt; for (i = 0; i < I915_NUM_ENGINES; i++) - INIT_LIST_HEAD(&vgpu->workload_q_head[i]); + INIT_LIST_HEAD(&vgpu->submission.workload_q_head[i]); ret = intel_vgpu_init_sched_policy(vgpu); if (ret) @@ -346,8 +353,8 @@ static struct intel_vgpu *__intel_gvt_create_vgpu(struct intel_gvt *gvt, vgpu->handle = param->handle; vgpu->gvt = gvt; vgpu->sched_ctl.weight = param->weight; - bitmap_zero(vgpu->tlb_handle_pending, I915_NUM_ENGINES); - + INIT_LIST_HEAD(&vgpu->dmabuf_obj_list_head); + idr_init(&vgpu->object_idr); intel_vgpu_init_cfg_space(vgpu, param->primary); ret = intel_vgpu_init_mmio(vgpu); @@ -368,32 +375,42 @@ static struct intel_vgpu *__intel_gvt_create_vgpu(struct intel_gvt *gvt, if (ret) goto out_detach_hypervisor_vgpu; - ret = intel_vgpu_init_display(vgpu, param->resolution); + ret = intel_vgpu_init_opregion(vgpu); if (ret) goto out_clean_gtt; - ret = intel_vgpu_init_execlist(vgpu); + ret = intel_vgpu_init_display(vgpu, param->resolution); if (ret) - goto out_clean_display; + goto out_clean_opregion; - ret = intel_vgpu_init_gvt_context(vgpu); + ret = intel_vgpu_setup_submission(vgpu); if (ret) - goto out_clean_execlist; + goto out_clean_display; ret = intel_vgpu_init_sched_policy(vgpu); if (ret) - goto out_clean_shadow_ctx; + goto out_clean_submission; + + ret = intel_gvt_debugfs_add_vgpu(vgpu); + if (ret) + goto out_clean_sched_policy; + + ret = intel_gvt_hypervisor_set_opregion(vgpu); + if (ret) + goto out_clean_sched_policy; mutex_unlock(&gvt->lock); return vgpu; -out_clean_shadow_ctx: - intel_vgpu_clean_gvt_context(vgpu); -out_clean_execlist: - intel_vgpu_clean_execlist(vgpu); +out_clean_sched_policy: + intel_vgpu_clean_sched_policy(vgpu); +out_clean_submission: + intel_vgpu_clean_submission(vgpu); out_clean_display: intel_vgpu_clean_display(vgpu); +out_clean_opregion: + intel_vgpu_clean_opregion(vgpu); out_clean_gtt: intel_vgpu_clean_gtt(vgpu); out_detach_hypervisor_vgpu: @@ -500,11 +517,10 @@ void intel_gvt_reset_vgpu_locked(struct intel_vgpu *vgpu, bool dmlr, mutex_lock(&gvt->lock); } - intel_vgpu_reset_execlist(vgpu, resetting_eng); - + intel_vgpu_reset_submission(vgpu, resetting_eng); /* full GPU reset or device model level reset */ if (engine_mask == ALL_ENGINES || dmlr) { - + intel_vgpu_select_submission_ops(vgpu, ALL_ENGINES, 0); /*fence will not be reset during virtual reset */ if (dmlr) { intel_vgpu_reset_gtt(vgpu); |