diff options
Diffstat (limited to 'drivers/gpu/drm/i915')
231 files changed, 27444 insertions, 17778 deletions
diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig index 33a458b7f1fc..148be8e1a090 100644 --- a/drivers/gpu/drm/i915/Kconfig +++ b/drivers/gpu/drm/i915/Kconfig @@ -131,5 +131,5 @@ config DRM_I915_GVT_KVMGT menu "drm/i915 Debugging" depends on DRM_I915 depends on EXPERT -source drivers/gpu/drm/i915/Kconfig.debug +source "drivers/gpu/drm/i915/Kconfig.debug" endmenu diff --git a/drivers/gpu/drm/i915/Kconfig.debug b/drivers/gpu/drm/i915/Kconfig.debug index 9e36ffb5eb7c..ad4d71161dda 100644 --- a/drivers/gpu/drm/i915/Kconfig.debug +++ b/drivers/gpu/drm/i915/Kconfig.debug @@ -21,11 +21,11 @@ config DRM_I915_DEBUG select DEBUG_FS select PREEMPT_COUNT select I2C_CHARDEV + select STACKDEPOT select DRM_DP_AUX_CHARDEV select X86_MSR # used by igt/pm_rpm select DRM_VGEM # used by igt/prime_vgem (dmabuf interop checks) select DRM_DEBUG_MM if DRM=y - select STACKDEPOT if DRM=y # for DRM_DEBUG_MM select DRM_DEBUG_SELFTEST select SW_SYNC # signaling validation framework (igt/syncobj*) select DRM_I915_SW_FENCE_DEBUG_OBJECTS @@ -173,6 +173,7 @@ config DRM_I915_DEBUG_RUNTIME_PM bool "Enable extra state checking for runtime PM" depends on DRM_I915 default n + select STACKDEPOT help Choose this option to turn on extra state checking for the runtime PM functionality. This may introduce overhead during diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile index 1c2857f13ad4..1787e1299b1b 100644 --- a/drivers/gpu/drm/i915/Makefile +++ b/drivers/gpu/drm/i915/Makefile @@ -22,6 +22,7 @@ subdir-ccflags-y += $(call cc-disable-warning, unused-but-set-variable) subdir-ccflags-y += $(call cc-disable-warning, sign-compare) subdir-ccflags-y += $(call cc-disable-warning, sometimes-uninitialized) subdir-ccflags-y += $(call cc-disable-warning, initializer-overrides) +subdir-ccflags-y += $(call cc-disable-warning, uninitialized) subdir-ccflags-$(CONFIG_DRM_I915_WERROR) += -Werror # Fine grained warnings disable @@ -40,9 +41,10 @@ i915-y := i915_drv.o \ i915_mm.o \ i915_params.o \ i915_pci.o \ - i915_suspend.o \ - i915_syncmap.o \ + i915_reset.o \ + i915_suspend.o \ i915_sw_fence.o \ + i915_syncmap.o \ i915_sysfs.o \ intel_csr.o \ intel_device_info.o \ @@ -55,7 +57,9 @@ i915-$(CONFIG_DEBUG_FS) += i915_debugfs.o intel_pipe_crc.o i915-$(CONFIG_PERF_EVENTS) += i915_pmu.o # GEM code -i915-y += i915_cmd_parser.o \ +i915-y += \ + i915_active.o \ + i915_cmd_parser.o \ i915_gem_batch_pool.o \ i915_gem_clflush.o \ i915_gem_context.o \ @@ -75,6 +79,7 @@ i915-y += i915_cmd_parser.o \ i915_gemfs.o \ i915_query.o \ i915_request.o \ + i915_scheduler.o \ i915_timeline.o \ i915_trace_points.o \ i915_vma.o \ @@ -112,6 +117,8 @@ i915-y += intel_audio.o \ intel_bios.o \ intel_cdclk.o \ intel_color.o \ + intel_combo_phy.o \ + intel_connector.o \ intel_display.o \ intel_dpio_phy.o \ intel_dpll_mgr.o \ @@ -120,9 +127,9 @@ i915-y += intel_audio.o \ intel_frontbuffer.o \ intel_hdcp.o \ intel_hotplug.o \ - intel_modes.o \ intel_overlay.o \ intel_psr.o \ + intel_quirks.o \ intel_sideband.o \ intel_sprite.o i915-$(CONFIG_ACPI) += intel_acpi.o intel_opregion.o @@ -142,6 +149,7 @@ i915-y += dvo_ch7017.o \ intel_dp_link_training.o \ intel_dp_mst.o \ intel_dp.o \ + intel_dsi.o \ intel_dsi_dcs_backlight.o \ intel_dsi_vbt.o \ intel_dvo.o \ @@ -153,14 +161,18 @@ i915-y += dvo_ch7017.o \ intel_sdvo.o \ intel_tv.o \ vlv_dsi.o \ - vlv_dsi_pll.o + vlv_dsi_pll.o \ + intel_vdsc.o # Post-mortem debug and GPU hang state capture i915-$(CONFIG_DRM_I915_CAPTURE_ERROR) += i915_gpu_error.o i915-$(CONFIG_DRM_I915_SELFTEST) += \ selftests/i915_random.o \ selftests/i915_selftest.o \ - selftests/igt_flush_test.o + selftests/igt_flush_test.o \ + selftests/igt_live_test.o \ + selftests/igt_reset.o \ + selftests/igt_spinner.o # virtual gpu code i915-y += i915_vgpu.o @@ -191,3 +203,4 @@ endif i915-y += intel_lpe_audio.o obj-$(CONFIG_DRM_I915) += i915.o +obj-$(CONFIG_DRM_I915_GVT_KVMGT) += gvt/kvmgt.o diff --git a/drivers/gpu/drm/i915/dvo.h b/drivers/gpu/drm/i915/dvo.h index 5e6a3013da49..16e0345b711f 100644 --- a/drivers/gpu/drm/i915/dvo.h +++ b/drivers/gpu/drm/i915/dvo.h @@ -24,7 +24,6 @@ #define _INTEL_DVO_H #include <linux/i2c.h> -#include <drm/drmP.h> #include <drm/drm_crtc.h> #include "intel_drv.h" diff --git a/drivers/gpu/drm/i915/gvt/Makefile b/drivers/gpu/drm/i915/gvt/Makefile index b016dc753db9..271fb46d4dd0 100644 --- a/drivers/gpu/drm/i915/gvt/Makefile +++ b/drivers/gpu/drm/i915/gvt/Makefile @@ -7,4 +7,3 @@ GVT_SOURCE := gvt.o aperture_gm.o handlers.o vgpu.o trace_points.o firmware.o \ ccflags-y += -I$(src) -I$(src)/$(GVT_DIR) i915-y += $(addprefix $(GVT_DIR)/, $(GVT_SOURCE)) -obj-$(CONFIG_DRM_I915_GVT_KVMGT) += $(GVT_DIR)/kvmgt.o diff --git a/drivers/gpu/drm/i915/gvt/aperture_gm.c b/drivers/gpu/drm/i915/gvt/aperture_gm.c index 359d37d5c958..1fa2f65c3cd1 100644 --- a/drivers/gpu/drm/i915/gvt/aperture_gm.c +++ b/drivers/gpu/drm/i915/gvt/aperture_gm.c @@ -180,7 +180,7 @@ static void free_vgpu_fence(struct intel_vgpu *vgpu) } mutex_unlock(&dev_priv->drm.struct_mutex); - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put_unchecked(dev_priv); } static int alloc_vgpu_fence(struct intel_vgpu *vgpu) @@ -206,7 +206,7 @@ static int alloc_vgpu_fence(struct intel_vgpu *vgpu) _clear_vgpu_fence(vgpu); mutex_unlock(&dev_priv->drm.struct_mutex); - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put_unchecked(dev_priv); return 0; out_free_fence: gvt_vgpu_err("Failed to alloc fences\n"); @@ -219,7 +219,7 @@ out_free_fence: vgpu->fence.regs[i] = NULL; } mutex_unlock(&dev_priv->drm.struct_mutex); - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put_unchecked(dev_priv); return -ENOSPC; } @@ -317,7 +317,7 @@ void intel_vgpu_reset_resource(struct intel_vgpu *vgpu) intel_runtime_pm_get(dev_priv); _clear_vgpu_fence(vgpu); - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put_unchecked(dev_priv); } /** diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c index 77edbfcb0f75..3592d04c33b2 100644 --- a/drivers/gpu/drm/i915/gvt/cmd_parser.c +++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c @@ -55,10 +55,10 @@ struct sub_op_bits { int low; }; struct decode_info { - char *name; + const char *name; int op_len; int nr_sub_op; - struct sub_op_bits *sub_op; + const struct sub_op_bits *sub_op; }; #define MAX_CMD_BUDGET 0x7fffffff @@ -375,7 +375,7 @@ typedef int (*parser_cmd_handler)(struct parser_exec_state *s); #define ADDR_FIX_5(x1, x2, x3, x4, x5) (ADDR_FIX_1(x1) | ADDR_FIX_4(x2, x3, x4, x5)) struct cmd_info { - char *name; + const char *name; u32 opcode; #define F_LEN_MASK (1U<<0) @@ -399,10 +399,10 @@ struct cmd_info { #define R_VECS (1 << VECS) #define R_ALL (R_RCS | R_VCS | R_BCS | R_VECS) /* rings that support this cmd: BLT/RCS/VCS/VECS */ - uint16_t rings; + u16 rings; /* devices that support this cmd: SNB/IVB/HSW/... */ - uint16_t devices; + u16 devices; /* which DWords are address that need fix up. * bit 0 means a 32-bit non address operand in command @@ -412,20 +412,20 @@ struct cmd_info { * No matter the address length, each address only takes * one bit in the bitmap. */ - uint16_t addr_bitmap; + u16 addr_bitmap; /* flag == F_LEN_CONST : command length * flag == F_LEN_VAR : length bias bits * Note: length is in DWord */ - uint8_t len; + u8 len; parser_cmd_handler handler; }; struct cmd_entry { struct hlist_node hlist; - struct cmd_info *info; + const struct cmd_info *info; }; enum { @@ -474,7 +474,7 @@ struct parser_exec_state { int saved_buf_addr_type; bool is_ctx_wa; - struct cmd_info *info; + const struct cmd_info *info; struct intel_vgpu_workload *workload; }; @@ -485,12 +485,12 @@ struct parser_exec_state { static unsigned long bypass_scan_mask = 0; /* ring ALL, type = 0 */ -static struct sub_op_bits sub_op_mi[] = { +static const struct sub_op_bits sub_op_mi[] = { {31, 29}, {28, 23}, }; -static struct decode_info decode_info_mi = { +static const struct decode_info decode_info_mi = { "MI", OP_LEN_MI, ARRAY_SIZE(sub_op_mi), @@ -498,12 +498,12 @@ static struct decode_info decode_info_mi = { }; /* ring RCS, command type 2 */ -static struct sub_op_bits sub_op_2d[] = { +static const struct sub_op_bits sub_op_2d[] = { {31, 29}, {28, 22}, }; -static struct decode_info decode_info_2d = { +static const struct decode_info decode_info_2d = { "2D", OP_LEN_2D, ARRAY_SIZE(sub_op_2d), @@ -511,14 +511,14 @@ static struct decode_info decode_info_2d = { }; /* ring RCS, command type 3 */ -static struct sub_op_bits sub_op_3d_media[] = { +static const struct sub_op_bits sub_op_3d_media[] = { {31, 29}, {28, 27}, {26, 24}, {23, 16}, }; -static struct decode_info decode_info_3d_media = { +static const struct decode_info decode_info_3d_media = { "3D_Media", OP_LEN_3D_MEDIA, ARRAY_SIZE(sub_op_3d_media), @@ -526,7 +526,7 @@ static struct decode_info decode_info_3d_media = { }; /* ring VCS, command type 3 */ -static struct sub_op_bits sub_op_mfx_vc[] = { +static const struct sub_op_bits sub_op_mfx_vc[] = { {31, 29}, {28, 27}, {26, 24}, @@ -534,7 +534,7 @@ static struct sub_op_bits sub_op_mfx_vc[] = { {20, 16}, }; -static struct decode_info decode_info_mfx_vc = { +static const struct decode_info decode_info_mfx_vc = { "MFX_VC", OP_LEN_MFX_VC, ARRAY_SIZE(sub_op_mfx_vc), @@ -542,7 +542,7 @@ static struct decode_info decode_info_mfx_vc = { }; /* ring VECS, command type 3 */ -static struct sub_op_bits sub_op_vebox[] = { +static const struct sub_op_bits sub_op_vebox[] = { {31, 29}, {28, 27}, {26, 24}, @@ -550,14 +550,14 @@ static struct sub_op_bits sub_op_vebox[] = { {20, 16}, }; -static struct decode_info decode_info_vebox = { +static const struct decode_info decode_info_vebox = { "VEBOX", OP_LEN_VEBOX, ARRAY_SIZE(sub_op_vebox), sub_op_vebox, }; -static struct decode_info *ring_decode_info[I915_NUM_ENGINES][8] = { +static const struct decode_info *ring_decode_info[I915_NUM_ENGINES][8] = { [RCS] = { &decode_info_mi, NULL, @@ -616,7 +616,7 @@ static struct decode_info *ring_decode_info[I915_NUM_ENGINES][8] = { static inline u32 get_opcode(u32 cmd, int ring_id) { - struct decode_info *d_info; + const struct decode_info *d_info; d_info = ring_decode_info[ring_id][CMD_TYPE(cmd)]; if (d_info == NULL) @@ -625,7 +625,7 @@ static inline u32 get_opcode(u32 cmd, int ring_id) return cmd >> (32 - d_info->op_len); } -static inline struct cmd_info *find_cmd_entry(struct intel_gvt *gvt, +static inline const struct cmd_info *find_cmd_entry(struct intel_gvt *gvt, unsigned int opcode, int ring_id) { struct cmd_entry *e; @@ -638,7 +638,7 @@ static inline struct cmd_info *find_cmd_entry(struct intel_gvt *gvt, return NULL; } -static inline struct cmd_info *get_cmd_info(struct intel_gvt *gvt, +static inline const struct cmd_info *get_cmd_info(struct intel_gvt *gvt, u32 cmd, int ring_id) { u32 opcode; @@ -657,7 +657,7 @@ static inline u32 sub_op_val(u32 cmd, u32 hi, u32 low) static inline void print_opcode(u32 cmd, int ring_id) { - struct decode_info *d_info; + const struct decode_info *d_info; int i; d_info = ring_decode_info[ring_id][CMD_TYPE(cmd)]; @@ -776,7 +776,7 @@ static inline int ip_gma_advance(struct parser_exec_state *s, return 0; } -static inline int get_cmd_length(struct cmd_info *info, u32 cmd) +static inline int get_cmd_length(const struct cmd_info *info, u32 cmd) { if ((info->flag & F_LEN_MASK) == F_LEN_CONST) return info->len; @@ -901,7 +901,8 @@ static int cmd_reg_handler(struct parser_exec_state *s, * It's good enough to support initializing mmio by lri command in * vgpu inhibit context on KBL. */ - if (IS_KABYLAKE(s->vgpu->gvt->dev_priv) && + if ((IS_KABYLAKE(s->vgpu->gvt->dev_priv) + || IS_COFFEELAKE(s->vgpu->gvt->dev_priv)) && intel_gvt_mmio_is_in_ctx(gvt, offset) && !strncmp(cmd, "lri", 3)) { intel_gvt_hypervisor_read_gpa(s->vgpu, @@ -1280,9 +1281,7 @@ static int gen8_check_mi_display_flip(struct parser_exec_state *s, if (!info->async_flip) return 0; - if (IS_SKYLAKE(dev_priv) - || IS_KABYLAKE(dev_priv) - || IS_BROXTON(dev_priv)) { + if (INTEL_GEN(dev_priv) >= 9) { stride = vgpu_vreg_t(s->vgpu, info->stride_reg) & GENMASK(9, 0); tile = (vgpu_vreg_t(s->vgpu, info->ctrl_reg) & GENMASK(12, 10)) >> 10; @@ -1310,9 +1309,7 @@ static int gen8_update_plane_mmio_from_mi_display_flip( set_mask_bits(&vgpu_vreg_t(vgpu, info->surf_reg), GENMASK(31, 12), info->surf_val << 12); - if (IS_SKYLAKE(dev_priv) - || IS_KABYLAKE(dev_priv) - || IS_BROXTON(dev_priv)) { + if (INTEL_GEN(dev_priv) >= 9) { set_mask_bits(&vgpu_vreg_t(vgpu, info->stride_reg), GENMASK(9, 0), info->stride_val); set_mask_bits(&vgpu_vreg_t(vgpu, info->ctrl_reg), GENMASK(12, 10), @@ -1336,9 +1333,7 @@ static int decode_mi_display_flip(struct parser_exec_state *s, if (IS_BROADWELL(dev_priv)) return gen8_decode_mi_display_flip(s, info); - if (IS_SKYLAKE(dev_priv) - || IS_KABYLAKE(dev_priv) - || IS_BROXTON(dev_priv)) + if (INTEL_GEN(dev_priv) >= 9) return skl_decode_mi_display_flip(s, info); return -ENODEV; @@ -1446,7 +1441,7 @@ static inline int cmd_address_audit(struct parser_exec_state *s, } if (index_mode) { - if (guest_gma >= I915_GTT_PAGE_SIZE / sizeof(u64)) { + if (guest_gma >= I915_GTT_PAGE_SIZE) { ret = -EFAULT; goto err; } @@ -1643,8 +1638,8 @@ static int batch_buffer_needs_scan(struct parser_exec_state *s) static int find_bb_size(struct parser_exec_state *s, unsigned long *bb_size) { unsigned long gma = 0; - struct cmd_info *info; - uint32_t cmd_len = 0; + const struct cmd_info *info; + u32 cmd_len = 0; bool bb_end = false; struct intel_vgpu *vgpu = s->vgpu; u32 cmd; @@ -1842,7 +1837,7 @@ static int cmd_handler_mi_batch_buffer_start(struct parser_exec_state *s) static int mi_noop_index; -static struct cmd_info cmd_info[] = { +static const struct cmd_info cmd_info[] = { {"MI_NOOP", OP_MI_NOOP, F_LEN_CONST, R_ALL, D_ALL, 0, 1, NULL}, {"MI_SET_PREDICATE", OP_MI_SET_PREDICATE, F_LEN_CONST, R_ALL, D_ALL, @@ -1900,11 +1895,11 @@ static struct cmd_info cmd_info[] = { {"MI_URB_CLEAR", OP_MI_URB_CLEAR, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, - {"ME_SEMAPHORE_SIGNAL", OP_MI_SEMAPHORE_SIGNAL, F_LEN_VAR, R_ALL, + {"MI_SEMAPHORE_SIGNAL", OP_MI_SEMAPHORE_SIGNAL, F_LEN_VAR, R_ALL, D_BDW_PLUS, 0, 8, NULL}, - {"ME_SEMAPHORE_WAIT", OP_MI_SEMAPHORE_WAIT, F_LEN_VAR, R_ALL, D_BDW_PLUS, - ADDR_FIX_1(2), 8, cmd_handler_mi_semaphore_wait}, + {"MI_SEMAPHORE_WAIT", OP_MI_SEMAPHORE_WAIT, F_LEN_VAR, R_ALL, + D_BDW_PLUS, ADDR_FIX_1(2), 8, cmd_handler_mi_semaphore_wait}, {"MI_STORE_DATA_IMM", OP_MI_STORE_DATA_IMM, F_LEN_VAR, R_ALL, D_BDW_PLUS, ADDR_FIX_1(1), 10, cmd_handler_mi_store_data_imm}, @@ -2521,7 +2516,7 @@ static void add_cmd_entry(struct intel_gvt *gvt, struct cmd_entry *e) static int cmd_parser_exec(struct parser_exec_state *s) { struct intel_vgpu *vgpu = s->vgpu; - struct cmd_info *info; + const struct cmd_info *info; u32 cmd; int ret = 0; @@ -2683,7 +2678,7 @@ static int scan_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) I915_GTT_PAGE_SIZE))) return -EINVAL; - ring_tail = wa_ctx->indirect_ctx.size + 3 * sizeof(uint32_t); + ring_tail = wa_ctx->indirect_ctx.size + 3 * sizeof(u32); ring_size = round_up(wa_ctx->indirect_ctx.size + CACHELINE_BYTES, PAGE_SIZE); gma_head = wa_ctx->indirect_ctx.guest_gma; @@ -2850,7 +2845,7 @@ put_obj: static int combine_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) { - uint32_t per_ctx_start[CACHELINE_DWORDS] = {0}; + u32 per_ctx_start[CACHELINE_DWORDS] = {0}; unsigned char *bb_start_sva; if (!wa_ctx->per_ctx.valid) @@ -2895,10 +2890,10 @@ int intel_gvt_scan_and_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) return 0; } -static struct cmd_info *find_cmd_entry_any_ring(struct intel_gvt *gvt, +static const struct cmd_info *find_cmd_entry_any_ring(struct intel_gvt *gvt, unsigned int opcode, unsigned long rings) { - struct cmd_info *info = NULL; + const struct cmd_info *info = NULL; unsigned int ring; for_each_set_bit(ring, &rings, I915_NUM_ENGINES) { @@ -2913,7 +2908,7 @@ static int init_cmd_table(struct intel_gvt *gvt) { int i; struct cmd_entry *e; - struct cmd_info *info; + const struct cmd_info *info; unsigned int gen_type; gen_type = intel_gvt_get_device_type(gvt); diff --git a/drivers/gpu/drm/i915/gvt/display.c b/drivers/gpu/drm/i915/gvt/display.c index df1e14145747..e3f9caa7839f 100644 --- a/drivers/gpu/drm/i915/gvt/display.c +++ b/drivers/gpu/drm/i915/gvt/display.c @@ -198,7 +198,8 @@ static void emulate_monitor_status_change(struct intel_vgpu *vgpu) SDE_PORTC_HOTPLUG_CPT | SDE_PORTD_HOTPLUG_CPT); - if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { + if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv) || + IS_COFFEELAKE(dev_priv)) { vgpu_vreg_t(vgpu, SDEISR) &= ~(SDE_PORTA_HOTPLUG_SPT | SDE_PORTE_HOTPLUG_SPT); vgpu_vreg_t(vgpu, SKL_FUSE_STATUS) |= @@ -273,7 +274,8 @@ static void emulate_monitor_status_change(struct intel_vgpu *vgpu) vgpu_vreg_t(vgpu, SFUSE_STRAP) |= SFUSE_STRAP_DDID_DETECTED; } - if ((IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) && + if ((IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv) || + IS_COFFEELAKE(dev_priv)) && intel_vgpu_has_monitor_on_port(vgpu, PORT_E)) { vgpu_vreg_t(vgpu, SDEISR) |= SDE_PORTE_HOTPLUG_SPT; } @@ -340,6 +342,7 @@ static int setup_virtual_dp_monitor(struct intel_vgpu *vgpu, int port_num, port->dpcd->data_valid = true; port->dpcd->data[DPCD_SINK_COUNT] = 0x1; port->type = type; + port->id = resolution; emulate_monitor_status_change(vgpu); @@ -443,6 +446,36 @@ void intel_gvt_emulate_vblank(struct intel_gvt *gvt) } /** + * intel_vgpu_emulate_hotplug - trigger hotplug event for vGPU + * @vgpu: a vGPU + * @connected: link state + * + * This function is used to trigger hotplug interrupt for vGPU + * + */ +void intel_vgpu_emulate_hotplug(struct intel_vgpu *vgpu, bool connected) +{ + struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; + + /* TODO: add more platforms support */ + if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { + if (connected) { + vgpu_vreg_t(vgpu, SFUSE_STRAP) |= + SFUSE_STRAP_DDID_DETECTED; + vgpu_vreg_t(vgpu, SDEISR) |= SDE_PORTD_HOTPLUG_CPT; + } else { + vgpu_vreg_t(vgpu, SFUSE_STRAP) &= + ~SFUSE_STRAP_DDID_DETECTED; + vgpu_vreg_t(vgpu, SDEISR) &= ~SDE_PORTD_HOTPLUG_CPT; + } + vgpu_vreg_t(vgpu, SDEIIR) |= SDE_PORTD_HOTPLUG_CPT; + vgpu_vreg_t(vgpu, PCH_PORT_HOTPLUG) |= + PORTD_HOTPLUG_STATUS_MASK; + intel_vgpu_trigger_virtual_event(vgpu, DP_D_HOTPLUG); + } +} + +/** * intel_vgpu_clean_display - clean vGPU virtual display emulation * @vgpu: a vGPU * @@ -453,7 +486,8 @@ void intel_vgpu_clean_display(struct intel_vgpu *vgpu) { struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; - if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) + if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv) || + IS_COFFEELAKE(dev_priv)) clean_virtual_dp_monitor(vgpu, PORT_D); else clean_virtual_dp_monitor(vgpu, PORT_B); @@ -476,7 +510,8 @@ int intel_vgpu_init_display(struct intel_vgpu *vgpu, u64 resolution) intel_vgpu_init_i2c_edid(vgpu); - if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) + if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv) || + IS_COFFEELAKE(dev_priv)) return setup_virtual_dp_monitor(vgpu, PORT_D, GVT_DP_D, resolution); else diff --git a/drivers/gpu/drm/i915/gvt/display.h b/drivers/gpu/drm/i915/gvt/display.h index ea7c1c525b8c..a87f33e6a23c 100644 --- a/drivers/gpu/drm/i915/gvt/display.h +++ b/drivers/gpu/drm/i915/gvt/display.h @@ -146,18 +146,19 @@ enum intel_vgpu_port_type { GVT_PORT_MAX }; +enum intel_vgpu_edid { + GVT_EDID_1024_768, + GVT_EDID_1920_1200, + GVT_EDID_NUM, +}; + struct intel_vgpu_port { /* per display EDID information */ struct intel_vgpu_edid_data *edid; /* per display DPCD information */ struct intel_vgpu_dpcd_data *dpcd; int type; -}; - -enum intel_vgpu_edid { - GVT_EDID_1024_768, - GVT_EDID_1920_1200, - GVT_EDID_NUM, + enum intel_vgpu_edid id; }; static inline char *vgpu_edid_str(enum intel_vgpu_edid id) @@ -172,6 +173,30 @@ static inline char *vgpu_edid_str(enum intel_vgpu_edid id) } } +static inline unsigned int vgpu_edid_xres(enum intel_vgpu_edid id) +{ + switch (id) { + case GVT_EDID_1024_768: + return 1024; + case GVT_EDID_1920_1200: + return 1920; + default: + return 0; + } +} + +static inline unsigned int vgpu_edid_yres(enum intel_vgpu_edid id) +{ + switch (id) { + case GVT_EDID_1024_768: + return 768; + case GVT_EDID_1920_1200: + return 1200; + default: + return 0; + } +} + void intel_gvt_emulate_vblank(struct intel_gvt *gvt); void intel_gvt_check_vblank_emulation(struct intel_gvt *gvt); diff --git a/drivers/gpu/drm/i915/gvt/dmabuf.c b/drivers/gpu/drm/i915/gvt/dmabuf.c index 51ed99a37803..69a9a1b2ea4a 100644 --- a/drivers/gpu/drm/i915/gvt/dmabuf.c +++ b/drivers/gpu/drm/i915/gvt/dmabuf.c @@ -29,7 +29,6 @@ */ #include <linux/dma-buf.h> -#include <drm/drmP.h> #include <linux/vfio.h> #include "i915_drv.h" @@ -164,9 +163,7 @@ static struct drm_i915_gem_object *vgpu_create_gem(struct drm_device *dev, obj->read_domains = I915_GEM_DOMAIN_GTT; obj->write_domain = 0; - if (IS_SKYLAKE(dev_priv) - || IS_KABYLAKE(dev_priv) - || IS_BROXTON(dev_priv)) { + if (INTEL_GEN(dev_priv) >= 9) { unsigned int tiling_mode = 0; unsigned int stride = 0; @@ -212,7 +209,7 @@ static int vgpu_get_plane_info(struct drm_device *dev, struct drm_i915_private *dev_priv = to_i915(dev); struct intel_vgpu_primary_plane_format p; struct intel_vgpu_cursor_plane_format c; - int ret; + int ret, tile_height = 1; if (plane_id == DRM_PLANE_TYPE_PRIMARY) { ret = intel_vgpu_decode_primary_plane(vgpu, &p); @@ -231,19 +228,19 @@ static int vgpu_get_plane_info(struct drm_device *dev, break; case PLANE_CTL_TILED_X: info->drm_format_mod = I915_FORMAT_MOD_X_TILED; + tile_height = 8; break; case PLANE_CTL_TILED_Y: info->drm_format_mod = I915_FORMAT_MOD_Y_TILED; + tile_height = 32; break; case PLANE_CTL_TILED_YF: info->drm_format_mod = I915_FORMAT_MOD_Yf_TILED; + tile_height = 32; break; default: gvt_vgpu_err("invalid tiling mode: %x\n", p.tiled); } - - info->size = (((p.stride * p.height * p.bpp) / 8) + - (PAGE_SIZE - 1)) >> PAGE_SHIFT; } else if (plane_id == DRM_PLANE_TYPE_CURSOR) { ret = intel_vgpu_decode_cursor_plane(vgpu, &c); if (ret) @@ -265,14 +262,13 @@ static int vgpu_get_plane_info(struct drm_device *dev, info->x_hot = UINT_MAX; info->y_hot = UINT_MAX; } - - info->size = (((info->stride * c.height * c.bpp) / 8) - + (PAGE_SIZE - 1)) >> PAGE_SHIFT; } else { gvt_vgpu_err("invalid plane id:%d\n", plane_id); return -EINVAL; } + info->size = (info->stride * roundup(info->height, tile_height) + + PAGE_SIZE - 1) >> PAGE_SHIFT; if (info->size == 0) { gvt_vgpu_err("fb size is zero\n"); return -EINVAL; diff --git a/drivers/gpu/drm/i915/gvt/edid.c b/drivers/gpu/drm/i915/gvt/edid.c index 5d4bb35bb889..1fe6124918f1 100644 --- a/drivers/gpu/drm/i915/gvt/edid.c +++ b/drivers/gpu/drm/i915/gvt/edid.c @@ -77,16 +77,32 @@ static unsigned char edid_get_byte(struct intel_vgpu *vgpu) return chr; } +static inline int cnp_get_port_from_gmbus0(u32 gmbus0) +{ + int port_select = gmbus0 & _GMBUS_PIN_SEL_MASK; + int port = -EINVAL; + + if (port_select == GMBUS_PIN_1_BXT) + port = PORT_B; + else if (port_select == GMBUS_PIN_2_BXT) + port = PORT_C; + else if (port_select == GMBUS_PIN_3_BXT) + port = PORT_D; + else if (port_select == GMBUS_PIN_4_CNP) + port = PORT_E; + return port; +} + static inline int bxt_get_port_from_gmbus0(u32 gmbus0) { int port_select = gmbus0 & _GMBUS_PIN_SEL_MASK; int port = -EINVAL; - if (port_select == 1) + if (port_select == GMBUS_PIN_1_BXT) port = PORT_B; - else if (port_select == 2) + else if (port_select == GMBUS_PIN_2_BXT) port = PORT_C; - else if (port_select == 3) + else if (port_select == GMBUS_PIN_3_BXT) port = PORT_D; return port; } @@ -96,13 +112,13 @@ static inline int get_port_from_gmbus0(u32 gmbus0) int port_select = gmbus0 & _GMBUS_PIN_SEL_MASK; int port = -EINVAL; - if (port_select == 2) + if (port_select == GMBUS_PIN_VGADDC) port = PORT_E; - else if (port_select == 4) + else if (port_select == GMBUS_PIN_DPC) port = PORT_C; - else if (port_select == 5) + else if (port_select == GMBUS_PIN_DPB) port = PORT_B; - else if (port_select == 6) + else if (port_select == GMBUS_PIN_DPD) port = PORT_D; return port; } @@ -133,6 +149,8 @@ static int gmbus0_mmio_write(struct intel_vgpu *vgpu, if (IS_BROXTON(dev_priv)) port = bxt_get_port_from_gmbus0(pin_select); + else if (IS_COFFEELAKE(dev_priv)) + port = cnp_get_port_from_gmbus0(pin_select); else port = get_port_from_gmbus0(pin_select); if (WARN_ON(port < 0)) diff --git a/drivers/gpu/drm/i915/gvt/fb_decoder.c b/drivers/gpu/drm/i915/gvt/fb_decoder.c index 85e6736f0a32..65e847392aea 100644 --- a/drivers/gpu/drm/i915/gvt/fb_decoder.c +++ b/drivers/gpu/drm/i915/gvt/fb_decoder.c @@ -151,9 +151,7 @@ static u32 intel_vgpu_get_stride(struct intel_vgpu *vgpu, int pipe, u32 stride_reg = vgpu_vreg_t(vgpu, DSPSTRIDE(pipe)) & stride_mask; u32 stride = stride_reg; - if (IS_SKYLAKE(dev_priv) - || IS_KABYLAKE(dev_priv) - || IS_BROXTON(dev_priv)) { + if (INTEL_GEN(dev_priv) >= 9) { switch (tiled) { case PLANE_CTL_TILED_LINEAR: stride = stride_reg * 64; @@ -217,9 +215,7 @@ int intel_vgpu_decode_primary_plane(struct intel_vgpu *vgpu, if (!plane->enabled) return -ENODEV; - if (IS_SKYLAKE(dev_priv) - || IS_KABYLAKE(dev_priv) - || IS_BROXTON(dev_priv)) { + if (INTEL_GEN(dev_priv) >= 9) { plane->tiled = val & PLANE_CTL_TILED_MASK; fmt = skl_format_to_drm( val & PLANE_CTL_FORMAT_MASK, @@ -260,9 +256,7 @@ int intel_vgpu_decode_primary_plane(struct intel_vgpu *vgpu, } plane->stride = intel_vgpu_get_stride(vgpu, pipe, plane->tiled, - (IS_SKYLAKE(dev_priv) - || IS_KABYLAKE(dev_priv) - || IS_BROXTON(dev_priv)) ? + (INTEL_GEN(dev_priv) >= 9) ? (_PRI_PLANE_STRIDE_MASK >> 6) : _PRI_PLANE_STRIDE_MASK, plane->bpp); diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c index c7103dd2d8d5..9814773882ec 100644 --- a/drivers/gpu/drm/i915/gvt/gtt.c +++ b/drivers/gpu/drm/i915/gvt/gtt.c @@ -750,14 +750,20 @@ static void ppgtt_free_spt(struct intel_vgpu_ppgtt_spt *spt) static void ppgtt_free_all_spt(struct intel_vgpu *vgpu) { - struct intel_vgpu_ppgtt_spt *spt; + struct intel_vgpu_ppgtt_spt *spt, *spn; struct radix_tree_iter iter; - void **slot; + LIST_HEAD(all_spt); + void __rcu **slot; + rcu_read_lock(); radix_tree_for_each_slot(slot, &vgpu->gtt.spt_tree, &iter, 0) { spt = radix_tree_deref_slot(slot); - ppgtt_free_spt(spt); + list_move(&spt->post_shadow_list, &all_spt); } + rcu_read_unlock(); + + list_for_each_entry_safe(spt, spn, &all_spt, post_shadow_list) + ppgtt_free_spt(spt); } static int ppgtt_handle_guest_write_page_table_bytes( @@ -1882,7 +1888,11 @@ struct intel_vgpu_mm *intel_vgpu_create_ppgtt_mm(struct intel_vgpu *vgpu, } list_add_tail(&mm->ppgtt_mm.list, &vgpu->gtt.ppgtt_mm_list_head); + + mutex_lock(&gvt->gtt.ppgtt_mm_lock); list_add_tail(&mm->ppgtt_mm.lru_list, &gvt->gtt.ppgtt_mm_lru_list_head); + mutex_unlock(&gvt->gtt.ppgtt_mm_lock); + return mm; } @@ -1942,7 +1952,7 @@ void _intel_vgpu_mm_release(struct kref *mm_ref) */ void intel_vgpu_unpin_mm(struct intel_vgpu_mm *mm) { - atomic_dec(&mm->pincount); + atomic_dec_if_positive(&mm->pincount); } /** @@ -1967,9 +1977,10 @@ int intel_vgpu_pin_mm(struct intel_vgpu_mm *mm) if (ret) return ret; + mutex_lock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock); list_move_tail(&mm->ppgtt_mm.lru_list, &mm->vgpu->gvt->gtt.ppgtt_mm_lru_list_head); - + mutex_unlock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock); } return 0; @@ -1980,6 +1991,8 @@ static int reclaim_one_ppgtt_mm(struct intel_gvt *gvt) struct intel_vgpu_mm *mm; struct list_head *pos, *n; + mutex_lock(&gvt->gtt.ppgtt_mm_lock); + list_for_each_safe(pos, n, &gvt->gtt.ppgtt_mm_lru_list_head) { mm = container_of(pos, struct intel_vgpu_mm, ppgtt_mm.lru_list); @@ -1987,9 +2000,11 @@ static int reclaim_one_ppgtt_mm(struct intel_gvt *gvt) continue; list_del_init(&mm->ppgtt_mm.lru_list); + mutex_unlock(&gvt->gtt.ppgtt_mm_lock); invalidate_ppgtt_mm(mm); return 1; } + mutex_unlock(&gvt->gtt.ppgtt_mm_lock); return 0; } @@ -2659,6 +2674,7 @@ int intel_gvt_init_gtt(struct intel_gvt *gvt) } } INIT_LIST_HEAD(&gvt->gtt.ppgtt_mm_lru_list_head); + mutex_init(&gvt->gtt.ppgtt_mm_lock); return 0; } @@ -2699,7 +2715,9 @@ void intel_vgpu_invalidate_ppgtt(struct intel_vgpu *vgpu) list_for_each_safe(pos, n, &vgpu->gtt.ppgtt_mm_list_head) { mm = container_of(pos, struct intel_vgpu_mm, ppgtt_mm.list); if (mm->type == INTEL_GVT_MM_PPGTT) { + mutex_lock(&vgpu->gvt->gtt.ppgtt_mm_lock); list_del_init(&mm->ppgtt_mm.lru_list); + mutex_unlock(&vgpu->gvt->gtt.ppgtt_mm_lock); if (mm->ppgtt_mm.shadowed) invalidate_ppgtt_mm(mm); } diff --git a/drivers/gpu/drm/i915/gvt/gtt.h b/drivers/gpu/drm/i915/gvt/gtt.h index d8cb04cc946d..edb610dc5d86 100644 --- a/drivers/gpu/drm/i915/gvt/gtt.h +++ b/drivers/gpu/drm/i915/gvt/gtt.h @@ -88,6 +88,7 @@ struct intel_gvt_gtt { void (*mm_free_page_table)(struct intel_vgpu_mm *mm); struct list_head oos_page_use_list_head; struct list_head oos_page_free_list_head; + struct mutex ppgtt_mm_lock; struct list_head ppgtt_mm_lru_list_head; struct page *scratch_page; diff --git a/drivers/gpu/drm/i915/gvt/gvt.c b/drivers/gpu/drm/i915/gvt/gvt.c index 6ef5a7fc70df..43f4242062dd 100644 --- a/drivers/gpu/drm/i915/gvt/gvt.c +++ b/drivers/gpu/drm/i915/gvt/gvt.c @@ -185,54 +185,9 @@ static const struct intel_gvt_ops intel_gvt_ops = { .vgpu_query_plane = intel_vgpu_query_plane, .vgpu_get_dmabuf = intel_vgpu_get_dmabuf, .write_protect_handler = intel_vgpu_page_track_handler, + .emulate_hotplug = intel_vgpu_emulate_hotplug, }; -/** - * intel_gvt_init_host - Load MPT modules and detect if we're running in host - * - * This function is called at the driver loading stage. If failed to find a - * loadable MPT module or detect currently we're running in a VM, then GVT-g - * will be disabled - * - * Returns: - * Zero on success, negative error code if failed. - * - */ -int intel_gvt_init_host(void) -{ - if (intel_gvt_host.initialized) - return 0; - - /* Xen DOM U */ - if (xen_domain() && !xen_initial_domain()) - return -ENODEV; - - /* Try to load MPT modules for hypervisors */ - if (xen_initial_domain()) { - /* In Xen dom0 */ - intel_gvt_host.mpt = try_then_request_module( - symbol_get(xengt_mpt), "xengt"); - intel_gvt_host.hypervisor_type = INTEL_GVT_HYPERVISOR_XEN; - } else { -#if IS_ENABLED(CONFIG_DRM_I915_GVT_KVMGT) - /* not in Xen. Try KVMGT */ - intel_gvt_host.mpt = try_then_request_module( - symbol_get(kvmgt_mpt), "kvmgt"); - intel_gvt_host.hypervisor_type = INTEL_GVT_HYPERVISOR_KVM; -#endif - } - - /* Fail to load MPT modules - bail out */ - if (!intel_gvt_host.mpt) - return -EINVAL; - - gvt_dbg_core("Running with hypervisor %s in host mode\n", - supported_hypervisors[intel_gvt_host.hypervisor_type]); - - intel_gvt_host.initialized = true; - return 0; -} - static void init_device_info(struct intel_gvt *gvt) { struct intel_gvt_device_info *info = &gvt->device_info; @@ -316,7 +271,6 @@ void intel_gvt_clean_device(struct drm_i915_private *dev_priv) return; intel_gvt_destroy_idle_vgpu(gvt->idle_vgpu); - intel_gvt_hypervisor_host_exit(&dev_priv->drm.pdev->dev, gvt); intel_gvt_cleanup_vgpu_type_groups(gvt); intel_gvt_clean_vgpu_types(gvt); @@ -352,13 +306,6 @@ int intel_gvt_init_device(struct drm_i915_private *dev_priv) struct intel_vgpu *vgpu; int ret; - /* - * Cannot initialize GVT device without intel_gvt_host gets - * initialized first. - */ - if (WARN_ON(!intel_gvt_host.initialized)) - return -EINVAL; - if (WARN_ON(dev_priv->gvt)) return -EEXIST; @@ -420,13 +367,6 @@ int intel_gvt_init_device(struct drm_i915_private *dev_priv) goto out_clean_types; } - ret = intel_gvt_hypervisor_host_init(&dev_priv->drm.pdev->dev, gvt, - &intel_gvt_ops); - if (ret) { - gvt_err("failed to register gvt-g host device: %d\n", ret); - goto out_clean_types; - } - vgpu = intel_gvt_create_idle_vgpu(gvt); if (IS_ERR(vgpu)) { ret = PTR_ERR(vgpu); @@ -437,10 +377,12 @@ int intel_gvt_init_device(struct drm_i915_private *dev_priv) ret = intel_gvt_debugfs_init(gvt); if (ret) - gvt_err("debugfs registeration failed, go on.\n"); + gvt_err("debugfs registration failed, go on.\n"); gvt_dbg_core("gvt device initialization is done\n"); dev_priv->gvt = gvt; + intel_gvt_host.dev = &dev_priv->drm.pdev->dev; + intel_gvt_host.initialized = true; return 0; out_clean_types: @@ -467,6 +409,45 @@ out_clean_idr: return ret; } -#if IS_ENABLED(CONFIG_DRM_I915_GVT_KVMGT) -MODULE_SOFTDEP("pre: kvmgt"); -#endif +int +intel_gvt_register_hypervisor(struct intel_gvt_mpt *m) +{ + int ret; + void *gvt; + + if (!intel_gvt_host.initialized) + return -ENODEV; + + if (m->type != INTEL_GVT_HYPERVISOR_KVM && + m->type != INTEL_GVT_HYPERVISOR_XEN) + return -EINVAL; + + /* Get a reference for device model module */ + if (!try_module_get(THIS_MODULE)) + return -ENODEV; + + intel_gvt_host.mpt = m; + intel_gvt_host.hypervisor_type = m->type; + gvt = (void *)kdev_to_i915(intel_gvt_host.dev)->gvt; + + ret = intel_gvt_hypervisor_host_init(intel_gvt_host.dev, gvt, + &intel_gvt_ops); + if (ret < 0) { + gvt_err("Failed to init %s hypervisor module\n", + supported_hypervisors[intel_gvt_host.hypervisor_type]); + module_put(THIS_MODULE); + return -ENODEV; + } + gvt_dbg_core("Running with hypervisor %s in host mode\n", + supported_hypervisors[intel_gvt_host.hypervisor_type]); + return 0; +} +EXPORT_SYMBOL_GPL(intel_gvt_register_hypervisor); + +void +intel_gvt_unregister_hypervisor(void) +{ + intel_gvt_hypervisor_host_exit(intel_gvt_host.dev); + module_put(THIS_MODULE); +} +EXPORT_SYMBOL_GPL(intel_gvt_unregister_hypervisor); diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h index 31f6cdbe5c42..8bce09de4b82 100644 --- a/drivers/gpu/drm/i915/gvt/gvt.h +++ b/drivers/gpu/drm/i915/gvt/gvt.h @@ -52,12 +52,8 @@ #define GVT_MAX_VGPU 8 -enum { - INTEL_GVT_HYPERVISOR_XEN = 0, - INTEL_GVT_HYPERVISOR_KVM, -}; - struct intel_gvt_host { + struct device *dev; bool initialized; int hypervisor_type; struct intel_gvt_mpt *mpt; @@ -159,6 +155,10 @@ struct intel_vgpu_submission { struct kmem_cache *workloads; atomic_t running_workload_num; struct i915_gem_context *shadow_ctx; + union { + u64 i915_context_pml4; + u64 i915_context_pdps[GEN8_3LVL_PDPES]; + }; DECLARE_BITMAP(shadow_ctx_desc_updated, I915_NUM_ENGINES); DECLARE_BITMAP(tlb_handle_pending, I915_NUM_ENGINES); void *ring_scan_buffer[I915_NUM_ENGINES]; @@ -536,6 +536,8 @@ int intel_vgpu_emulate_cfg_read(struct intel_vgpu *vgpu, unsigned int offset, int intel_vgpu_emulate_cfg_write(struct intel_vgpu *vgpu, unsigned int offset, void *p_data, unsigned int bytes); +void intel_vgpu_emulate_hotplug(struct intel_vgpu *vgpu, bool connected); + static inline u64 intel_vgpu_get_bar_gpa(struct intel_vgpu *vgpu, int bar) { /* We are 64bit bar. */ @@ -577,6 +579,7 @@ struct intel_gvt_ops { int (*vgpu_get_dmabuf)(struct intel_vgpu *vgpu, unsigned int); int (*write_protect_handler)(struct intel_vgpu *, u64, void *, unsigned int); + void (*emulate_hotplug)(struct intel_vgpu *vgpu, bool connected); }; @@ -593,7 +596,7 @@ static inline void mmio_hw_access_pre(struct drm_i915_private *dev_priv) static inline void mmio_hw_access_post(struct drm_i915_private *dev_priv) { - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put_unchecked(dev_priv); } /** diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c index aa280bb07125..bc64b810e0d5 100644 --- a/drivers/gpu/drm/i915/gvt/handlers.c +++ b/drivers/gpu/drm/i915/gvt/handlers.c @@ -57,6 +57,8 @@ unsigned long intel_gvt_get_device_type(struct intel_gvt *gvt) return D_KBL; else if (IS_BROXTON(gvt->dev_priv)) return D_BXT; + else if (IS_COFFEELAKE(gvt->dev_priv)) + return D_CFL; return 0; } @@ -276,14 +278,12 @@ static int mul_force_wake_write(struct intel_vgpu *vgpu, unsigned int offset, void *p_data, unsigned int bytes) { u32 old, new; - uint32_t ack_reg_offset; + u32 ack_reg_offset; old = vgpu_vreg(vgpu, offset); new = CALC_MODE_MASK_REG(old, *(u32 *)p_data); - if (IS_SKYLAKE(vgpu->gvt->dev_priv) - || IS_KABYLAKE(vgpu->gvt->dev_priv) - || IS_BROXTON(vgpu->gvt->dev_priv)) { + if (INTEL_GEN(vgpu->gvt->dev_priv) >= 9) { switch (offset) { case FORCEWAKE_RENDER_GEN9_REG: ack_reg_offset = FORCEWAKE_ACK_RENDER_GEN9_REG; @@ -475,6 +475,7 @@ static i915_reg_t force_nonpriv_white_list[] = { _MMIO(0x7704), _MMIO(0x7708), _MMIO(0x770c), + _MMIO(0x83a8), _MMIO(0xb110), GEN8_L3SQCREG4,//_MMIO(0xb118) _MMIO(0xe100), @@ -832,7 +833,7 @@ static int dp_aux_ch_ctl_trans_done(struct intel_vgpu *vgpu, u32 value, } static void dp_aux_ch_ctl_link_training(struct intel_vgpu_dpcd_data *dpcd, - uint8_t t) + u8 t) { if ((t & DPCD_TRAINING_PATTERN_SET_MASK) == DPCD_TRAINING_PATTERN_1) { /* training pattern 1 for CR */ @@ -888,9 +889,7 @@ static int dp_aux_ch_ctl_mmio_write(struct intel_vgpu *vgpu, write_vreg(vgpu, offset, p_data, bytes); data = vgpu_vreg(vgpu, offset); - if ((IS_SKYLAKE(vgpu->gvt->dev_priv) - || IS_KABYLAKE(vgpu->gvt->dev_priv) - || IS_BROXTON(vgpu->gvt->dev_priv)) + if ((INTEL_GEN(vgpu->gvt->dev_priv) >= 9) && offset != _REG_SKL_DP_AUX_CH_CTL(port_index)) { /* SKL DPB/C/D aux ctl register changed */ return 0; @@ -918,7 +917,7 @@ static int dp_aux_ch_ctl_mmio_write(struct intel_vgpu *vgpu, if (op == GVT_AUX_NATIVE_WRITE) { int t; - uint8_t buf[16]; + u8 buf[16]; if ((addr + len + 1) >= DPCD_SIZE) { /* @@ -1406,7 +1405,8 @@ static int mailbox_write(struct intel_vgpu *vgpu, unsigned int offset, switch (cmd) { case GEN9_PCODE_READ_MEM_LATENCY: if (IS_SKYLAKE(vgpu->gvt->dev_priv) - || IS_KABYLAKE(vgpu->gvt->dev_priv)) { + || IS_KABYLAKE(vgpu->gvt->dev_priv) + || IS_COFFEELAKE(vgpu->gvt->dev_priv)) { /** * "Read memory latency" command on gen9. * Below memory latency values are read @@ -1430,7 +1430,8 @@ static int mailbox_write(struct intel_vgpu *vgpu, unsigned int offset, break; case SKL_PCODE_CDCLK_CONTROL: if (IS_SKYLAKE(vgpu->gvt->dev_priv) - || IS_KABYLAKE(vgpu->gvt->dev_priv)) + || IS_KABYLAKE(vgpu->gvt->dev_priv) + || IS_COFFEELAKE(vgpu->gvt->dev_priv)) *data0 = SKL_CDCLK_READY_FOR_CHANGE; break; case GEN6_PCODE_READ_RC6VIDS: @@ -2798,6 +2799,7 @@ static int init_broadwell_mmio_info(struct intel_gvt *gvt) MMIO_DFH(_MMIO(0xe2a0), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); MMIO_DFH(_MMIO(0xe2b0), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); MMIO_DFH(_MMIO(0xe2c0), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); + MMIO_DFH(_MMIO(0x21f0), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); return 0; } @@ -3040,8 +3042,8 @@ static int init_skl_mmio_info(struct intel_gvt *gvt) MMIO_DFH(GEN9_WM_CHICKEN3, D_SKL_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); - MMIO_D(_MMIO(0x4ab8), D_KBL); - MMIO_D(_MMIO(0x2248), D_KBL | D_SKL); + MMIO_D(_MMIO(0x4ab8), D_KBL | D_CFL); + MMIO_D(_MMIO(0x2248), D_SKL_PLUS); return 0; } @@ -3301,7 +3303,8 @@ int intel_gvt_setup_mmio_info(struct intel_gvt *gvt) if (ret) goto err; } else if (IS_SKYLAKE(dev_priv) - || IS_KABYLAKE(dev_priv)) { + || IS_KABYLAKE(dev_priv) + || IS_COFFEELAKE(dev_priv)) { ret = init_broadwell_mmio_info(gvt); if (ret) goto err; diff --git a/drivers/gpu/drm/i915/gvt/hypercall.h b/drivers/gpu/drm/i915/gvt/hypercall.h index 5af11cf1b482..4862fb12778e 100644 --- a/drivers/gpu/drm/i915/gvt/hypercall.h +++ b/drivers/gpu/drm/i915/gvt/hypercall.h @@ -33,15 +33,21 @@ #ifndef _GVT_HYPERCALL_H_ #define _GVT_HYPERCALL_H_ +enum hypervisor_type { + INTEL_GVT_HYPERVISOR_XEN = 0, + INTEL_GVT_HYPERVISOR_KVM, +}; + /* * Specific GVT-g MPT modules function collections. Currently GVT-g supports * both Xen and KVM by providing dedicated hypervisor-related MPT modules. */ struct intel_gvt_mpt { + enum hypervisor_type type; int (*host_init)(struct device *dev, void *gvt, const void *ops); - void (*host_exit)(struct device *dev, void *gvt); + void (*host_exit)(struct device *dev); int (*attach_vgpu)(void *vgpu, unsigned long *handle); - void (*detach_vgpu)(unsigned long handle); + void (*detach_vgpu)(void *vgpu); int (*inject_msi)(unsigned long handle, u32 addr, u16 data); unsigned long (*from_virt_to_mfn)(void *p); int (*enable_page_track)(unsigned long handle, u64 gfn); @@ -61,12 +67,12 @@ struct intel_gvt_mpt { int (*set_trap_area)(unsigned long handle, u64 start, u64 end, bool map); int (*set_opregion)(void *vgpu); + int (*set_edid)(void *vgpu, int port_num); int (*get_vfio_device)(void *vgpu); void (*put_vfio_device)(void *vgpu); bool (*is_valid_gfn)(unsigned long handle, unsigned long gfn); }; extern struct intel_gvt_mpt xengt_mpt; -extern struct intel_gvt_mpt kvmgt_mpt; #endif /* _GVT_HYPERCALL_H_ */ diff --git a/drivers/gpu/drm/i915/gvt/interrupt.c b/drivers/gpu/drm/i915/gvt/interrupt.c index 5daa23ae566b..67125c5eec6e 100644 --- a/drivers/gpu/drm/i915/gvt/interrupt.c +++ b/drivers/gpu/drm/i915/gvt/interrupt.c @@ -126,7 +126,7 @@ static const char * const irq_name[INTEL_GVT_EVENT_MAX] = { [FDI_RX_INTERRUPTS_TRANSCODER_C] = "FDI RX Interrupts Combined C", [AUDIO_CP_CHANGE_TRANSCODER_C] = "Audio CP Change Transcoder C", [AUDIO_CP_REQUEST_TRANSCODER_C] = "Audio CP Request Transcoder C", - [ERR_AND_DBG] = "South Error and Debug Interupts Combined", + [ERR_AND_DBG] = "South Error and Debug Interrupts Combined", [GMBUS] = "Gmbus", [SDVO_B_HOTPLUG] = "SDVO B hotplug", [CRT_HOTPLUG] = "CRT Hotplug", @@ -581,9 +581,7 @@ static void gen8_init_irq( SET_BIT_INFO(irq, 4, PRIMARY_C_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_C); SET_BIT_INFO(irq, 5, SPRITE_C_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_C); - } else if (IS_SKYLAKE(gvt->dev_priv) - || IS_KABYLAKE(gvt->dev_priv) - || IS_BROXTON(gvt->dev_priv)) { + } else if (INTEL_GEN(gvt->dev_priv) >= 9) { SET_BIT_INFO(irq, 25, AUX_CHANNEL_B, INTEL_GVT_IRQ_INFO_DE_PORT); SET_BIT_INFO(irq, 26, AUX_CHANNEL_C, INTEL_GVT_IRQ_INFO_DE_PORT); SET_BIT_INFO(irq, 27, AUX_CHANNEL_D, INTEL_GVT_IRQ_INFO_DE_PORT); diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c index c1072143da1d..a68addf95c23 100644 --- a/drivers/gpu/drm/i915/gvt/kvmgt.c +++ b/drivers/gpu/drm/i915/gvt/kvmgt.c @@ -57,6 +57,8 @@ static const struct intel_gvt_ops *intel_gvt_ops; #define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT) #define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1) +#define EDID_BLOB_OFFSET (PAGE_SIZE/2) + #define OPREGION_SIGNATURE "IntelGraphicsMem" struct vfio_region; @@ -76,6 +78,11 @@ struct vfio_region { void *data; }; +struct vfio_edid_region { + struct vfio_region_gfx_edid vfio_edid_regs; + void *edid_blob; +}; + struct kvmgt_pgfn { gfn_t gfn; struct hlist_node hnode; @@ -427,6 +434,111 @@ static const struct intel_vgpu_regops intel_vgpu_regops_opregion = { .release = intel_vgpu_reg_release_opregion, }; +static int handle_edid_regs(struct intel_vgpu *vgpu, + struct vfio_edid_region *region, char *buf, + size_t count, u16 offset, bool is_write) +{ + struct vfio_region_gfx_edid *regs = ®ion->vfio_edid_regs; + unsigned int data; + + if (offset + count > sizeof(*regs)) + return -EINVAL; + + if (count != 4) + return -EINVAL; + + if (is_write) { + data = *((unsigned int *)buf); + switch (offset) { + case offsetof(struct vfio_region_gfx_edid, link_state): + if (data == VFIO_DEVICE_GFX_LINK_STATE_UP) { + if (!drm_edid_block_valid( + (u8 *)region->edid_blob, + 0, + true, + NULL)) { + gvt_vgpu_err("invalid EDID blob\n"); + return -EINVAL; + } + intel_gvt_ops->emulate_hotplug(vgpu, true); + } else if (data == VFIO_DEVICE_GFX_LINK_STATE_DOWN) + intel_gvt_ops->emulate_hotplug(vgpu, false); + else { + gvt_vgpu_err("invalid EDID link state %d\n", + regs->link_state); + return -EINVAL; + } + regs->link_state = data; + break; + case offsetof(struct vfio_region_gfx_edid, edid_size): + if (data > regs->edid_max_size) { + gvt_vgpu_err("EDID size is bigger than %d!\n", + regs->edid_max_size); + return -EINVAL; + } + regs->edid_size = data; + break; + default: + /* read-only regs */ + gvt_vgpu_err("write read-only EDID region at offset %d\n", + offset); + return -EPERM; + } + } else { + memcpy(buf, (char *)regs + offset, count); + } + + return count; +} + +static int handle_edid_blob(struct vfio_edid_region *region, char *buf, + size_t count, u16 offset, bool is_write) +{ + if (offset + count > region->vfio_edid_regs.edid_size) + return -EINVAL; + + if (is_write) + memcpy(region->edid_blob + offset, buf, count); + else + memcpy(buf, region->edid_blob + offset, count); + + return count; +} + +static size_t intel_vgpu_reg_rw_edid(struct intel_vgpu *vgpu, char *buf, + size_t count, loff_t *ppos, bool iswrite) +{ + int ret; + unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) - + VFIO_PCI_NUM_REGIONS; + struct vfio_edid_region *region = + (struct vfio_edid_region *)vgpu->vdev.region[i].data; + loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK; + + if (pos < region->vfio_edid_regs.edid_offset) { + ret = handle_edid_regs(vgpu, region, buf, count, pos, iswrite); + } else { + pos -= EDID_BLOB_OFFSET; + ret = handle_edid_blob(region, buf, count, pos, iswrite); + } + + if (ret < 0) + gvt_vgpu_err("failed to access EDID region\n"); + + return ret; +} + +static void intel_vgpu_reg_release_edid(struct intel_vgpu *vgpu, + struct vfio_region *region) +{ + kfree(region->data); +} + +static const struct intel_vgpu_regops intel_vgpu_regops_edid = { + .rw = intel_vgpu_reg_rw_edid, + .release = intel_vgpu_reg_release_edid, +}; + static int intel_vgpu_register_reg(struct intel_vgpu *vgpu, unsigned int type, unsigned int subtype, const struct intel_vgpu_regops *ops, @@ -493,6 +605,36 @@ static int kvmgt_set_opregion(void *p_vgpu) return ret; } +static int kvmgt_set_edid(void *p_vgpu, int port_num) +{ + struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu; + struct intel_vgpu_port *port = intel_vgpu_port(vgpu, port_num); + struct vfio_edid_region *base; + int ret; + + base = kzalloc(sizeof(*base), GFP_KERNEL); + if (!base) + return -ENOMEM; + + /* TODO: Add multi-port and EDID extension block support */ + base->vfio_edid_regs.edid_offset = EDID_BLOB_OFFSET; + base->vfio_edid_regs.edid_max_size = EDID_SIZE; + base->vfio_edid_regs.edid_size = EDID_SIZE; + base->vfio_edid_regs.max_xres = vgpu_edid_xres(port->id); + base->vfio_edid_regs.max_yres = vgpu_edid_yres(port->id); + base->edid_blob = port->edid->edid_block; + + ret = intel_vgpu_register_reg(vgpu, + VFIO_REGION_TYPE_GFX, + VFIO_REGION_SUBTYPE_GFX_EDID, + &intel_vgpu_regops_edid, EDID_SIZE, + VFIO_REGION_INFO_FLAG_READ | + VFIO_REGION_INFO_FLAG_WRITE | + VFIO_REGION_INFO_FLAG_CAPS, base); + + return ret; +} + static void kvmgt_put_vfio_device(void *vgpu) { if (WARN_ON(!((struct intel_vgpu *)vgpu)->vdev.vfio_device)) @@ -627,6 +769,12 @@ static int intel_vgpu_open(struct mdev_device *mdev) goto undo_iommu; } + /* Take a module reference as mdev core doesn't take + * a reference for vendor driver. + */ + if (!try_module_get(THIS_MODULE)) + goto undo_group; + ret = kvmgt_guest_init(mdev); if (ret) goto undo_group; @@ -679,6 +827,9 @@ static void __intel_vgpu_release(struct intel_vgpu *vgpu) &vgpu->vdev.group_notifier); WARN(ret, "vfio_unregister_notifier for group failed: %d\n", ret); + /* dereference module reference taken at open */ + module_put(THIS_MODULE); + info = (struct kvmgt_guest_info *)vgpu->handle; kvmgt_guest_exit(info); @@ -703,7 +854,7 @@ static void intel_vgpu_release_work(struct work_struct *work) __intel_vgpu_release(vgpu); } -static uint64_t intel_vgpu_get_bar_addr(struct intel_vgpu *vgpu, int bar) +static u64 intel_vgpu_get_bar_addr(struct intel_vgpu *vgpu, int bar) { u32 start_lo, start_hi; u32 mem_type; @@ -730,10 +881,10 @@ static uint64_t intel_vgpu_get_bar_addr(struct intel_vgpu *vgpu, int bar) return ((u64)start_hi << 32) | start_lo; } -static int intel_vgpu_bar_rw(struct intel_vgpu *vgpu, int bar, uint64_t off, +static int intel_vgpu_bar_rw(struct intel_vgpu *vgpu, int bar, u64 off, void *buf, unsigned int count, bool is_write) { - uint64_t bar_start = intel_vgpu_get_bar_addr(vgpu, bar); + u64 bar_start = intel_vgpu_get_bar_addr(vgpu, bar); int ret; if (is_write) @@ -745,16 +896,16 @@ static int intel_vgpu_bar_rw(struct intel_vgpu *vgpu, int bar, uint64_t off, return ret; } -static inline bool intel_vgpu_in_aperture(struct intel_vgpu *vgpu, uint64_t off) +static inline bool intel_vgpu_in_aperture(struct intel_vgpu *vgpu, u64 off) { return off >= vgpu_aperture_offset(vgpu) && off < vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu); } -static int intel_vgpu_aperture_rw(struct intel_vgpu *vgpu, uint64_t off, +static int intel_vgpu_aperture_rw(struct intel_vgpu *vgpu, u64 off, void *buf, unsigned long count, bool is_write) { - void *aperture_va; + void __iomem *aperture_va; if (!intel_vgpu_in_aperture(vgpu, off) || !intel_vgpu_in_aperture(vgpu, off + count)) { @@ -769,9 +920,9 @@ static int intel_vgpu_aperture_rw(struct intel_vgpu *vgpu, uint64_t off, return -EIO; if (is_write) - memcpy(aperture_va + offset_in_page(off), buf, count); + memcpy_toio(aperture_va + offset_in_page(off), buf, count); else - memcpy(buf, aperture_va + offset_in_page(off), count); + memcpy_fromio(buf, aperture_va + offset_in_page(off), count); io_mapping_unmap(aperture_va); @@ -783,7 +934,7 @@ static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf, { struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); - uint64_t pos = *ppos & VFIO_PCI_OFFSET_MASK; + u64 pos = *ppos & VFIO_PCI_OFFSET_MASK; int ret = -EINVAL; @@ -996,7 +1147,7 @@ static int intel_vgpu_mmap(struct mdev_device *mdev, struct vm_area_struct *vma) { unsigned int index; u64 virtaddr; - unsigned long req_size, pgoff = 0; + unsigned long req_size, pgoff, req_start; pgprot_t pg_prot; struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); @@ -1014,7 +1165,17 @@ static int intel_vgpu_mmap(struct mdev_device *mdev, struct vm_area_struct *vma) pg_prot = vma->vm_page_prot; virtaddr = vma->vm_start; req_size = vma->vm_end - vma->vm_start; - pgoff = vgpu_aperture_pa_base(vgpu) >> PAGE_SHIFT; + pgoff = vma->vm_pgoff & + ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); + req_start = pgoff << PAGE_SHIFT; + + if (!intel_vgpu_in_aperture(vgpu, req_start)) + return -EINVAL; + if (req_start + req_size > + vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu)) + return -EINVAL; + + pgoff = (gvt_aperture_pa_base(vgpu->gvt) >> PAGE_SHIFT) + pgoff; return remap_pfn_range(vma, virtaddr, pgoff, req_size, pg_prot); } @@ -1029,7 +1190,7 @@ static int intel_vgpu_get_irq_count(struct intel_vgpu *vgpu, int type) static int intel_vgpu_set_intx_mask(struct intel_vgpu *vgpu, unsigned int index, unsigned int start, - unsigned int count, uint32_t flags, + unsigned int count, u32 flags, void *data) { return 0; @@ -1037,21 +1198,21 @@ static int intel_vgpu_set_intx_mask(struct intel_vgpu *vgpu, static int intel_vgpu_set_intx_unmask(struct intel_vgpu *vgpu, unsigned int index, unsigned int start, - unsigned int count, uint32_t flags, void *data) + unsigned int count, u32 flags, void *data) { return 0; } static int intel_vgpu_set_intx_trigger(struct intel_vgpu *vgpu, unsigned int index, unsigned int start, unsigned int count, - uint32_t flags, void *data) + u32 flags, void *data) { return 0; } static int intel_vgpu_set_msi_trigger(struct intel_vgpu *vgpu, unsigned int index, unsigned int start, unsigned int count, - uint32_t flags, void *data) + u32 flags, void *data) { struct eventfd_ctx *trigger; @@ -1070,12 +1231,12 @@ static int intel_vgpu_set_msi_trigger(struct intel_vgpu *vgpu, return 0; } -static int intel_vgpu_set_irqs(struct intel_vgpu *vgpu, uint32_t flags, +static int intel_vgpu_set_irqs(struct intel_vgpu *vgpu, u32 flags, unsigned int index, unsigned int start, unsigned int count, void *data) { int (*func)(struct intel_vgpu *vgpu, unsigned int index, - unsigned int start, unsigned int count, uint32_t flags, + unsigned int start, unsigned int count, u32 flags, void *data) = NULL; switch (index) { @@ -1467,7 +1628,7 @@ static int kvmgt_host_init(struct device *dev, void *gvt, const void *ops) return mdev_register_device(dev, &intel_vgpu_ops); } -static void kvmgt_host_exit(struct device *dev, void *gvt) +static void kvmgt_host_exit(struct device *dev) { mdev_unregister_device(dev); } @@ -1662,9 +1823,21 @@ static int kvmgt_attach_vgpu(void *vgpu, unsigned long *handle) return 0; } -static void kvmgt_detach_vgpu(unsigned long handle) +static void kvmgt_detach_vgpu(void *p_vgpu) { - /* nothing to do here */ + int i; + struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu; + + if (!vgpu->vdev.region) + return; + + for (i = 0; i < vgpu->vdev.num_regions; i++) + if (vgpu->vdev.region[i].ops->release) + vgpu->vdev.region[i].ops->release(vgpu, + &vgpu->vdev.region[i]); + vgpu->vdev.num_regions = 0; + kfree(vgpu->vdev.region); + vgpu->vdev.region = NULL; } static int kvmgt_inject_msi(unsigned long handle, u32 addr, u16 data) @@ -1849,7 +2022,8 @@ static bool kvmgt_is_valid_gfn(unsigned long handle, unsigned long gfn) return ret; } -struct intel_gvt_mpt kvmgt_mpt = { +static struct intel_gvt_mpt kvmgt_mpt = { + .type = INTEL_GVT_HYPERVISOR_KVM, .host_init = kvmgt_host_init, .host_exit = kvmgt_host_exit, .attach_vgpu = kvmgt_attach_vgpu, @@ -1864,19 +2038,22 @@ struct intel_gvt_mpt kvmgt_mpt = { .dma_map_guest_page = kvmgt_dma_map_guest_page, .dma_unmap_guest_page = kvmgt_dma_unmap_guest_page, .set_opregion = kvmgt_set_opregion, + .set_edid = kvmgt_set_edid, .get_vfio_device = kvmgt_get_vfio_device, .put_vfio_device = kvmgt_put_vfio_device, .is_valid_gfn = kvmgt_is_valid_gfn, }; -EXPORT_SYMBOL_GPL(kvmgt_mpt); static int __init kvmgt_init(void) { + if (intel_gvt_register_hypervisor(&kvmgt_mpt) < 0) + return -ENODEV; return 0; } static void __exit kvmgt_exit(void) { + intel_gvt_unregister_hypervisor(); } module_init(kvmgt_init); diff --git a/drivers/gpu/drm/i915/gvt/mmio.c b/drivers/gpu/drm/i915/gvt/mmio.c index 43f65848ecd6..ed4df2f6d60b 100644 --- a/drivers/gpu/drm/i915/gvt/mmio.c +++ b/drivers/gpu/drm/i915/gvt/mmio.c @@ -57,7 +57,7 @@ int intel_vgpu_gpa_to_mmio_offset(struct intel_vgpu *vgpu, u64 gpa) (reg >= gvt->device_info.gtt_start_offset \ && reg < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt)) -static void failsafe_emulate_mmio_rw(struct intel_vgpu *vgpu, uint64_t pa, +static void failsafe_emulate_mmio_rw(struct intel_vgpu *vgpu, u64 pa, void *p_data, unsigned int bytes, bool read) { struct intel_gvt *gvt = NULL; @@ -99,7 +99,7 @@ static void failsafe_emulate_mmio_rw(struct intel_vgpu *vgpu, uint64_t pa, * Returns: * Zero on success, negative error code if failed */ -int intel_vgpu_emulate_mmio_read(struct intel_vgpu *vgpu, uint64_t pa, +int intel_vgpu_emulate_mmio_read(struct intel_vgpu *vgpu, u64 pa, void *p_data, unsigned int bytes) { struct intel_gvt *gvt = vgpu->gvt; @@ -171,7 +171,7 @@ out: * Returns: * Zero on success, negative error code if failed */ -int intel_vgpu_emulate_mmio_write(struct intel_vgpu *vgpu, uint64_t pa, +int intel_vgpu_emulate_mmio_write(struct intel_vgpu *vgpu, u64 pa, void *p_data, unsigned int bytes) { struct intel_gvt *gvt = vgpu->gvt; diff --git a/drivers/gpu/drm/i915/gvt/mmio.h b/drivers/gpu/drm/i915/gvt/mmio.h index 1ffc69eba30e..5874f1cb4306 100644 --- a/drivers/gpu/drm/i915/gvt/mmio.h +++ b/drivers/gpu/drm/i915/gvt/mmio.h @@ -43,15 +43,16 @@ struct intel_vgpu; #define D_SKL (1 << 1) #define D_KBL (1 << 2) #define D_BXT (1 << 3) +#define D_CFL (1 << 4) -#define D_GEN9PLUS (D_SKL | D_KBL | D_BXT) -#define D_GEN8PLUS (D_BDW | D_SKL | D_KBL | D_BXT) +#define D_GEN9PLUS (D_SKL | D_KBL | D_BXT | D_CFL) +#define D_GEN8PLUS (D_BDW | D_SKL | D_KBL | D_BXT | D_CFL) -#define D_SKL_PLUS (D_SKL | D_KBL | D_BXT) -#define D_BDW_PLUS (D_BDW | D_SKL | D_KBL | D_BXT) +#define D_SKL_PLUS (D_SKL | D_KBL | D_BXT | D_CFL) +#define D_BDW_PLUS (D_BDW | D_SKL | D_KBL | D_BXT | D_CFL) #define D_PRE_SKL (D_BDW) -#define D_ALL (D_BDW | D_SKL | D_KBL | D_BXT) +#define D_ALL (D_BDW | D_SKL | D_KBL | D_BXT | D_CFL) typedef int (*gvt_mmio_func)(struct intel_vgpu *, unsigned int, void *, unsigned int); diff --git a/drivers/gpu/drm/i915/gvt/mmio_context.c b/drivers/gpu/drm/i915/gvt/mmio_context.c index d6e02c15ef97..7902fb162d09 100644 --- a/drivers/gpu/drm/i915/gvt/mmio_context.c +++ b/drivers/gpu/drm/i915/gvt/mmio_context.c @@ -132,6 +132,7 @@ static struct engine_mmio gen9_engine_mmio_list[] __cacheline_aligned = { {RCS, GEN9_GAMT_ECO_REG_RW_IA, 0x0, false}, /* 0x4ab0 */ {RCS, GEN9_CSFE_CHICKEN1_RCS, 0xffff, false}, /* 0x20d4 */ + {RCS, _MMIO(0x20D8), 0xffff, true}, /* 0x20d8 */ {RCS, GEN8_GARBCNTL, 0x0, false}, /* 0xb004 */ {RCS, GEN7_FF_THREAD_MODE, 0x0, false}, /* 0x20a0 */ @@ -353,8 +354,7 @@ static void handle_tlb_pending_event(struct intel_vgpu *vgpu, int ring_id) */ fw = intel_uncore_forcewake_for_reg(dev_priv, reg, FW_REG_READ | FW_REG_WRITE); - if (ring_id == RCS && (IS_SKYLAKE(dev_priv) || - IS_KABYLAKE(dev_priv) || IS_BROXTON(dev_priv))) + if (ring_id == RCS && (INTEL_GEN(dev_priv) >= 9)) fw |= FORCEWAKE_RENDER; intel_uncore_forcewake_get(dev_priv, fw); @@ -391,7 +391,8 @@ static void switch_mocs(struct intel_vgpu *pre, struct intel_vgpu *next, if (WARN_ON(ring_id >= ARRAY_SIZE(regs))) return; - if ((IS_KABYLAKE(dev_priv) || IS_BROXTON(dev_priv)) && ring_id == RCS) + if ((IS_KABYLAKE(dev_priv) || IS_BROXTON(dev_priv) + || IS_COFFEELAKE(dev_priv)) && ring_id == RCS) return; if (!pre && !gen9_render_mocs.initialized) @@ -457,9 +458,7 @@ static void switch_mmio(struct intel_vgpu *pre, u32 old_v, new_v; dev_priv = pre ? pre->gvt->dev_priv : next->gvt->dev_priv; - if (IS_SKYLAKE(dev_priv) - || IS_KABYLAKE(dev_priv) - || IS_BROXTON(dev_priv)) + if (INTEL_GEN(dev_priv) >= 9) switch_mocs(pre, next, ring_id); for (mmio = dev_priv->gvt->engine_mmio_list.mmio; @@ -471,8 +470,8 @@ static void switch_mmio(struct intel_vgpu *pre, * state image on kabylake, it's initialized by lri command and * save or restore with context together. */ - if ((IS_KABYLAKE(dev_priv) || IS_BROXTON(dev_priv)) - && mmio->in_context) + if ((IS_KABYLAKE(dev_priv) || IS_BROXTON(dev_priv) + || IS_COFFEELAKE(dev_priv)) && mmio->in_context) continue; // save @@ -565,9 +564,7 @@ void intel_gvt_init_engine_mmio_context(struct intel_gvt *gvt) { struct engine_mmio *mmio; - if (IS_SKYLAKE(gvt->dev_priv) || - IS_KABYLAKE(gvt->dev_priv) || - IS_BROXTON(gvt->dev_priv)) + if (INTEL_GEN(gvt->dev_priv) >= 9) gvt->engine_mmio_list.mmio = gen9_engine_mmio_list; else gvt->engine_mmio_list.mmio = gen8_engine_mmio_list; diff --git a/drivers/gpu/drm/i915/gvt/mpt.h b/drivers/gpu/drm/i915/gvt/mpt.h index 67f19992b226..0f9440128123 100644 --- a/drivers/gpu/drm/i915/gvt/mpt.h +++ b/drivers/gpu/drm/i915/gvt/mpt.h @@ -50,11 +50,10 @@ * Zero on success, negative error code if failed */ static inline int intel_gvt_hypervisor_host_init(struct device *dev, - void *gvt, const void *ops) + void *gvt, const void *ops) { - /* optional to provide */ if (!intel_gvt_host.mpt->host_init) - return 0; + return -ENODEV; return intel_gvt_host.mpt->host_init(dev, gvt, ops); } @@ -62,14 +61,13 @@ static inline int intel_gvt_hypervisor_host_init(struct device *dev, /** * intel_gvt_hypervisor_host_exit - exit GVT-g host side */ -static inline void intel_gvt_hypervisor_host_exit(struct device *dev, - void *gvt) +static inline void intel_gvt_hypervisor_host_exit(struct device *dev) { /* optional to provide */ if (!intel_gvt_host.mpt->host_exit) return; - intel_gvt_host.mpt->host_exit(dev, gvt); + intel_gvt_host.mpt->host_exit(dev); } /** @@ -101,7 +99,7 @@ static inline void intel_gvt_hypervisor_detach_vgpu(struct intel_vgpu *vgpu) if (!intel_gvt_host.mpt->detach_vgpu) return; - intel_gvt_host.mpt->detach_vgpu(vgpu->handle); + intel_gvt_host.mpt->detach_vgpu(vgpu); } #define MSI_CAP_CONTROL(offset) (offset + 2) @@ -316,6 +314,23 @@ static inline int intel_gvt_hypervisor_set_opregion(struct intel_vgpu *vgpu) } /** + * intel_gvt_hypervisor_set_edid - Set EDID region for guest + * @vgpu: a vGPU + * @port_num: display port number + * + * Returns: + * Zero on success, negative error code if failed. + */ +static inline int intel_gvt_hypervisor_set_edid(struct intel_vgpu *vgpu, + int port_num) +{ + if (!intel_gvt_host.mpt->set_edid) + return 0; + + return intel_gvt_host.mpt->set_edid(vgpu, port_num); +} + +/** * intel_gvt_hypervisor_get_vfio_device - increase vfio device ref count * @vgpu: a vGPU * @@ -362,4 +377,7 @@ static inline bool intel_gvt_hypervisor_is_valid_gfn( return intel_gvt_host.mpt->is_valid_gfn(vgpu->handle, gfn); } +int intel_gvt_register_hypervisor(struct intel_gvt_mpt *); +void intel_gvt_unregister_hypervisor(void); + #endif /* _GVT_MPT_H_ */ diff --git a/drivers/gpu/drm/i915/gvt/sched_policy.c b/drivers/gpu/drm/i915/gvt/sched_policy.c index c32e7d5e8629..1c763a27a412 100644 --- a/drivers/gpu/drm/i915/gvt/sched_policy.c +++ b/drivers/gpu/drm/i915/gvt/sched_policy.c @@ -94,7 +94,7 @@ static void gvt_balance_timeslice(struct gvt_sched_data *sched_data) { struct vgpu_sched_data *vgpu_data; struct list_head *pos; - static uint64_t stage_check; + static u64 stage_check; int stage = stage_check++ % GVT_TS_BALANCE_STAGE_NUM; /* The timeslice accumulation reset at stage 0, which is @@ -474,6 +474,6 @@ void intel_vgpu_stop_schedule(struct intel_vgpu *vgpu) } } spin_unlock_bh(&scheduler->mmio_context_lock); - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put_unchecked(dev_priv); mutex_unlock(&vgpu->gvt->sched_lock); } diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c index ea34003d6dd2..05b953793316 100644 --- a/drivers/gpu/drm/i915/gvt/scheduler.c +++ b/drivers/gpu/drm/i915/gvt/scheduler.c @@ -299,7 +299,8 @@ static int copy_workload_to_ring_buffer(struct intel_vgpu_workload *workload) void *shadow_ring_buffer_va; u32 *cs; - if ((IS_KABYLAKE(req->i915) || IS_BROXTON(req->i915)) + if ((IS_KABYLAKE(req->i915) || IS_BROXTON(req->i915) + || IS_COFFEELAKE(req->i915)) && is_inhibit_context(req->hw_context)) intel_vgpu_restore_inhibit_context(vgpu, req); @@ -332,6 +333,58 @@ static void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) i915_gem_object_unpin_map(wa_ctx->indirect_ctx.obj); i915_gem_object_put(wa_ctx->indirect_ctx.obj); + + wa_ctx->indirect_ctx.obj = NULL; + wa_ctx->indirect_ctx.shadow_va = NULL; +} + +static int set_context_ppgtt_from_shadow(struct intel_vgpu_workload *workload, + struct i915_gem_context *ctx) +{ + struct intel_vgpu_mm *mm = workload->shadow_mm; + struct i915_hw_ppgtt *ppgtt = ctx->ppgtt; + int i = 0; + + if (mm->type != INTEL_GVT_MM_PPGTT || !mm->ppgtt_mm.shadowed) + return -EINVAL; + + if (mm->ppgtt_mm.root_entry_type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY) { + px_dma(&ppgtt->pml4) = mm->ppgtt_mm.shadow_pdps[0]; + } else { + for (i = 0; i < GVT_RING_CTX_NR_PDPS; i++) { + px_dma(ppgtt->pdp.page_directory[i]) = + mm->ppgtt_mm.shadow_pdps[i]; + } + } + + return 0; +} + +static int +intel_gvt_workload_req_alloc(struct intel_vgpu_workload *workload) +{ + struct intel_vgpu *vgpu = workload->vgpu; + struct intel_vgpu_submission *s = &vgpu->submission; + struct i915_gem_context *shadow_ctx = s->shadow_ctx; + struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; + struct intel_engine_cs *engine = dev_priv->engine[workload->ring_id]; + struct i915_request *rq; + int ret = 0; + + lockdep_assert_held(&dev_priv->drm.struct_mutex); + + if (workload->req) + goto out; + + rq = i915_request_alloc(engine, shadow_ctx); + if (IS_ERR(rq)) { + gvt_vgpu_err("fail to allocate gem request\n"); + ret = PTR_ERR(rq); + goto out; + } + workload->req = i915_request_get(rq); +out: + return ret; } /** @@ -350,12 +403,11 @@ int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload) struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; struct intel_engine_cs *engine = dev_priv->engine[workload->ring_id]; struct intel_context *ce; - struct i915_request *rq; int ret; lockdep_assert_held(&dev_priv->drm.struct_mutex); - if (workload->req) + if (workload->shadow) return 0; /* pin shadow context by gvt even the shadow context will be pinned @@ -389,22 +441,8 @@ int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload) goto err_shadow; } - rq = i915_request_alloc(engine, shadow_ctx); - if (IS_ERR(rq)) { - gvt_vgpu_err("fail to allocate gem request\n"); - ret = PTR_ERR(rq); - goto err_shadow; - } - workload->req = i915_request_get(rq); - - ret = populate_shadow_context(workload); - if (ret) - goto err_req; - + workload->shadow = true; return 0; -err_req: - rq = fetch_and_zero(&workload->req); - i915_request_put(rq); err_shadow: release_shadow_wa_ctx(&workload->wa_ctx); err_unpin: @@ -634,6 +672,9 @@ static int dispatch_workload(struct intel_vgpu_workload *workload) { struct intel_vgpu *vgpu = workload->vgpu; struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; + struct intel_vgpu_submission *s = &vgpu->submission; + struct i915_gem_context *shadow_ctx = s->shadow_ctx; + struct i915_request *rq; int ring_id = workload->ring_id; int ret; @@ -643,15 +684,35 @@ static int dispatch_workload(struct intel_vgpu_workload *workload) mutex_lock(&vgpu->vgpu_lock); mutex_lock(&dev_priv->drm.struct_mutex); + ret = set_context_ppgtt_from_shadow(workload, shadow_ctx); + if (ret < 0) { + gvt_vgpu_err("workload shadow ppgtt isn't ready\n"); + goto err_req; + } + + ret = intel_gvt_workload_req_alloc(workload); + if (ret) + goto err_req; + ret = intel_gvt_scan_and_shadow_workload(workload); if (ret) goto out; - ret = prepare_workload(workload); + ret = populate_shadow_context(workload); + if (ret) { + release_shadow_wa_ctx(&workload->wa_ctx); + goto out; + } + ret = prepare_workload(workload); out: - if (ret) - workload->status = ret; + if (ret) { + /* We might still need to add request with + * clean ctx to retire it properly.. + */ + rq = fetch_and_zero(&workload->req); + i915_request_put(rq); + } if (!IS_ERR_OR_NULL(workload->req)) { gvt_dbg_sched("ring id %d submit workload to i915 %p\n", @@ -659,7 +720,9 @@ out: i915_request_add(workload->req); workload->dispatched = true; } - +err_req: + if (ret) + workload->status = ret; mutex_unlock(&dev_priv->drm.struct_mutex); mutex_unlock(&vgpu->vgpu_lock); return ret; @@ -687,7 +750,8 @@ static struct intel_vgpu_workload *pick_next_workload( goto out; } - if (list_empty(workload_q_head(scheduler->current_vgpu, ring_id))) + if (!scheduler->current_vgpu->active || + list_empty(workload_q_head(scheduler->current_vgpu, ring_id))) goto out; /* @@ -863,11 +927,6 @@ static void complete_current_workload(struct intel_gvt *gvt, int ring_id) list_del_init(&workload->list); - if (!workload->status) { - release_shadow_batch_buffer(workload); - release_shadow_wa_ctx(&workload->wa_ctx); - } - if (workload->status || (vgpu->resetting_eng & ENGINE_MASK(ring_id))) { /* if workload->status is not successful means HW GPU * has occurred GPU hang or something wrong with i915/GVT, @@ -911,9 +970,7 @@ static int workload_thread(void *priv) struct intel_vgpu_workload *workload = NULL; struct intel_vgpu *vgpu = NULL; int ret; - bool need_force_wake = IS_SKYLAKE(gvt->dev_priv) - || IS_KABYLAKE(gvt->dev_priv) - || IS_BROXTON(gvt->dev_priv); + bool need_force_wake = (INTEL_GEN(gvt->dev_priv) >= 9); DEFINE_WAIT_FUNC(wait, woken_wake_function); kfree(p); @@ -969,7 +1026,7 @@ complete: intel_uncore_forcewake_put(gvt->dev_priv, FORCEWAKE_ALL); - intel_runtime_pm_put(gvt->dev_priv); + intel_runtime_pm_put_unchecked(gvt->dev_priv); if (ret && (vgpu_is_vm_unhealthy(ret))) enter_failsafe_mode(vgpu, GVT_FAILSAFE_GUEST_ERR); } @@ -1051,6 +1108,21 @@ err: return ret; } +static void +i915_context_ppgtt_root_restore(struct intel_vgpu_submission *s) +{ + struct i915_hw_ppgtt *i915_ppgtt = s->shadow_ctx->ppgtt; + int i; + + if (i915_vm_is_48bit(&i915_ppgtt->vm)) + px_dma(&i915_ppgtt->pml4) = s->i915_context_pml4; + else { + for (i = 0; i < GEN8_3LVL_PDPES; i++) + px_dma(i915_ppgtt->pdp.page_directory[i]) = + s->i915_context_pdps[i]; + } +} + /** * intel_vgpu_clean_submission - free submission-related resource for vGPU * @vgpu: a vGPU @@ -1063,6 +1135,7 @@ void intel_vgpu_clean_submission(struct intel_vgpu *vgpu) struct intel_vgpu_submission *s = &vgpu->submission; intel_vgpu_select_submission_ops(vgpu, ALL_ENGINES, 0); + i915_context_ppgtt_root_restore(s); i915_gem_context_put(s->shadow_ctx); kmem_cache_destroy(s->workloads); } @@ -1088,6 +1161,21 @@ void intel_vgpu_reset_submission(struct intel_vgpu *vgpu, s->ops->reset(vgpu, engine_mask); } +static void +i915_context_ppgtt_root_save(struct intel_vgpu_submission *s) +{ + struct i915_hw_ppgtt *i915_ppgtt = s->shadow_ctx->ppgtt; + int i; + + if (i915_vm_is_48bit(&i915_ppgtt->vm)) + s->i915_context_pml4 = px_dma(&i915_ppgtt->pml4); + else { + for (i = 0; i < GEN8_3LVL_PDPES; i++) + s->i915_context_pdps[i] = + px_dma(i915_ppgtt->pdp.page_directory[i]); + } +} + /** * intel_vgpu_setup_submission - setup submission-related resource for vGPU * @vgpu: a vGPU @@ -1110,6 +1198,8 @@ int intel_vgpu_setup_submission(struct intel_vgpu *vgpu) if (IS_ERR(s->shadow_ctx)) return PTR_ERR(s->shadow_ctx); + i915_context_ppgtt_root_save(s); + bitmap_zero(s->shadow_ctx_desc_updated, I915_NUM_ENGINES); s->workloads = kmem_cache_create_usercopy("gvt-g_vgpu_workload", @@ -1202,6 +1292,9 @@ void intel_vgpu_destroy_workload(struct intel_vgpu_workload *workload) { struct intel_vgpu_submission *s = &workload->vgpu->submission; + release_shadow_batch_buffer(workload); + release_shadow_wa_ctx(&workload->wa_ctx); + if (workload->shadow_mm) intel_vgpu_mm_put(workload->shadow_mm); @@ -1390,11 +1483,12 @@ intel_vgpu_create_workload(struct intel_vgpu *vgpu, int ring_id, mutex_lock(&dev_priv->drm.struct_mutex); ret = intel_gvt_scan_and_shadow_workload(workload); mutex_unlock(&dev_priv->drm.struct_mutex); - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put_unchecked(dev_priv); } - if (ret && (vgpu_is_vm_unhealthy(ret))) { - enter_failsafe_mode(vgpu, GVT_FAILSAFE_GUEST_ERR); + if (ret) { + if (vgpu_is_vm_unhealthy(ret)) + enter_failsafe_mode(vgpu, GVT_FAILSAFE_GUEST_ERR); intel_vgpu_destroy_workload(workload); return ERR_PTR(ret); } diff --git a/drivers/gpu/drm/i915/gvt/scheduler.h b/drivers/gpu/drm/i915/gvt/scheduler.h index ca5529d0e48e..0635b2c4bed7 100644 --- a/drivers/gpu/drm/i915/gvt/scheduler.h +++ b/drivers/gpu/drm/i915/gvt/scheduler.h @@ -61,7 +61,7 @@ struct shadow_indirect_ctx { unsigned long guest_gma; unsigned long shadow_gma; void *shadow_va; - uint32_t size; + u32 size; }; #define PER_CTX_ADDR_MASK 0xfffff000 @@ -83,6 +83,7 @@ struct intel_vgpu_workload { struct i915_request *req; /* if this workload has been dispatched to i915? */ bool dispatched; + bool shadow; /* if workload has done shadow of guest request */ int status; struct intel_vgpu_mm *shadow_mm; diff --git a/drivers/gpu/drm/i915/gvt/trace.h b/drivers/gpu/drm/i915/gvt/trace.h index 1fd64202d74e..6d787750d279 100644 --- a/drivers/gpu/drm/i915/gvt/trace.h +++ b/drivers/gpu/drm/i915/gvt/trace.h @@ -228,7 +228,7 @@ TRACE_EVENT(oos_sync, TRACE_EVENT(gvt_command, TP_PROTO(u8 vgpu_id, u8 ring_id, u32 ip_gma, u32 *cmd_va, u32 cmd_len, u32 buf_type, u32 buf_addr_type, - void *workload, char *cmd_name), + void *workload, const char *cmd_name), TP_ARGS(vgpu_id, ring_id, ip_gma, cmd_va, cmd_len, buf_type, buf_addr_type, workload, cmd_name), diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c index c628be05fbfe..720e2b10adaa 100644 --- a/drivers/gpu/drm/i915/gvt/vgpu.c +++ b/drivers/gpu/drm/i915/gvt/vgpu.c @@ -148,10 +148,10 @@ int intel_gvt_init_vgpu_types(struct intel_gvt *gvt) gvt->types[i].avail_instance = min(low_avail / vgpu_types[i].low_mm, high_avail / vgpu_types[i].high_mm); - if (IS_GEN8(gvt->dev_priv)) + if (IS_GEN(gvt->dev_priv, 8)) sprintf(gvt->types[i].name, "GVTg_V4_%s", vgpu_types[i].name); - else if (IS_GEN9(gvt->dev_priv)) + else if (IS_GEN(gvt->dev_priv, 9)) sprintf(gvt->types[i].name, "GVTg_V5_%s", vgpu_types[i].name); @@ -428,6 +428,12 @@ static struct intel_vgpu *__intel_gvt_create_vgpu(struct intel_gvt *gvt, if (ret) goto out_clean_sched_policy; + /*TODO: add more platforms support */ + if (IS_SKYLAKE(gvt->dev_priv) || IS_KABYLAKE(gvt->dev_priv)) + ret = intel_gvt_hypervisor_set_edid(vgpu, PORT_D); + if (ret) + goto out_clean_sched_policy; + return vgpu; out_clean_sched_policy: diff --git a/drivers/gpu/drm/i915/i915_active.c b/drivers/gpu/drm/i915/i915_active.c new file mode 100644 index 000000000000..db7bb5bd5add --- /dev/null +++ b/drivers/gpu/drm/i915/i915_active.c @@ -0,0 +1,300 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright © 2019 Intel Corporation + */ + +#include "i915_drv.h" +#include "i915_active.h" + +#define BKL(ref) (&(ref)->i915->drm.struct_mutex) + +/* + * Active refs memory management + * + * To be more economical with memory, we reap all the i915_active trees as + * they idle (when we know the active requests are inactive) and allocate the + * nodes from a local slab cache to hopefully reduce the fragmentation. + */ +static struct i915_global_active { + struct kmem_cache *slab_cache; +} global; + +struct active_node { + struct i915_active_request base; + struct i915_active *ref; + struct rb_node node; + u64 timeline; +}; + +static void +__active_park(struct i915_active *ref) +{ + struct active_node *it, *n; + + rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) { + GEM_BUG_ON(i915_active_request_isset(&it->base)); + kmem_cache_free(global.slab_cache, it); + } + ref->tree = RB_ROOT; +} + +static void +__active_retire(struct i915_active *ref) +{ + GEM_BUG_ON(!ref->count); + if (--ref->count) + return; + + /* return the unused nodes to our slabcache */ + __active_park(ref); + + ref->retire(ref); +} + +static void +node_retire(struct i915_active_request *base, struct i915_request *rq) +{ + __active_retire(container_of(base, struct active_node, base)->ref); +} + +static void +last_retire(struct i915_active_request *base, struct i915_request *rq) +{ + __active_retire(container_of(base, struct i915_active, last)); +} + +static struct i915_active_request * +active_instance(struct i915_active *ref, u64 idx) +{ + struct active_node *node; + struct rb_node **p, *parent; + struct i915_request *old; + + /* + * We track the most recently used timeline to skip a rbtree search + * for the common case, under typical loads we never need the rbtree + * at all. We can reuse the last slot if it is empty, that is + * after the previous activity has been retired, or if it matches the + * current timeline. + * + * Note that we allow the timeline to be active simultaneously in + * the rbtree and the last cache. We do this to avoid having + * to search and replace the rbtree element for a new timeline, with + * the cost being that we must be aware that the ref may be retired + * twice for the same timeline (as the older rbtree element will be + * retired before the new request added to last). + */ + old = i915_active_request_raw(&ref->last, BKL(ref)); + if (!old || old->fence.context == idx) + goto out; + + /* Move the currently active fence into the rbtree */ + idx = old->fence.context; + + parent = NULL; + p = &ref->tree.rb_node; + while (*p) { + parent = *p; + + node = rb_entry(parent, struct active_node, node); + if (node->timeline == idx) + goto replace; + + if (node->timeline < idx) + p = &parent->rb_right; + else + p = &parent->rb_left; + } + + node = kmem_cache_alloc(global.slab_cache, GFP_KERNEL); + + /* kmalloc may retire the ref->last (thanks shrinker)! */ + if (unlikely(!i915_active_request_raw(&ref->last, BKL(ref)))) { + kmem_cache_free(global.slab_cache, node); + goto out; + } + + if (unlikely(!node)) + return ERR_PTR(-ENOMEM); + + i915_active_request_init(&node->base, NULL, node_retire); + node->ref = ref; + node->timeline = idx; + + rb_link_node(&node->node, parent, p); + rb_insert_color(&node->node, &ref->tree); + +replace: + /* + * Overwrite the previous active slot in the rbtree with last, + * leaving last zeroed. If the previous slot is still active, + * we must be careful as we now only expect to receive one retire + * callback not two, and so much undo the active counting for the + * overwritten slot. + */ + if (i915_active_request_isset(&node->base)) { + /* Retire ourselves from the old rq->active_list */ + __list_del_entry(&node->base.link); + ref->count--; + GEM_BUG_ON(!ref->count); + } + GEM_BUG_ON(list_empty(&ref->last.link)); + list_replace_init(&ref->last.link, &node->base.link); + node->base.request = fetch_and_zero(&ref->last.request); + +out: + return &ref->last; +} + +void i915_active_init(struct drm_i915_private *i915, + struct i915_active *ref, + void (*retire)(struct i915_active *ref)) +{ + ref->i915 = i915; + ref->retire = retire; + ref->tree = RB_ROOT; + i915_active_request_init(&ref->last, NULL, last_retire); + ref->count = 0; +} + +int i915_active_ref(struct i915_active *ref, + u64 timeline, + struct i915_request *rq) +{ + struct i915_active_request *active; + int err = 0; + + /* Prevent reaping in case we malloc/wait while building the tree */ + i915_active_acquire(ref); + + active = active_instance(ref, timeline); + if (IS_ERR(active)) { + err = PTR_ERR(active); + goto out; + } + + if (!i915_active_request_isset(active)) + ref->count++; + __i915_active_request_set(active, rq); + + GEM_BUG_ON(!ref->count); +out: + i915_active_release(ref); + return err; +} + +bool i915_active_acquire(struct i915_active *ref) +{ + lockdep_assert_held(BKL(ref)); + return !ref->count++; +} + +void i915_active_release(struct i915_active *ref) +{ + lockdep_assert_held(BKL(ref)); + __active_retire(ref); +} + +int i915_active_wait(struct i915_active *ref) +{ + struct active_node *it, *n; + int ret = 0; + + if (i915_active_acquire(ref)) + goto out_release; + + ret = i915_active_request_retire(&ref->last, BKL(ref)); + if (ret) + goto out_release; + + rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) { + ret = i915_active_request_retire(&it->base, BKL(ref)); + if (ret) + break; + } + +out_release: + i915_active_release(ref); + return ret; +} + +int i915_request_await_active_request(struct i915_request *rq, + struct i915_active_request *active) +{ + struct i915_request *barrier = + i915_active_request_raw(active, &rq->i915->drm.struct_mutex); + + return barrier ? i915_request_await_dma_fence(rq, &barrier->fence) : 0; +} + +int i915_request_await_active(struct i915_request *rq, struct i915_active *ref) +{ + struct active_node *it, *n; + int err = 0; + + /* await allocates and so we need to avoid hitting the shrinker */ + if (i915_active_acquire(ref)) + goto out; /* was idle */ + + err = i915_request_await_active_request(rq, &ref->last); + if (err) + goto out; + + rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) { + err = i915_request_await_active_request(rq, &it->base); + if (err) + goto out; + } + +out: + i915_active_release(ref); + return err; +} + +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) +void i915_active_fini(struct i915_active *ref) +{ + GEM_BUG_ON(i915_active_request_isset(&ref->last)); + GEM_BUG_ON(!RB_EMPTY_ROOT(&ref->tree)); + GEM_BUG_ON(ref->count); +} +#endif + +int i915_active_request_set(struct i915_active_request *active, + struct i915_request *rq) +{ + int err; + + /* Must maintain ordering wrt previous active requests */ + err = i915_request_await_active_request(rq, active); + if (err) + return err; + + __i915_active_request_set(active, rq); + return 0; +} + +void i915_active_retire_noop(struct i915_active_request *active, + struct i915_request *request) +{ + /* Space left intentionally blank */ +} + +#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) +#include "selftests/i915_active.c" +#endif + +int __init i915_global_active_init(void) +{ + global.slab_cache = KMEM_CACHE(active_node, SLAB_HWCACHE_ALIGN); + if (!global.slab_cache) + return -ENOMEM; + + return 0; +} + +void __exit i915_global_active_exit(void) +{ + kmem_cache_destroy(global.slab_cache); +} diff --git a/drivers/gpu/drm/i915/i915_active.h b/drivers/gpu/drm/i915/i915_active.h new file mode 100644 index 000000000000..12b5c1d287d1 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_active.h @@ -0,0 +1,425 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright © 2019 Intel Corporation + */ + +#ifndef _I915_ACTIVE_H_ +#define _I915_ACTIVE_H_ + +#include <linux/lockdep.h> + +#include "i915_active_types.h" +#include "i915_request.h" + +/* + * We treat requests as fences. This is not be to confused with our + * "fence registers" but pipeline synchronisation objects ala GL_ARB_sync. + * We use the fences to synchronize access from the CPU with activity on the + * GPU, for example, we should not rewrite an object's PTE whilst the GPU + * is reading them. We also track fences at a higher level to provide + * implicit synchronisation around GEM objects, e.g. set-domain will wait + * for outstanding GPU rendering before marking the object ready for CPU + * access, or a pageflip will wait until the GPU is complete before showing + * the frame on the scanout. + * + * In order to use a fence, the object must track the fence it needs to + * serialise with. For example, GEM objects want to track both read and + * write access so that we can perform concurrent read operations between + * the CPU and GPU engines, as well as waiting for all rendering to + * complete, or waiting for the last GPU user of a "fence register". The + * object then embeds a #i915_active_request to track the most recent (in + * retirement order) request relevant for the desired mode of access. + * The #i915_active_request is updated with i915_active_request_set() to + * track the most recent fence request, typically this is done as part of + * i915_vma_move_to_active(). + * + * When the #i915_active_request completes (is retired), it will + * signal its completion to the owner through a callback as well as mark + * itself as idle (i915_active_request.request == NULL). The owner + * can then perform any action, such as delayed freeing of an active + * resource including itself. + */ + +void i915_active_retire_noop(struct i915_active_request *active, + struct i915_request *request); + +/** + * i915_active_request_init - prepares the activity tracker for use + * @active - the active tracker + * @rq - initial request to track, can be NULL + * @func - a callback when then the tracker is retired (becomes idle), + * can be NULL + * + * i915_active_request_init() prepares the embedded @active struct for use as + * an activity tracker, that is for tracking the last known active request + * associated with it. When the last request becomes idle, when it is retired + * after completion, the optional callback @func is invoked. + */ +static inline void +i915_active_request_init(struct i915_active_request *active, + struct i915_request *rq, + i915_active_retire_fn retire) +{ + RCU_INIT_POINTER(active->request, rq); + INIT_LIST_HEAD(&active->link); + active->retire = retire ?: i915_active_retire_noop; +} + +#define INIT_ACTIVE_REQUEST(name) i915_active_request_init((name), NULL, NULL) + +/** + * i915_active_request_set - updates the tracker to watch the current request + * @active - the active tracker + * @request - the request to watch + * + * __i915_active_request_set() watches the given @request for completion. Whilst + * that @request is busy, the @active reports busy. When that @request is + * retired, the @active tracker is updated to report idle. + */ +static inline void +__i915_active_request_set(struct i915_active_request *active, + struct i915_request *request) +{ + list_move(&active->link, &request->active_list); + rcu_assign_pointer(active->request, request); +} + +int __must_check +i915_active_request_set(struct i915_active_request *active, + struct i915_request *rq); + +/** + * i915_active_request_set_retire_fn - updates the retirement callback + * @active - the active tracker + * @fn - the routine called when the request is retired + * @mutex - struct_mutex used to guard retirements + * + * i915_active_request_set_retire_fn() updates the function pointer that + * is called when the final request associated with the @active tracker + * is retired. + */ +static inline void +i915_active_request_set_retire_fn(struct i915_active_request *active, + i915_active_retire_fn fn, + struct mutex *mutex) +{ + lockdep_assert_held(mutex); + active->retire = fn ?: i915_active_retire_noop; +} + +static inline struct i915_request * +__i915_active_request_peek(const struct i915_active_request *active) +{ + /* + * Inside the error capture (running with the driver in an unknown + * state), we want to bend the rules slightly (a lot). + * + * Work is in progress to make it safer, in the meantime this keeps + * the known issue from spamming the logs. + */ + return rcu_dereference_protected(active->request, 1); +} + +/** + * i915_active_request_raw - return the active request + * @active - the active tracker + * + * i915_active_request_raw() returns the current request being tracked, or NULL. + * It does not obtain a reference on the request for the caller, so the caller + * must hold struct_mutex. + */ +static inline struct i915_request * +i915_active_request_raw(const struct i915_active_request *active, + struct mutex *mutex) +{ + return rcu_dereference_protected(active->request, + lockdep_is_held(mutex)); +} + +/** + * i915_active_request_peek - report the active request being monitored + * @active - the active tracker + * + * i915_active_request_peek() returns the current request being tracked if + * still active, or NULL. It does not obtain a reference on the request + * for the caller, so the caller must hold struct_mutex. + */ +static inline struct i915_request * +i915_active_request_peek(const struct i915_active_request *active, + struct mutex *mutex) +{ + struct i915_request *request; + + request = i915_active_request_raw(active, mutex); + if (!request || i915_request_completed(request)) + return NULL; + + return request; +} + +/** + * i915_active_request_get - return a reference to the active request + * @active - the active tracker + * + * i915_active_request_get() returns a reference to the active request, or NULL + * if the active tracker is idle. The caller must hold struct_mutex. + */ +static inline struct i915_request * +i915_active_request_get(const struct i915_active_request *active, + struct mutex *mutex) +{ + return i915_request_get(i915_active_request_peek(active, mutex)); +} + +/** + * __i915_active_request_get_rcu - return a reference to the active request + * @active - the active tracker + * + * __i915_active_request_get() returns a reference to the active request, + * or NULL if the active tracker is idle. The caller must hold the RCU read + * lock, but the returned pointer is safe to use outside of RCU. + */ +static inline struct i915_request * +__i915_active_request_get_rcu(const struct i915_active_request *active) +{ + /* + * Performing a lockless retrieval of the active request is super + * tricky. SLAB_TYPESAFE_BY_RCU merely guarantees that the backing + * slab of request objects will not be freed whilst we hold the + * RCU read lock. It does not guarantee that the request itself + * will not be freed and then *reused*. Viz, + * + * Thread A Thread B + * + * rq = active.request + * retire(rq) -> free(rq); + * (rq is now first on the slab freelist) + * active.request = NULL + * + * rq = new submission on a new object + * ref(rq) + * + * To prevent the request from being reused whilst the caller + * uses it, we take a reference like normal. Whilst acquiring + * the reference we check that it is not in a destroyed state + * (refcnt == 0). That prevents the request being reallocated + * whilst the caller holds on to it. To check that the request + * was not reallocated as we acquired the reference we have to + * check that our request remains the active request across + * the lookup, in the same manner as a seqlock. The visibility + * of the pointer versus the reference counting is controlled + * by using RCU barriers (rcu_dereference and rcu_assign_pointer). + * + * In the middle of all that, we inspect whether the request is + * complete. Retiring is lazy so the request may be completed long + * before the active tracker is updated. Querying whether the + * request is complete is far cheaper (as it involves no locked + * instructions setting cachelines to exclusive) than acquiring + * the reference, so we do it first. The RCU read lock ensures the + * pointer dereference is valid, but does not ensure that the + * seqno nor HWS is the right one! However, if the request was + * reallocated, that means the active tracker's request was complete. + * If the new request is also complete, then both are and we can + * just report the active tracker is idle. If the new request is + * incomplete, then we acquire a reference on it and check that + * it remained the active request. + * + * It is then imperative that we do not zero the request on + * reallocation, so that we can chase the dangling pointers! + * See i915_request_alloc(). + */ + do { + struct i915_request *request; + + request = rcu_dereference(active->request); + if (!request || i915_request_completed(request)) + return NULL; + + /* + * An especially silly compiler could decide to recompute the + * result of i915_request_completed, more specifically + * re-emit the load for request->fence.seqno. A race would catch + * a later seqno value, which could flip the result from true to + * false. Which means part of the instructions below might not + * be executed, while later on instructions are executed. Due to + * barriers within the refcounting the inconsistency can't reach + * past the call to i915_request_get_rcu, but not executing + * that while still executing i915_request_put() creates + * havoc enough. Prevent this with a compiler barrier. + */ + barrier(); + + request = i915_request_get_rcu(request); + + /* + * What stops the following rcu_access_pointer() from occurring + * before the above i915_request_get_rcu()? If we were + * to read the value before pausing to get the reference to + * the request, we may not notice a change in the active + * tracker. + * + * The rcu_access_pointer() is a mere compiler barrier, which + * means both the CPU and compiler are free to perform the + * memory read without constraint. The compiler only has to + * ensure that any operations after the rcu_access_pointer() + * occur afterwards in program order. This means the read may + * be performed earlier by an out-of-order CPU, or adventurous + * compiler. + * + * The atomic operation at the heart of + * i915_request_get_rcu(), see dma_fence_get_rcu(), is + * atomic_inc_not_zero() which is only a full memory barrier + * when successful. That is, if i915_request_get_rcu() + * returns the request (and so with the reference counted + * incremented) then the following read for rcu_access_pointer() + * must occur after the atomic operation and so confirm + * that this request is the one currently being tracked. + * + * The corresponding write barrier is part of + * rcu_assign_pointer(). + */ + if (!request || request == rcu_access_pointer(active->request)) + return rcu_pointer_handoff(request); + + i915_request_put(request); + } while (1); +} + +/** + * i915_active_request_get_unlocked - return a reference to the active request + * @active - the active tracker + * + * i915_active_request_get_unlocked() returns a reference to the active request, + * or NULL if the active tracker is idle. The reference is obtained under RCU, + * so no locking is required by the caller. + * + * The reference should be freed with i915_request_put(). + */ +static inline struct i915_request * +i915_active_request_get_unlocked(const struct i915_active_request *active) +{ + struct i915_request *request; + + rcu_read_lock(); + request = __i915_active_request_get_rcu(active); + rcu_read_unlock(); + + return request; +} + +/** + * i915_active_request_isset - report whether the active tracker is assigned + * @active - the active tracker + * + * i915_active_request_isset() returns true if the active tracker is currently + * assigned to a request. Due to the lazy retiring, that request may be idle + * and this may report stale information. + */ +static inline bool +i915_active_request_isset(const struct i915_active_request *active) +{ + return rcu_access_pointer(active->request); +} + +/** + * i915_active_request_retire - waits until the request is retired + * @active - the active request on which to wait + * + * i915_active_request_retire() waits until the request is completed, + * and then ensures that at least the retirement handler for this + * @active tracker is called before returning. If the @active + * tracker is idle, the function returns immediately. + */ +static inline int __must_check +i915_active_request_retire(struct i915_active_request *active, + struct mutex *mutex) +{ + struct i915_request *request; + long ret; + + request = i915_active_request_raw(active, mutex); + if (!request) + return 0; + + ret = i915_request_wait(request, + I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED, + MAX_SCHEDULE_TIMEOUT); + if (ret < 0) + return ret; + + list_del_init(&active->link); + RCU_INIT_POINTER(active->request, NULL); + + active->retire(active, request); + + return 0; +} + +/* + * GPU activity tracking + * + * Each set of commands submitted to the GPU compromises a single request that + * signals a fence upon completion. struct i915_request combines the + * command submission, scheduling and fence signaling roles. If we want to see + * if a particular task is complete, we need to grab the fence (struct + * i915_request) for that task and check or wait for it to be signaled. More + * often though we want to track the status of a bunch of tasks, for example + * to wait for the GPU to finish accessing some memory across a variety of + * different command pipelines from different clients. We could choose to + * track every single request associated with the task, but knowing that + * each request belongs to an ordered timeline (later requests within a + * timeline must wait for earlier requests), we need only track the + * latest request in each timeline to determine the overall status of the + * task. + * + * struct i915_active provides this tracking across timelines. It builds a + * composite shared-fence, and is updated as new work is submitted to the task, + * forming a snapshot of the current status. It should be embedded into the + * different resources that need to track their associated GPU activity to + * provide a callback when that GPU activity has ceased, or otherwise to + * provide a serialisation point either for request submission or for CPU + * synchronisation. + */ + +void i915_active_init(struct drm_i915_private *i915, + struct i915_active *ref, + void (*retire)(struct i915_active *ref)); + +int i915_active_ref(struct i915_active *ref, + u64 timeline, + struct i915_request *rq); + +int i915_active_wait(struct i915_active *ref); + +int i915_request_await_active(struct i915_request *rq, + struct i915_active *ref); +int i915_request_await_active_request(struct i915_request *rq, + struct i915_active_request *active); + +bool i915_active_acquire(struct i915_active *ref); + +static inline void i915_active_cancel(struct i915_active *ref) +{ + GEM_BUG_ON(ref->count != 1); + ref->count = 0; +} + +void i915_active_release(struct i915_active *ref); + +static inline bool +i915_active_is_idle(const struct i915_active *ref) +{ + return !ref->count; +} + +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) +void i915_active_fini(struct i915_active *ref); +#else +static inline void i915_active_fini(struct i915_active *ref) { } +#endif + +int i915_global_active_init(void); +void i915_global_active_exit(void); + +#endif /* _I915_ACTIVE_H_ */ diff --git a/drivers/gpu/drm/i915/i915_active_types.h b/drivers/gpu/drm/i915/i915_active_types.h new file mode 100644 index 000000000000..b679253b53a5 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_active_types.h @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright © 2019 Intel Corporation + */ + +#ifndef _I915_ACTIVE_TYPES_H_ +#define _I915_ACTIVE_TYPES_H_ + +#include <linux/rbtree.h> +#include <linux/rcupdate.h> + +struct drm_i915_private; +struct i915_active_request; +struct i915_request; + +typedef void (*i915_active_retire_fn)(struct i915_active_request *, + struct i915_request *); + +struct i915_active_request { + struct i915_request __rcu *request; + struct list_head link; + i915_active_retire_fn retire; +}; + +struct i915_active { + struct drm_i915_private *i915; + + struct rb_root tree; + struct i915_active_request last; + unsigned int count; + + void (*retire)(struct i915_active *ref); +}; + +#endif /* _I915_ACTIVE_TYPES_H_ */ diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c index 95478db9998b..33e8eed64423 100644 --- a/drivers/gpu/drm/i915/i915_cmd_parser.c +++ b/drivers/gpu/drm/i915/i915_cmd_parser.c @@ -865,7 +865,7 @@ void intel_engine_init_cmd_parser(struct intel_engine_cs *engine) int cmd_table_count; int ret; - if (!IS_GEN7(engine->i915)) + if (!IS_GEN(engine->i915, 7)) return; switch (engine->id) { diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 4f3ac0a12889..f6f6e5b78e97 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -26,12 +26,15 @@ * */ -#include <linux/debugfs.h> #include <linux/sort.h> #include <linux/sched/mm.h> +#include <drm/drm_debugfs.h> +#include <drm/drm_fourcc.h> #include "intel_drv.h" #include "intel_guc_submission.h" +#include "i915_reset.h" + static inline struct drm_i915_private *node_to_i915(struct drm_info_node *node) { return to_i915(node->minor->dev); @@ -48,7 +51,7 @@ static int i915_capabilities(struct seq_file *m, void *data) seq_printf(m, "pch: %d\n", INTEL_PCH_TYPE(dev_priv)); intel_device_info_dump_flags(info, &p); - intel_device_info_dump_runtime(info, &p); + intel_device_info_dump_runtime(RUNTIME_INFO(dev_priv), &p); intel_driver_caps_print(&dev_priv->caps, &p); kernel_param_lock(THIS_MODULE); @@ -157,14 +160,14 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) obj->mm.madv == I915_MADV_DONTNEED ? " purgeable" : ""); if (obj->base.name) seq_printf(m, " (name: %d)", obj->base.name); - list_for_each_entry(vma, &obj->vma_list, obj_link) { + list_for_each_entry(vma, &obj->vma.list, obj_link) { if (i915_vma_is_pinned(vma)) pin_count++; } seq_printf(m, " (pinned x %d)", pin_count); if (obj->pin_global) seq_printf(m, " (global)"); - list_for_each_entry(vma, &obj->vma_list, obj_link) { + list_for_each_entry(vma, &obj->vma.list, obj_link) { if (!drm_mm_node_allocated(&vma->node)) continue; @@ -204,7 +207,7 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) if (vma->fence) seq_printf(m, " , fence: %d%s", vma->fence->id, - i915_gem_active_isset(&vma->last_fence) ? "*" : ""); + i915_active_request_isset(&vma->last_fence) ? "*" : ""); seq_puts(m, ")"); } if (obj->stolen) @@ -297,11 +300,12 @@ out: } struct file_stats { - struct drm_i915_file_private *file_priv; + struct i915_address_space *vm; unsigned long count; u64 total, unbound; u64 global, shared; u64 active, inactive; + u64 closed; }; static int per_file_stats(int id, void *ptr, void *data) @@ -319,16 +323,14 @@ static int per_file_stats(int id, void *ptr, void *data) if (obj->base.name || obj->base.dma_buf) stats->shared += obj->base.size; - list_for_each_entry(vma, &obj->vma_list, obj_link) { + list_for_each_entry(vma, &obj->vma.list, obj_link) { if (!drm_mm_node_allocated(&vma->node)) continue; if (i915_vma_is_ggtt(vma)) { stats->global += vma->node.size; } else { - struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vma->vm); - - if (ppgtt->vm.file != stats->file_priv) + if (vma->vm != stats->vm) continue; } @@ -336,6 +338,9 @@ static int per_file_stats(int id, void *ptr, void *data) stats->active += vma->node.size; else stats->inactive += vma->node.size; + + if (i915_vma_is_closed(vma)) + stats->closed += vma->node.size; } return 0; @@ -343,7 +348,7 @@ static int per_file_stats(int id, void *ptr, void *data) #define print_file_stats(m, name, stats) do { \ if (stats.count) \ - seq_printf(m, "%s: %lu objects, %llu bytes (%llu active, %llu inactive, %llu global, %llu shared, %llu unbound)\n", \ + seq_printf(m, "%s: %lu objects, %llu bytes (%llu active, %llu inactive, %llu global, %llu shared, %llu unbound, %llu closed)\n", \ name, \ stats.count, \ stats.total, \ @@ -351,20 +356,19 @@ static int per_file_stats(int id, void *ptr, void *data) stats.inactive, \ stats.global, \ stats.shared, \ - stats.unbound); \ + stats.unbound, \ + stats.closed); \ } while (0) static void print_batch_pool_stats(struct seq_file *m, struct drm_i915_private *dev_priv) { struct drm_i915_gem_object *obj; - struct file_stats stats; struct intel_engine_cs *engine; + struct file_stats stats = {}; enum intel_engine_id id; int j; - memset(&stats, 0, sizeof(stats)); - for_each_engine(engine, dev_priv, id) { for (j = 0; j < ARRAY_SIZE(engine->batch_pool.cache_list); j++) { list_for_each_entry(obj, @@ -377,44 +381,47 @@ static void print_batch_pool_stats(struct seq_file *m, print_file_stats(m, "[k]batch pool", stats); } -static int per_file_ctx_stats(int idx, void *ptr, void *data) +static void print_context_stats(struct seq_file *m, + struct drm_i915_private *i915) { - struct i915_gem_context *ctx = ptr; - struct intel_engine_cs *engine; - enum intel_engine_id id; + struct file_stats kstats = {}; + struct i915_gem_context *ctx; - for_each_engine(engine, ctx->i915, id) { - struct intel_context *ce = to_intel_context(ctx, engine); + list_for_each_entry(ctx, &i915->contexts.list, link) { + struct intel_engine_cs *engine; + enum intel_engine_id id; - if (ce->state) - per_file_stats(0, ce->state->obj, data); - if (ce->ring) - per_file_stats(0, ce->ring->vma->obj, data); - } + for_each_engine(engine, i915, id) { + struct intel_context *ce = to_intel_context(ctx, engine); - return 0; -} + if (ce->state) + per_file_stats(0, ce->state->obj, &kstats); + if (ce->ring) + per_file_stats(0, ce->ring->vma->obj, &kstats); + } -static void print_context_stats(struct seq_file *m, - struct drm_i915_private *dev_priv) -{ - struct drm_device *dev = &dev_priv->drm; - struct file_stats stats; - struct drm_file *file; + if (!IS_ERR_OR_NULL(ctx->file_priv)) { + struct file_stats stats = { .vm = &ctx->ppgtt->vm, }; + struct drm_file *file = ctx->file_priv->file; + struct task_struct *task; + char name[80]; - memset(&stats, 0, sizeof(stats)); + spin_lock(&file->table_lock); + idr_for_each(&file->object_idr, per_file_stats, &stats); + spin_unlock(&file->table_lock); - mutex_lock(&dev->struct_mutex); - if (dev_priv->kernel_context) - per_file_ctx_stats(0, dev_priv->kernel_context, &stats); + rcu_read_lock(); + task = pid_task(ctx->pid ?: file->pid, PIDTYPE_PID); + snprintf(name, sizeof(name), "%s/%d", + task ? task->comm : "<unknown>", + ctx->user_handle); + rcu_read_unlock(); - list_for_each_entry(file, &dev->filelist, lhead) { - struct drm_i915_file_private *fpriv = file->driver_priv; - idr_for_each(&fpriv->context_idr, per_file_ctx_stats, &stats); + print_file_stats(m, name, stats); + } } - mutex_unlock(&dev->struct_mutex); - print_file_stats(m, "[k]contexts", stats); + print_file_stats(m, "[k]contexts", kstats); } static int i915_gem_object_info(struct seq_file *m, void *data) @@ -426,14 +433,9 @@ static int i915_gem_object_info(struct seq_file *m, void *data) u64 size, mapped_size, purgeable_size, dpy_size, huge_size; struct drm_i915_gem_object *obj; unsigned int page_sizes = 0; - struct drm_file *file; char buf[80]; int ret; - ret = mutex_lock_interruptible(&dev->struct_mutex); - if (ret) - return ret; - seq_printf(m, "%u objects, %llu bytes\n", dev_priv->mm.object_count, dev_priv->mm.object_memory); @@ -514,43 +516,14 @@ static int i915_gem_object_info(struct seq_file *m, void *data) buf, sizeof(buf))); seq_putc(m, '\n'); - print_batch_pool_stats(m, dev_priv); - mutex_unlock(&dev->struct_mutex); - mutex_lock(&dev->filelist_mutex); - print_context_stats(m, dev_priv); - list_for_each_entry_reverse(file, &dev->filelist, lhead) { - struct file_stats stats; - struct drm_i915_file_private *file_priv = file->driver_priv; - struct i915_request *request; - struct task_struct *task; - - mutex_lock(&dev->struct_mutex); - - memset(&stats, 0, sizeof(stats)); - stats.file_priv = file->driver_priv; - spin_lock(&file->table_lock); - idr_for_each(&file->object_idr, per_file_stats, &stats); - spin_unlock(&file->table_lock); - /* - * Although we have a valid reference on file->pid, that does - * not guarantee that the task_struct who called get_pid() is - * still alive (e.g. get_pid(current) => fork() => exit()). - * Therefore, we need to protect this ->comm access using RCU. - */ - request = list_first_entry_or_null(&file_priv->mm.request_list, - struct i915_request, - client_link); - rcu_read_lock(); - task = pid_task(request && request->gem_context->pid ? - request->gem_context->pid : file->pid, - PIDTYPE_PID); - print_file_stats(m, task ? task->comm : "<unknown>", stats); - rcu_read_unlock(); + ret = mutex_lock_interruptible(&dev->struct_mutex); + if (ret) + return ret; - mutex_unlock(&dev->struct_mutex); - } - mutex_unlock(&dev->filelist_mutex); + print_batch_pool_stats(m, dev_priv); + print_context_stats(m, dev_priv); + mutex_unlock(&dev->struct_mutex); return 0; } @@ -656,10 +629,12 @@ static void gen8_display_interrupt_info(struct seq_file *m) for_each_pipe(dev_priv, pipe) { enum intel_display_power_domain power_domain; + intel_wakeref_t wakeref; power_domain = POWER_DOMAIN_PIPE(pipe); - if (!intel_display_power_get_if_enabled(dev_priv, - power_domain)) { + wakeref = intel_display_power_get_if_enabled(dev_priv, + power_domain); + if (!wakeref) { seq_printf(m, "Pipe %c power disabled\n", pipe_name(pipe)); continue; @@ -674,7 +649,7 @@ static void gen8_display_interrupt_info(struct seq_file *m) pipe_name(pipe), I915_READ(GEN8_DE_PIPE_IER(pipe))); - intel_display_power_put(dev_priv, power_domain); + intel_display_power_put(dev_priv, power_domain, wakeref); } seq_printf(m, "Display Engine port interrupt mask:\t%08x\n", @@ -704,11 +679,14 @@ static int i915_interrupt_info(struct seq_file *m, void *data) struct drm_i915_private *dev_priv = node_to_i915(m->private); struct intel_engine_cs *engine; enum intel_engine_id id; + intel_wakeref_t wakeref; int i, pipe; - intel_runtime_pm_get(dev_priv); + wakeref = intel_runtime_pm_get(dev_priv); if (IS_CHERRYVIEW(dev_priv)) { + intel_wakeref_t pref; + seq_printf(m, "Master Interrupt Control:\t%08x\n", I915_READ(GEN8_MASTER_IRQ)); @@ -724,8 +702,9 @@ static int i915_interrupt_info(struct seq_file *m, void *data) enum intel_display_power_domain power_domain; power_domain = POWER_DOMAIN_PIPE(pipe); - if (!intel_display_power_get_if_enabled(dev_priv, - power_domain)) { + pref = intel_display_power_get_if_enabled(dev_priv, + power_domain); + if (!pref) { seq_printf(m, "Pipe %c power disabled\n", pipe_name(pipe)); continue; @@ -735,17 +714,17 @@ static int i915_interrupt_info(struct seq_file *m, void *data) pipe_name(pipe), I915_READ(PIPESTAT(pipe))); - intel_display_power_put(dev_priv, power_domain); + intel_display_power_put(dev_priv, power_domain, pref); } - intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); + pref = intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); seq_printf(m, "Port hotplug:\t%08x\n", I915_READ(PORT_HOTPLUG_EN)); seq_printf(m, "DPFLIPSTAT:\t%08x\n", I915_READ(VLV_DPFLIPSTAT)); seq_printf(m, "DPINVGTT:\t%08x\n", I915_READ(DPINVGTT)); - intel_display_power_put(dev_priv, POWER_DOMAIN_INIT); + intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, pref); for (i = 0; i < 4; i++) { seq_printf(m, "GT Interrupt IMR %d:\t%08x\n", @@ -808,10 +787,12 @@ static int i915_interrupt_info(struct seq_file *m, void *data) I915_READ(VLV_IMR)); for_each_pipe(dev_priv, pipe) { enum intel_display_power_domain power_domain; + intel_wakeref_t pref; power_domain = POWER_DOMAIN_PIPE(pipe); - if (!intel_display_power_get_if_enabled(dev_priv, - power_domain)) { + pref = intel_display_power_get_if_enabled(dev_priv, + power_domain); + if (!pref) { seq_printf(m, "Pipe %c power disabled\n", pipe_name(pipe)); continue; @@ -820,7 +801,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data) seq_printf(m, "Pipe %c stat:\t%08x\n", pipe_name(pipe), I915_READ(PIPESTAT(pipe))); - intel_display_power_put(dev_priv, power_domain); + intel_display_power_put(dev_priv, power_domain, pref); } seq_printf(m, "Master IER:\t%08x\n", @@ -907,7 +888,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data) } } - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, wakeref); return 0; } @@ -943,30 +924,30 @@ static int i915_gem_fence_regs_info(struct seq_file *m, void *data) static ssize_t gpu_state_read(struct file *file, char __user *ubuf, size_t count, loff_t *pos) { - struct i915_gpu_state *error = file->private_data; - struct drm_i915_error_state_buf str; + struct i915_gpu_state *error; ssize_t ret; - loff_t tmp; + void *buf; + error = file->private_data; if (!error) return 0; - ret = i915_error_state_buf_init(&str, error->i915, count, *pos); - if (ret) - return ret; + /* Bounce buffer required because of kernfs __user API convenience. */ + buf = kmalloc(count, GFP_KERNEL); + if (!buf) + return -ENOMEM; - ret = i915_error_state_to_str(&str, error); - if (ret) + ret = i915_gpu_state_copy_to_buffer(error, buf, *pos, count); + if (ret <= 0) goto out; - tmp = 0; - ret = simple_read_from_buffer(ubuf, count, &tmp, str.buf, str.bytes); - if (ret < 0) - goto out; + if (!copy_to_user(ubuf, buf, ret)) + *pos += ret; + else + ret = -EFAULT; - *pos = str.start + ret; out: - i915_error_state_buf_release(&str); + kfree(buf); return ret; } @@ -980,12 +961,13 @@ static int i915_gpu_info_open(struct inode *inode, struct file *file) { struct drm_i915_private *i915 = inode->i_private; struct i915_gpu_state *gpu; + intel_wakeref_t wakeref; - intel_runtime_pm_get(i915); - gpu = i915_capture_gpu_state(i915); - intel_runtime_pm_put(i915); - if (!gpu) - return -ENOMEM; + gpu = NULL; + with_intel_runtime_pm(i915, wakeref) + gpu = i915_capture_gpu_state(i915); + if (IS_ERR(gpu)) + return PTR_ERR(gpu); file->private_data = gpu; return 0; @@ -1018,7 +1000,13 @@ i915_error_state_write(struct file *filp, static int i915_error_state_open(struct inode *inode, struct file *file) { - file->private_data = i915_first_error_state(inode->i_private); + struct i915_gpu_state *error; + + error = i915_first_error_state(inode->i_private); + if (IS_ERR(error)) + return PTR_ERR(error); + + file->private_data = error; return 0; } @@ -1032,39 +1020,16 @@ static const struct file_operations i915_error_state_fops = { }; #endif -static int -i915_next_seqno_set(void *data, u64 val) -{ - struct drm_i915_private *dev_priv = data; - struct drm_device *dev = &dev_priv->drm; - int ret; - - ret = mutex_lock_interruptible(&dev->struct_mutex); - if (ret) - return ret; - - intel_runtime_pm_get(dev_priv); - ret = i915_gem_set_global_seqno(dev, val); - intel_runtime_pm_put(dev_priv); - - mutex_unlock(&dev->struct_mutex); - - return ret; -} - -DEFINE_SIMPLE_ATTRIBUTE(i915_next_seqno_fops, - NULL, i915_next_seqno_set, - "0x%llx\n"); - static int i915_frequency_info(struct seq_file *m, void *unused) { struct drm_i915_private *dev_priv = node_to_i915(m->private); struct intel_rps *rps = &dev_priv->gt_pm.rps; + intel_wakeref_t wakeref; int ret = 0; - intel_runtime_pm_get(dev_priv); + wakeref = intel_runtime_pm_get(dev_priv); - if (IS_GEN5(dev_priv)) { + if (IS_GEN(dev_priv, 5)) { u16 rgvswctl = I915_READ16(MEMSWCTL); u16 rgvstat = I915_READ16(MEMSTAT_ILK); @@ -1274,7 +1239,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused) seq_printf(m, "Max CD clock frequency: %d kHz\n", dev_priv->max_cdclk_freq); seq_printf(m, "Max pixel clock frequency: %d kHz\n", dev_priv->max_dotclk_freq); - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, wakeref); return ret; } @@ -1313,14 +1278,13 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused) u64 acthd[I915_NUM_ENGINES]; u32 seqno[I915_NUM_ENGINES]; struct intel_instdone instdone; + intel_wakeref_t wakeref; enum intel_engine_id id; if (test_bit(I915_WEDGED, &dev_priv->gpu_error.flags)) seq_puts(m, "Wedged\n"); if (test_bit(I915_RESET_BACKOFF, &dev_priv->gpu_error.flags)) seq_puts(m, "Reset in progress: struct_mutex backoff\n"); - if (test_bit(I915_RESET_HANDOFF, &dev_priv->gpu_error.flags)) - seq_puts(m, "Reset in progress: reset handoff to waiter\n"); if (waitqueue_active(&dev_priv->gpu_error.wait_queue)) seq_puts(m, "Waiter holding struct mutex\n"); if (waitqueue_active(&dev_priv->gpu_error.reset_queue)) @@ -1331,17 +1295,15 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused) return 0; } - intel_runtime_pm_get(dev_priv); + with_intel_runtime_pm(dev_priv, wakeref) { + for_each_engine(engine, dev_priv, id) { + acthd[id] = intel_engine_get_active_head(engine); + seqno[id] = intel_engine_get_seqno(engine); + } - for_each_engine(engine, dev_priv, id) { - acthd[id] = intel_engine_get_active_head(engine); - seqno[id] = intel_engine_get_seqno(engine); + intel_engine_get_instdone(dev_priv->engine[RCS], &instdone); } - intel_engine_get_instdone(dev_priv->engine[RCS], &instdone); - - intel_runtime_pm_put(dev_priv); - if (timer_pending(&dev_priv->gpu_error.hangcheck_work.timer)) seq_printf(m, "Hangcheck active, timer fires in %dms\n", jiffies_to_msecs(dev_priv->gpu_error.hangcheck_work.timer.expires - @@ -1354,37 +1316,16 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused) seq_printf(m, "GT active? %s\n", yesno(dev_priv->gt.awake)); for_each_engine(engine, dev_priv, id) { - struct intel_breadcrumbs *b = &engine->breadcrumbs; - struct rb_node *rb; - seq_printf(m, "%s:\n", engine->name); - seq_printf(m, "\tseqno = %x [current %x, last %x]\n", + seq_printf(m, "\tseqno = %x [current %x, last %x], %dms ago\n", engine->hangcheck.seqno, seqno[id], - intel_engine_last_submit(engine)); - seq_printf(m, "\twaiters? %s, fake irq active? %s, stalled? %s, wedged? %s\n", - yesno(intel_engine_has_waiter(engine)), - yesno(test_bit(engine->id, - &dev_priv->gpu_error.missed_irq_rings)), - yesno(engine->hangcheck.stalled), - yesno(engine->hangcheck.wedged)); - - spin_lock_irq(&b->rb_lock); - for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) { - struct intel_wait *w = rb_entry(rb, typeof(*w), node); - - seq_printf(m, "\t%s [%d] waiting for %x\n", - w->tsk->comm, w->tsk->pid, w->seqno); - } - spin_unlock_irq(&b->rb_lock); + intel_engine_last_submit(engine), + jiffies_to_msecs(jiffies - + engine->hangcheck.action_timestamp)); seq_printf(m, "\tACTHD = 0x%08llx [current 0x%08llx]\n", (long long)engine->hangcheck.acthd, (long long)acthd[id]); - seq_printf(m, "\taction = %s(%d) %d ms ago\n", - hangcheck_action_to_str(engine->hangcheck.action), - engine->hangcheck.action, - jiffies_to_msecs(jiffies - - engine->hangcheck.action_timestamp)); if (engine->id == RCS) { seq_puts(m, "\tinstdone read =\n"); @@ -1616,18 +1557,17 @@ static int gen6_drpc_info(struct seq_file *m) static int i915_drpc_info(struct seq_file *m, void *unused) { struct drm_i915_private *dev_priv = node_to_i915(m->private); - int err; - - intel_runtime_pm_get(dev_priv); - - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) - err = vlv_drpc_info(m); - else if (INTEL_GEN(dev_priv) >= 6) - err = gen6_drpc_info(m); - else - err = ironlake_drpc_info(m); - - intel_runtime_pm_put(dev_priv); + intel_wakeref_t wakeref; + int err = -ENODEV; + + with_intel_runtime_pm(dev_priv, wakeref) { + if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) + err = vlv_drpc_info(m); + else if (INTEL_GEN(dev_priv) >= 6) + err = gen6_drpc_info(m); + else + err = ironlake_drpc_info(m); + } return err; } @@ -1649,11 +1589,12 @@ static int i915_fbc_status(struct seq_file *m, void *unused) { struct drm_i915_private *dev_priv = node_to_i915(m->private); struct intel_fbc *fbc = &dev_priv->fbc; + intel_wakeref_t wakeref; if (!HAS_FBC(dev_priv)) return -ENODEV; - intel_runtime_pm_get(dev_priv); + wakeref = intel_runtime_pm_get(dev_priv); mutex_lock(&fbc->lock); if (intel_fbc_is_active(dev_priv)) @@ -1680,7 +1621,7 @@ static int i915_fbc_status(struct seq_file *m, void *unused) } mutex_unlock(&fbc->lock); - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, wakeref); return 0; } @@ -1725,11 +1666,12 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_fbc_false_color_fops, static int i915_ips_status(struct seq_file *m, void *unused) { struct drm_i915_private *dev_priv = node_to_i915(m->private); + intel_wakeref_t wakeref; if (!HAS_IPS(dev_priv)) return -ENODEV; - intel_runtime_pm_get(dev_priv); + wakeref = intel_runtime_pm_get(dev_priv); seq_printf(m, "Enabled by kernel parameter: %s\n", yesno(i915_modparams.enable_ips)); @@ -1743,7 +1685,7 @@ static int i915_ips_status(struct seq_file *m, void *unused) seq_puts(m, "Currently: disabled\n"); } - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, wakeref); return 0; } @@ -1751,10 +1693,10 @@ static int i915_ips_status(struct seq_file *m, void *unused) static int i915_sr_status(struct seq_file *m, void *unused) { struct drm_i915_private *dev_priv = node_to_i915(m->private); + intel_wakeref_t wakeref; bool sr_enabled = false; - intel_runtime_pm_get(dev_priv); - intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); + wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); if (INTEL_GEN(dev_priv) >= 9) /* no global SR status; inspect per-plane WM */; @@ -1770,8 +1712,7 @@ static int i915_sr_status(struct seq_file *m, void *unused) else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) sr_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN; - intel_display_power_put(dev_priv, POWER_DOMAIN_INIT); - intel_runtime_pm_put(dev_priv); + intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, wakeref); seq_printf(m, "self-refresh: %s\n", enableddisabled(sr_enabled)); @@ -1780,27 +1721,24 @@ static int i915_sr_status(struct seq_file *m, void *unused) static int i915_emon_status(struct seq_file *m, void *unused) { - struct drm_i915_private *dev_priv = node_to_i915(m->private); - struct drm_device *dev = &dev_priv->drm; - unsigned long temp, chipset, gfx; - int ret; + struct drm_i915_private *i915 = node_to_i915(m->private); + intel_wakeref_t wakeref; - if (!IS_GEN5(dev_priv)) + if (!IS_GEN(i915, 5)) return -ENODEV; - ret = mutex_lock_interruptible(&dev->struct_mutex); - if (ret) - return ret; + with_intel_runtime_pm(i915, wakeref) { + unsigned long temp, chipset, gfx; - temp = i915_mch_val(dev_priv); - chipset = i915_chipset_val(dev_priv); - gfx = i915_gfx_val(dev_priv); - mutex_unlock(&dev->struct_mutex); + temp = i915_mch_val(i915); + chipset = i915_chipset_val(i915); + gfx = i915_gfx_val(i915); - seq_printf(m, "GMCH temp: %ld\n", temp); - seq_printf(m, "Chipset power: %ld\n", chipset); - seq_printf(m, "GFX power: %ld\n", gfx); - seq_printf(m, "Total power: %ld\n", chipset + gfx); + seq_printf(m, "GMCH temp: %ld\n", temp); + seq_printf(m, "Chipset power: %ld\n", chipset); + seq_printf(m, "GFX power: %ld\n", gfx); + seq_printf(m, "Total power: %ld\n", chipset + gfx); + } return 0; } @@ -1810,13 +1748,14 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused) struct drm_i915_private *dev_priv = node_to_i915(m->private); struct intel_rps *rps = &dev_priv->gt_pm.rps; unsigned int max_gpu_freq, min_gpu_freq; + intel_wakeref_t wakeref; int gpu_freq, ia_freq; int ret; if (!HAS_LLC(dev_priv)) return -ENODEV; - intel_runtime_pm_get(dev_priv); + wakeref = intel_runtime_pm_get(dev_priv); ret = mutex_lock_interruptible(&dev_priv->pcu_lock); if (ret) @@ -1849,7 +1788,7 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused) mutex_unlock(&dev_priv->pcu_lock); out: - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, wakeref); return ret; } @@ -2022,15 +1961,16 @@ static const char *swizzle_string(unsigned swizzle) static int i915_swizzle_info(struct seq_file *m, void *data) { struct drm_i915_private *dev_priv = node_to_i915(m->private); + intel_wakeref_t wakeref; - intel_runtime_pm_get(dev_priv); + wakeref = intel_runtime_pm_get(dev_priv); seq_printf(m, "bit6 swizzle for X-tiling = %s\n", swizzle_string(dev_priv->mm.bit_6_swizzle_x)); seq_printf(m, "bit6 swizzle for Y-tiling = %s\n", swizzle_string(dev_priv->mm.bit_6_swizzle_y)); - if (IS_GEN3(dev_priv) || IS_GEN4(dev_priv)) { + if (IS_GEN_RANGE(dev_priv, 3, 4)) { seq_printf(m, "DDC = 0x%08x\n", I915_READ(DCC)); seq_printf(m, "DDC2 = 0x%08x\n", @@ -2061,141 +2001,11 @@ static int i915_swizzle_info(struct seq_file *m, void *data) if (dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) seq_puts(m, "L-shaped memory detected\n"); - intel_runtime_pm_put(dev_priv); - - return 0; -} - -static int per_file_ctx(int id, void *ptr, void *data) -{ - struct i915_gem_context *ctx = ptr; - struct seq_file *m = data; - struct i915_hw_ppgtt *ppgtt = ctx->ppgtt; - - if (!ppgtt) { - seq_printf(m, " no ppgtt for context %d\n", - ctx->user_handle); - return 0; - } - - if (i915_gem_context_is_default(ctx)) - seq_puts(m, " default context:\n"); - else - seq_printf(m, " context %d:\n", ctx->user_handle); - ppgtt->debug_dump(ppgtt, m); + intel_runtime_pm_put(dev_priv, wakeref); return 0; } -static void gen8_ppgtt_info(struct seq_file *m, - struct drm_i915_private *dev_priv) -{ - struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; - struct intel_engine_cs *engine; - enum intel_engine_id id; - int i; - - if (!ppgtt) - return; - - for_each_engine(engine, dev_priv, id) { - seq_printf(m, "%s\n", engine->name); - for (i = 0; i < 4; i++) { - u64 pdp = I915_READ(GEN8_RING_PDP_UDW(engine, i)); - pdp <<= 32; - pdp |= I915_READ(GEN8_RING_PDP_LDW(engine, i)); - seq_printf(m, "\tPDP%d 0x%016llx\n", i, pdp); - } - } -} - -static void gen6_ppgtt_info(struct seq_file *m, - struct drm_i915_private *dev_priv) -{ - struct intel_engine_cs *engine; - enum intel_engine_id id; - - if (IS_GEN6(dev_priv)) - seq_printf(m, "GFX_MODE: 0x%08x\n", I915_READ(GFX_MODE)); - - for_each_engine(engine, dev_priv, id) { - seq_printf(m, "%s\n", engine->name); - if (IS_GEN7(dev_priv)) - seq_printf(m, "GFX_MODE: 0x%08x\n", - I915_READ(RING_MODE_GEN7(engine))); - seq_printf(m, "PP_DIR_BASE: 0x%08x\n", - I915_READ(RING_PP_DIR_BASE(engine))); - seq_printf(m, "PP_DIR_BASE_READ: 0x%08x\n", - I915_READ(RING_PP_DIR_BASE_READ(engine))); - seq_printf(m, "PP_DIR_DCLV: 0x%08x\n", - I915_READ(RING_PP_DIR_DCLV(engine))); - } - if (dev_priv->mm.aliasing_ppgtt) { - struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; - - seq_puts(m, "aliasing PPGTT:\n"); - seq_printf(m, "pd gtt offset: 0x%08x\n", ppgtt->pd.base.ggtt_offset); - - ppgtt->debug_dump(ppgtt, m); - } - - seq_printf(m, "ECOCHK: 0x%08x\n", I915_READ(GAM_ECOCHK)); -} - -static int i915_ppgtt_info(struct seq_file *m, void *data) -{ - struct drm_i915_private *dev_priv = node_to_i915(m->private); - struct drm_device *dev = &dev_priv->drm; - struct drm_file *file; - int ret; - - mutex_lock(&dev->filelist_mutex); - ret = mutex_lock_interruptible(&dev->struct_mutex); - if (ret) - goto out_unlock; - - intel_runtime_pm_get(dev_priv); - - if (INTEL_GEN(dev_priv) >= 8) - gen8_ppgtt_info(m, dev_priv); - else if (INTEL_GEN(dev_priv) >= 6) - gen6_ppgtt_info(m, dev_priv); - - list_for_each_entry_reverse(file, &dev->filelist, lhead) { - struct drm_i915_file_private *file_priv = file->driver_priv; - struct task_struct *task; - - task = get_pid_task(file->pid, PIDTYPE_PID); - if (!task) { - ret = -ESRCH; - goto out_rpm; - } - seq_printf(m, "\nproc: %s\n", task->comm); - put_task_struct(task); - idr_for_each(&file_priv->context_idr, per_file_ctx, - (void *)(unsigned long)m); - } - -out_rpm: - intel_runtime_pm_put(dev_priv); - mutex_unlock(&dev->struct_mutex); -out_unlock: - mutex_unlock(&dev->filelist_mutex); - return ret; -} - -static int count_irq_waiters(struct drm_i915_private *i915) -{ - struct intel_engine_cs *engine; - enum intel_engine_id id; - int count = 0; - - for_each_engine(engine, i915, id) - count += intel_engine_has_waiter(engine); - - return count; -} - static const char *rps_power_to_str(unsigned int power) { static const char * const strings[] = { @@ -2215,17 +2025,32 @@ static int i915_rps_boost_info(struct seq_file *m, void *data) struct drm_i915_private *dev_priv = node_to_i915(m->private); struct drm_device *dev = &dev_priv->drm; struct intel_rps *rps = &dev_priv->gt_pm.rps; + u32 act_freq = rps->cur_freq; + intel_wakeref_t wakeref; struct drm_file *file; + with_intel_runtime_pm_if_in_use(dev_priv, wakeref) { + if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { + mutex_lock(&dev_priv->pcu_lock); + act_freq = vlv_punit_read(dev_priv, + PUNIT_REG_GPU_FREQ_STS); + act_freq = (act_freq >> 8) & 0xff; + mutex_unlock(&dev_priv->pcu_lock); + } else { + act_freq = intel_get_cagf(dev_priv, + I915_READ(GEN6_RPSTAT1)); + } + } + seq_printf(m, "RPS enabled? %d\n", rps->enabled); seq_printf(m, "GPU busy? %s [%d requests]\n", yesno(dev_priv->gt.awake), dev_priv->gt.active_requests); - seq_printf(m, "CPU waiting? %d\n", count_irq_waiters(dev_priv)); seq_printf(m, "Boosts outstanding? %d\n", atomic_read(&rps->num_waiters)); seq_printf(m, "Interactive? %d\n", READ_ONCE(rps->power.interactive)); - seq_printf(m, "Frequency requested %d\n", - intel_gpu_freq(dev_priv, rps->cur_freq)); + seq_printf(m, "Frequency requested %d, actual %d\n", + intel_gpu_freq(dev_priv, rps->cur_freq), + intel_gpu_freq(dev_priv, act_freq)); seq_printf(m, " min hard:%d, soft:%d; max soft:%d, hard:%d\n", intel_gpu_freq(dev_priv, rps->min_freq), intel_gpu_freq(dev_priv, rps->min_freq_softlimit), @@ -2296,6 +2121,7 @@ static int i915_llc(struct seq_file *m, void *data) static int i915_huc_load_status_info(struct seq_file *m, void *data) { struct drm_i915_private *dev_priv = node_to_i915(m->private); + intel_wakeref_t wakeref; struct drm_printer p; if (!HAS_HUC(dev_priv)) @@ -2304,9 +2130,8 @@ static int i915_huc_load_status_info(struct seq_file *m, void *data) p = drm_seq_file_printer(m); intel_uc_fw_dump(&dev_priv->huc.fw, &p); - intel_runtime_pm_get(dev_priv); - seq_printf(m, "\nHuC status 0x%08x:\n", I915_READ(HUC_STATUS2)); - intel_runtime_pm_put(dev_priv); + with_intel_runtime_pm(dev_priv, wakeref) + seq_printf(m, "\nHuC status 0x%08x:\n", I915_READ(HUC_STATUS2)); return 0; } @@ -2314,8 +2139,8 @@ static int i915_huc_load_status_info(struct seq_file *m, void *data) static int i915_guc_load_status_info(struct seq_file *m, void *data) { struct drm_i915_private *dev_priv = node_to_i915(m->private); + intel_wakeref_t wakeref; struct drm_printer p; - u32 tmp, i; if (!HAS_GUC(dev_priv)) return -ENODEV; @@ -2323,22 +2148,23 @@ static int i915_guc_load_status_info(struct seq_file *m, void *data) p = drm_seq_file_printer(m); intel_uc_fw_dump(&dev_priv->guc.fw, &p); - intel_runtime_pm_get(dev_priv); - - tmp = I915_READ(GUC_STATUS); - - seq_printf(m, "\nGuC status 0x%08x:\n", tmp); - seq_printf(m, "\tBootrom status = 0x%x\n", - (tmp & GS_BOOTROM_MASK) >> GS_BOOTROM_SHIFT); - seq_printf(m, "\tuKernel status = 0x%x\n", - (tmp & GS_UKERNEL_MASK) >> GS_UKERNEL_SHIFT); - seq_printf(m, "\tMIA Core status = 0x%x\n", - (tmp & GS_MIA_MASK) >> GS_MIA_SHIFT); - seq_puts(m, "\nScratch registers:\n"); - for (i = 0; i < 16; i++) - seq_printf(m, "\t%2d: \t0x%x\n", i, I915_READ(SOFT_SCRATCH(i))); - - intel_runtime_pm_put(dev_priv); + with_intel_runtime_pm(dev_priv, wakeref) { + u32 tmp = I915_READ(GUC_STATUS); + u32 i; + + seq_printf(m, "\nGuC status 0x%08x:\n", tmp); + seq_printf(m, "\tBootrom status = 0x%x\n", + (tmp & GS_BOOTROM_MASK) >> GS_BOOTROM_SHIFT); + seq_printf(m, "\tuKernel status = 0x%x\n", + (tmp & GS_UKERNEL_MASK) >> GS_UKERNEL_SHIFT); + seq_printf(m, "\tMIA Core status = 0x%x\n", + (tmp & GS_MIA_MASK) >> GS_MIA_SHIFT); + seq_puts(m, "\nScratch registers:\n"); + for (i = 0; i < 16; i++) { + seq_printf(m, "\t%2d: \t0x%x\n", + i, I915_READ(SOFT_SCRATCH(i))); + } + } return 0; } @@ -2390,7 +2216,7 @@ static void i915_guc_client_info(struct seq_file *m, { struct intel_engine_cs *engine; enum intel_engine_id id; - uint64_t tot = 0; + u64 tot = 0; seq_printf(m, "\tPriority %d, GuC stage index: %u, PD offset 0x%x\n", client->priority, client->stage_id, client->proc_desc_offset); @@ -2645,7 +2471,8 @@ DEFINE_SHOW_ATTRIBUTE(i915_psr_sink_status); static void psr_source_status(struct drm_i915_private *dev_priv, struct seq_file *m) { - u32 val, psr_status; + u32 val, status_val; + const char *status = "unknown"; if (dev_priv->psr.psr2_enabled) { static const char * const live_status[] = { @@ -2661,14 +2488,11 @@ psr_source_status(struct drm_i915_private *dev_priv, struct seq_file *m) "BUF_ON", "TG_ON" }; - psr_status = I915_READ(EDP_PSR2_STATUS); - val = (psr_status & EDP_PSR2_STATUS_STATE_MASK) >> - EDP_PSR2_STATUS_STATE_SHIFT; - if (val < ARRAY_SIZE(live_status)) { - seq_printf(m, "Source PSR status: 0x%x [%s]\n", - psr_status, live_status[val]); - return; - } + val = I915_READ(EDP_PSR2_STATUS); + status_val = (val & EDP_PSR2_STATUS_STATE_MASK) >> + EDP_PSR2_STATUS_STATE_SHIFT; + if (status_val < ARRAY_SIZE(live_status)) + status = live_status[status_val]; } else { static const char * const live_status[] = { "IDLE", @@ -2680,74 +2504,102 @@ psr_source_status(struct drm_i915_private *dev_priv, struct seq_file *m) "SRDOFFACK", "SRDENT_ON", }; - psr_status = I915_READ(EDP_PSR_STATUS); - val = (psr_status & EDP_PSR_STATUS_STATE_MASK) >> - EDP_PSR_STATUS_STATE_SHIFT; - if (val < ARRAY_SIZE(live_status)) { - seq_printf(m, "Source PSR status: 0x%x [%s]\n", - psr_status, live_status[val]); - return; - } + val = I915_READ(EDP_PSR_STATUS); + status_val = (val & EDP_PSR_STATUS_STATE_MASK) >> + EDP_PSR_STATUS_STATE_SHIFT; + if (status_val < ARRAY_SIZE(live_status)) + status = live_status[status_val]; } - seq_printf(m, "Source PSR status: 0x%x [%s]\n", psr_status, "unknown"); + seq_printf(m, "Source PSR status: %s [0x%08x]\n", status, val); } static int i915_edp_psr_status(struct seq_file *m, void *data) { struct drm_i915_private *dev_priv = node_to_i915(m->private); - u32 psrperf = 0; - bool enabled = false; - bool sink_support; + struct i915_psr *psr = &dev_priv->psr; + intel_wakeref_t wakeref; + const char *status; + bool enabled; + u32 val; if (!HAS_PSR(dev_priv)) return -ENODEV; - sink_support = dev_priv->psr.sink_support; - seq_printf(m, "Sink_Support: %s\n", yesno(sink_support)); - if (!sink_support) - return 0; + seq_printf(m, "Sink support: %s", yesno(psr->sink_support)); + if (psr->dp) + seq_printf(m, " [0x%02x]", psr->dp->psr_dpcd[0]); + seq_puts(m, "\n"); - intel_runtime_pm_get(dev_priv); + if (!psr->sink_support) + return 0; - mutex_lock(&dev_priv->psr.lock); - seq_printf(m, "PSR mode: %s\n", - dev_priv->psr.psr2_enabled ? "PSR2" : "PSR1"); - seq_printf(m, "Enabled: %s\n", yesno(dev_priv->psr.enabled)); - seq_printf(m, "Busy frontbuffer bits: 0x%03x\n", - dev_priv->psr.busy_frontbuffer_bits); + wakeref = intel_runtime_pm_get(dev_priv); + mutex_lock(&psr->lock); - if (dev_priv->psr.psr2_enabled) - enabled = I915_READ(EDP_PSR2_CTL) & EDP_PSR2_ENABLE; + if (psr->enabled) + status = psr->psr2_enabled ? "PSR2 enabled" : "PSR1 enabled"; else - enabled = I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE; + status = "disabled"; + seq_printf(m, "PSR mode: %s\n", status); - seq_printf(m, "Main link in standby mode: %s\n", - yesno(dev_priv->psr.link_standby)); + if (!psr->enabled) + goto unlock; - seq_printf(m, "HW Enabled & Active bit: %s\n", yesno(enabled)); + if (psr->psr2_enabled) { + val = I915_READ(EDP_PSR2_CTL); + enabled = val & EDP_PSR2_ENABLE; + } else { + val = I915_READ(EDP_PSR_CTL); + enabled = val & EDP_PSR_ENABLE; + } + seq_printf(m, "Source PSR ctl: %s [0x%08x]\n", + enableddisabled(enabled), val); + psr_source_status(dev_priv, m); + seq_printf(m, "Busy frontbuffer bits: 0x%08x\n", + psr->busy_frontbuffer_bits); /* * SKL+ Perf counter is reset to 0 everytime DC state is entered */ if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { - psrperf = I915_READ(EDP_PSR_PERF_CNT) & - EDP_PSR_PERF_CNT_MASK; + val = I915_READ(EDP_PSR_PERF_CNT) & EDP_PSR_PERF_CNT_MASK; + seq_printf(m, "Performance counter: %u\n", val); + } - seq_printf(m, "Performance_Counter: %u\n", psrperf); + if (psr->debug & I915_PSR_DEBUG_IRQ) { + seq_printf(m, "Last attempted entry at: %lld\n", + psr->last_entry_attempt); + seq_printf(m, "Last exit at: %lld\n", psr->last_exit); } - psr_source_status(dev_priv, m); - mutex_unlock(&dev_priv->psr.lock); + if (psr->psr2_enabled) { + u32 su_frames_val[3]; + int frame; - if (READ_ONCE(dev_priv->psr.debug) & I915_PSR_DEBUG_IRQ) { - seq_printf(m, "Last attempted entry at: %lld\n", - dev_priv->psr.last_entry_attempt); - seq_printf(m, "Last exit at: %lld\n", - dev_priv->psr.last_exit); + /* + * Reading all 3 registers before hand to minimize crossing a + * frame boundary between register reads + */ + for (frame = 0; frame < PSR2_SU_STATUS_FRAMES; frame += 3) + su_frames_val[frame / 3] = I915_READ(PSR2_SU_STATUS(frame)); + + seq_puts(m, "Frame:\tPSR2 SU blocks:\n"); + + for (frame = 0; frame < PSR2_SU_STATUS_FRAMES; frame++) { + u32 su_blocks; + + su_blocks = su_frames_val[frame / 3] & + PSR2_SU_STATUS_MASK(frame); + su_blocks = su_blocks >> PSR2_SU_STATUS_SHIFT(frame); + seq_printf(m, "%d\t%d\n", frame, su_blocks); + } } - intel_runtime_pm_put(dev_priv); +unlock: + mutex_unlock(&psr->lock); + intel_runtime_pm_put(dev_priv, wakeref); + return 0; } @@ -2756,6 +2608,7 @@ i915_edp_psr_debug_set(void *data, u64 val) { struct drm_i915_private *dev_priv = data; struct drm_modeset_acquire_ctx ctx; + intel_wakeref_t wakeref; int ret; if (!CAN_PSR(dev_priv)) @@ -2763,7 +2616,7 @@ i915_edp_psr_debug_set(void *data, u64 val) DRM_DEBUG_KMS("Setting PSR debug to %llx\n", val); - intel_runtime_pm_get(dev_priv); + wakeref = intel_runtime_pm_get(dev_priv); drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE); @@ -2778,7 +2631,7 @@ retry: drm_modeset_drop_locks(&ctx); drm_modeset_acquire_fini(&ctx); - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, wakeref); return ret; } @@ -2803,24 +2656,20 @@ static int i915_energy_uJ(struct seq_file *m, void *data) { struct drm_i915_private *dev_priv = node_to_i915(m->private); unsigned long long power; + intel_wakeref_t wakeref; u32 units; if (INTEL_GEN(dev_priv) < 6) return -ENODEV; - intel_runtime_pm_get(dev_priv); - - if (rdmsrl_safe(MSR_RAPL_POWER_UNIT, &power)) { - intel_runtime_pm_put(dev_priv); + if (rdmsrl_safe(MSR_RAPL_POWER_UNIT, &power)) return -ENODEV; - } units = (power & 0x1f00) >> 8; - power = I915_READ(MCH_SECP_NRG_STTS); - power = (1000000 * power) >> units; /* convert to uJ */ - - intel_runtime_pm_put(dev_priv); + with_intel_runtime_pm(dev_priv, wakeref) + power = I915_READ(MCH_SECP_NRG_STTS); + power = (1000000 * power) >> units; /* convert to uJ */ seq_printf(m, "%llu", power); return 0; @@ -2834,6 +2683,9 @@ static int i915_runtime_pm_status(struct seq_file *m, void *unused) if (!HAS_RUNTIME_PM(dev_priv)) seq_puts(m, "Runtime power management not supported\n"); + seq_printf(m, "Runtime power status: %s\n", + enableddisabled(!dev_priv->power_domains.wakeref)); + seq_printf(m, "GPU idle: %s (epoch %u)\n", yesno(!dev_priv->gt.awake), dev_priv->gt.epoch); seq_printf(m, "IRQs disabled: %s\n", @@ -2848,6 +2700,12 @@ static int i915_runtime_pm_status(struct seq_file *m, void *unused) pci_power_name(pdev->current_state), pdev->current_state); + if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)) { + struct drm_printer p = drm_seq_file_printer(m); + + print_intel_runtime_pm_wakeref(dev_priv, &p); + } + return 0; } @@ -2882,6 +2740,7 @@ static int i915_power_domain_info(struct seq_file *m, void *unused) static int i915_dmc_info(struct seq_file *m, void *unused) { struct drm_i915_private *dev_priv = node_to_i915(m->private); + intel_wakeref_t wakeref; struct intel_csr *csr; if (!HAS_CSR(dev_priv)) @@ -2889,7 +2748,7 @@ static int i915_dmc_info(struct seq_file *m, void *unused) csr = &dev_priv->csr; - intel_runtime_pm_get(dev_priv); + wakeref = intel_runtime_pm_get(dev_priv); seq_printf(m, "fw loaded: %s\n", yesno(csr->dmc_payload != NULL)); seq_printf(m, "path: %s\n", csr->fw_path); @@ -2900,23 +2759,22 @@ static int i915_dmc_info(struct seq_file *m, void *unused) seq_printf(m, "version: %d.%d\n", CSR_VERSION_MAJOR(csr->version), CSR_VERSION_MINOR(csr->version)); - if (IS_KABYLAKE(dev_priv) || - (IS_SKYLAKE(dev_priv) && csr->version >= CSR_VERSION(1, 6))) { - seq_printf(m, "DC3 -> DC5 count: %d\n", - I915_READ(SKL_CSR_DC3_DC5_COUNT)); + if (WARN_ON(INTEL_GEN(dev_priv) > 11)) + goto out; + + seq_printf(m, "DC3 -> DC5 count: %d\n", + I915_READ(IS_BROXTON(dev_priv) ? BXT_CSR_DC3_DC5_COUNT : + SKL_CSR_DC3_DC5_COUNT)); + if (!IS_GEN9_LP(dev_priv)) seq_printf(m, "DC5 -> DC6 count: %d\n", I915_READ(SKL_CSR_DC5_DC6_COUNT)); - } else if (IS_BROXTON(dev_priv) && csr->version >= CSR_VERSION(1, 4)) { - seq_printf(m, "DC3 -> DC5 count: %d\n", - I915_READ(BXT_CSR_DC3_DC5_COUNT)); - } out: seq_printf(m, "program base: 0x%08x\n", I915_READ(CSR_PROGRAM(0))); seq_printf(m, "ssp base: 0x%08x\n", I915_READ(CSR_SSP_BASE)); seq_printf(m, "htp: 0x%08x\n", I915_READ(CSR_HTP_SKL)); - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, wakeref); return 0; } @@ -2929,14 +2787,7 @@ static void intel_seq_print_mode(struct seq_file *m, int tabs, for (i = 0; i < tabs; i++) seq_putc(m, '\t'); - seq_printf(m, "id %d:\"%s\" freq %d clock %d hdisp %d hss %d hse %d htot %d vdisp %d vss %d vse %d vtot %d type 0x%x flags 0x%x\n", - mode->base.id, mode->name, - mode->vrefresh, mode->clock, - mode->hdisplay, mode->hsync_start, - mode->hsync_end, mode->htotal, - mode->vdisplay, mode->vsync_start, - mode->vsync_end, mode->vtotal, - mode->type, mode->flags); + seq_printf(m, DRM_MODE_FMT "\n", DRM_MODE_ARG(mode)); } static void intel_encoder_info(struct seq_file *m, @@ -3049,16 +2900,17 @@ static void intel_connector_info(struct seq_file *m, seq_printf(m, "connector %d: type %s, status: %s\n", connector->base.id, connector->name, drm_get_connector_status_name(connector->status)); - if (connector->status == connector_status_connected) { - seq_printf(m, "\tname: %s\n", connector->display_info.name); - seq_printf(m, "\tphysical dimensions: %dx%dmm\n", - connector->display_info.width_mm, - connector->display_info.height_mm); - seq_printf(m, "\tsubpixel order: %s\n", - drm_get_subpixel_order_name(connector->display_info.subpixel_order)); - seq_printf(m, "\tCEA rev: %d\n", - connector->display_info.cea_rev); - } + + if (connector->status == connector_status_disconnected) + return; + + seq_printf(m, "\tname: %s\n", connector->display_info.name); + seq_printf(m, "\tphysical dimensions: %dx%dmm\n", + connector->display_info.width_mm, + connector->display_info.height_mm); + seq_printf(m, "\tsubpixel order: %s\n", + drm_get_subpixel_order_name(connector->display_info.subpixel_order)); + seq_printf(m, "\tCEA rev: %d\n", connector->display_info.cea_rev); if (!intel_encoder) return; @@ -3107,14 +2959,13 @@ static const char *plane_type(enum drm_plane_type type) return "unknown"; } -static const char *plane_rotation(unsigned int rotation) +static void plane_rotation(char *buf, size_t bufsize, unsigned int rotation) { - static char buf[48]; /* * According to doc only one DRM_MODE_ROTATE_ is allowed but this * will print them all to visualize if the values are misused */ - snprintf(buf, sizeof(buf), + snprintf(buf, bufsize, "%s%s%s%s%s%s(0x%08x)", (rotation & DRM_MODE_ROTATE_0) ? "0 " : "", (rotation & DRM_MODE_ROTATE_90) ? "90 " : "", @@ -3123,8 +2974,6 @@ static const char *plane_rotation(unsigned int rotation) (rotation & DRM_MODE_REFLECT_X) ? "FLIPX " : "", (rotation & DRM_MODE_REFLECT_Y) ? "FLIPY " : "", rotation); - - return buf; } static void intel_plane_info(struct seq_file *m, struct intel_crtc *intel_crtc) @@ -3137,6 +2986,7 @@ static void intel_plane_info(struct seq_file *m, struct intel_crtc *intel_crtc) struct drm_plane_state *state; struct drm_plane *plane = &intel_plane->base; struct drm_format_name_buf format_name; + char rot_str[48]; if (!plane->state) { seq_puts(m, "plane->state is NULL!\n"); @@ -3152,6 +3002,8 @@ static void intel_plane_info(struct seq_file *m, struct intel_crtc *intel_crtc) sprintf(format_name.str, "N/A"); } + plane_rotation(rot_str, sizeof(rot_str), state->rotation); + seq_printf(m, "\t--Plane id %d: type=%s, crtc_pos=%4dx%4d, crtc_size=%4dx%4d, src_pos=%d.%04ux%d.%04u, src_size=%d.%04ux%d.%04u, format=%s, rotation=%s\n", plane->base.id, plane_type(intel_plane->base.type), @@ -3166,7 +3018,7 @@ static void intel_plane_info(struct seq_file *m, struct intel_crtc *intel_crtc) (state->src_h >> 16), ((state->src_h & 0xffff) * 15625) >> 10, format_name.str, - plane_rotation(state->rotation)); + rot_str); } } @@ -3205,8 +3057,10 @@ static int i915_display_info(struct seq_file *m, void *unused) struct intel_crtc *crtc; struct drm_connector *connector; struct drm_connector_list_iter conn_iter; + intel_wakeref_t wakeref; + + wakeref = intel_runtime_pm_get(dev_priv); - intel_runtime_pm_get(dev_priv); seq_printf(m, "CRTC info\n"); seq_printf(m, "---------\n"); for_each_intel_crtc(dev, crtc) { @@ -3254,7 +3108,7 @@ static int i915_display_info(struct seq_file *m, void *unused) drm_connector_list_iter_end(&conn_iter); mutex_unlock(&dev->mode_config.mutex); - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, wakeref); return 0; } @@ -3263,23 +3117,24 @@ static int i915_engine_info(struct seq_file *m, void *unused) { struct drm_i915_private *dev_priv = node_to_i915(m->private); struct intel_engine_cs *engine; + intel_wakeref_t wakeref; enum intel_engine_id id; struct drm_printer p; - intel_runtime_pm_get(dev_priv); + wakeref = intel_runtime_pm_get(dev_priv); seq_printf(m, "GT awake? %s (epoch %u)\n", yesno(dev_priv->gt.awake), dev_priv->gt.epoch); seq_printf(m, "Global active requests: %d\n", dev_priv->gt.active_requests); seq_printf(m, "CS timestamp frequency: %u kHz\n", - dev_priv->info.cs_timestamp_frequency_khz); + RUNTIME_INFO(dev_priv)->cs_timestamp_frequency_khz); p = drm_seq_file_printer(m); for_each_engine(engine, dev_priv, id) intel_engine_dump(engine, &p, "%s\n", engine->name); - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, wakeref); return 0; } @@ -3289,7 +3144,7 @@ static int i915_rcs_topology(struct seq_file *m, void *unused) struct drm_i915_private *dev_priv = node_to_i915(m->private); struct drm_printer p = drm_seq_file_printer(m); - intel_device_info_dump_topology(&INTEL_INFO(dev_priv)->sseu, &p); + intel_device_info_dump_topology(&RUNTIME_INFO(dev_priv)->sseu, &p); return 0; } @@ -3355,13 +3210,15 @@ static int i915_shared_dplls_info(struct seq_file *m, void *unused) static int i915_wa_registers(struct seq_file *m, void *unused) { - struct i915_workarounds *wa = &node_to_i915(m->private)->workarounds; - int i; + struct drm_i915_private *i915 = node_to_i915(m->private); + const struct i915_wa_list *wal = &i915->engine[RCS]->ctx_wa_list; + struct i915_wa *wa; + unsigned int i; - seq_printf(m, "Workarounds applied: %d\n", wa->count); - for (i = 0; i < wa->count; ++i) + seq_printf(m, "Workarounds applied: %u\n", wal->count); + for (i = 0, wa = wal->list; i < wal->count; i++, wa++) seq_printf(m, "0x%X: 0x%08X, mask: 0x%08X\n", - wa->reg[i].addr, wa->reg[i].value, wa->reg[i].mask); + i915_mmio_reg_offset(wa->reg), wa->val, wa->mask); return 0; } @@ -3390,20 +3247,21 @@ static ssize_t i915_ipc_status_write(struct file *file, const char __user *ubuf, { struct seq_file *m = file->private_data; struct drm_i915_private *dev_priv = m->private; - int ret; + intel_wakeref_t wakeref; bool enable; + int ret; ret = kstrtobool_from_user(ubuf, len, &enable); if (ret < 0) return ret; - intel_runtime_pm_get(dev_priv); - if (!dev_priv->ipc_enabled && enable) - DRM_INFO("Enabling IPC: WM will be proper only after next commit\n"); - dev_priv->wm.distrust_bios_wm = true; - dev_priv->ipc_enabled = enable; - intel_enable_ipc(dev_priv); - intel_runtime_pm_put(dev_priv); + with_intel_runtime_pm(dev_priv, wakeref) { + if (!dev_priv->ipc_enabled && enable) + DRM_INFO("Enabling IPC: WM will be proper only after next commit\n"); + dev_priv->wm.distrust_bios_wm = true; + dev_priv->ipc_enabled = enable; + intel_enable_ipc(dev_priv); + } return len; } @@ -3421,31 +3279,32 @@ static int i915_ddb_info(struct seq_file *m, void *unused) { struct drm_i915_private *dev_priv = node_to_i915(m->private); struct drm_device *dev = &dev_priv->drm; - struct skl_ddb_allocation *ddb; struct skl_ddb_entry *entry; - enum pipe pipe; - int plane; + struct intel_crtc *crtc; if (INTEL_GEN(dev_priv) < 9) return -ENODEV; drm_modeset_lock_all(dev); - ddb = &dev_priv->wm.skl_hw.ddb; - seq_printf(m, "%-15s%8s%8s%8s\n", "", "Start", "End", "Size"); - for_each_pipe(dev_priv, pipe) { + for_each_intel_crtc(&dev_priv->drm, crtc) { + struct intel_crtc_state *crtc_state = + to_intel_crtc_state(crtc->base.state); + enum pipe pipe = crtc->pipe; + enum plane_id plane_id; + seq_printf(m, "Pipe %c\n", pipe_name(pipe)); - for_each_universal_plane(dev_priv, pipe, plane) { - entry = &ddb->plane[pipe][plane]; - seq_printf(m, " Plane%-8d%8u%8u%8u\n", plane + 1, + for_each_plane_id_on_crtc(crtc, plane_id) { + entry = &crtc_state->wm.skl.plane_ddb_y[plane_id]; + seq_printf(m, " Plane%-8d%8u%8u%8u\n", plane_id + 1, entry->start, entry->end, skl_ddb_entry_size(entry)); } - entry = &ddb->plane[pipe][PLANE_CURSOR]; + entry = &crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR]; seq_printf(m, " %-13s%8u%8u%8u\n", "Cursor", entry->start, entry->end, skl_ddb_entry_size(entry)); } @@ -3770,7 +3629,7 @@ static int i915_displayport_test_type_show(struct seq_file *m, void *data) } DEFINE_SHOW_ATTRIBUTE(i915_displayport_test_type); -static void wm_latency_show(struct seq_file *m, const uint16_t wm[8]) +static void wm_latency_show(struct seq_file *m, const u16 wm[8]) { struct drm_i915_private *dev_priv = m->private; struct drm_device *dev = &dev_priv->drm; @@ -3813,7 +3672,7 @@ static void wm_latency_show(struct seq_file *m, const uint16_t wm[8]) static int pri_wm_latency_show(struct seq_file *m, void *data) { struct drm_i915_private *dev_priv = m->private; - const uint16_t *latencies; + const u16 *latencies; if (INTEL_GEN(dev_priv) >= 9) latencies = dev_priv->wm.skl_latency; @@ -3828,7 +3687,7 @@ static int pri_wm_latency_show(struct seq_file *m, void *data) static int spr_wm_latency_show(struct seq_file *m, void *data) { struct drm_i915_private *dev_priv = m->private; - const uint16_t *latencies; + const u16 *latencies; if (INTEL_GEN(dev_priv) >= 9) latencies = dev_priv->wm.skl_latency; @@ -3843,7 +3702,7 @@ static int spr_wm_latency_show(struct seq_file *m, void *data) static int cur_wm_latency_show(struct seq_file *m, void *data) { struct drm_i915_private *dev_priv = m->private; - const uint16_t *latencies; + const u16 *latencies; if (INTEL_GEN(dev_priv) >= 9) latencies = dev_priv->wm.skl_latency; @@ -3869,7 +3728,7 @@ static int spr_wm_latency_open(struct inode *inode, struct file *file) { struct drm_i915_private *dev_priv = inode->i_private; - if (HAS_GMCH_DISPLAY(dev_priv)) + if (HAS_GMCH(dev_priv)) return -ENODEV; return single_open(file, spr_wm_latency_show, dev_priv); @@ -3879,19 +3738,19 @@ static int cur_wm_latency_open(struct inode *inode, struct file *file) { struct drm_i915_private *dev_priv = inode->i_private; - if (HAS_GMCH_DISPLAY(dev_priv)) + if (HAS_GMCH(dev_priv)) return -ENODEV; return single_open(file, cur_wm_latency_show, dev_priv); } static ssize_t wm_latency_write(struct file *file, const char __user *ubuf, - size_t len, loff_t *offp, uint16_t wm[8]) + size_t len, loff_t *offp, u16 wm[8]) { struct seq_file *m = file->private_data; struct drm_i915_private *dev_priv = m->private; struct drm_device *dev = &dev_priv->drm; - uint16_t new[8] = { 0 }; + u16 new[8] = { 0 }; int num_levels; int level; int ret; @@ -3936,7 +3795,7 @@ static ssize_t pri_wm_latency_write(struct file *file, const char __user *ubuf, { struct seq_file *m = file->private_data; struct drm_i915_private *dev_priv = m->private; - uint16_t *latencies; + u16 *latencies; if (INTEL_GEN(dev_priv) >= 9) latencies = dev_priv->wm.skl_latency; @@ -3951,7 +3810,7 @@ static ssize_t spr_wm_latency_write(struct file *file, const char __user *ubuf, { struct seq_file *m = file->private_data; struct drm_i915_private *dev_priv = m->private; - uint16_t *latencies; + u16 *latencies; if (INTEL_GEN(dev_priv) >= 9) latencies = dev_priv->wm.skl_latency; @@ -3966,7 +3825,7 @@ static ssize_t cur_wm_latency_write(struct file *file, const char __user *ubuf, { struct seq_file *m = file->private_data; struct drm_i915_private *dev_priv = m->private; - uint16_t *latencies; + u16 *latencies; if (INTEL_GEN(dev_priv) >= 9) latencies = dev_priv->wm.skl_latency; @@ -4017,8 +3876,6 @@ static int i915_wedged_set(void *data, u64 val) { struct drm_i915_private *i915 = data; - struct intel_engine_cs *engine; - unsigned int tmp; /* * There is no safeguard against this debugfs entry colliding @@ -4031,18 +3888,8 @@ i915_wedged_set(void *data, u64 val) if (i915_reset_backoff(&i915->gpu_error)) return -EAGAIN; - for_each_engine_masked(engine, i915, val, tmp) { - engine->hangcheck.seqno = intel_engine_get_seqno(engine); - engine->hangcheck.stalled = true; - } - i915_handle_error(i915, val, I915_ERROR_CAPTURE, "Manually set wedged engine mask = %llx", val); - - wait_on_bit(&i915->gpu_error.flags, - I915_RESET_HANDOFF, - TASK_UNINTERRUPTIBLE); - return 0; } @@ -4050,94 +3897,6 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_wedged_fops, i915_wedged_get, i915_wedged_set, "%llu\n"); -static int -fault_irq_set(struct drm_i915_private *i915, - unsigned long *irq, - unsigned long val) -{ - int err; - - err = mutex_lock_interruptible(&i915->drm.struct_mutex); - if (err) - return err; - - err = i915_gem_wait_for_idle(i915, - I915_WAIT_LOCKED | - I915_WAIT_INTERRUPTIBLE, - MAX_SCHEDULE_TIMEOUT); - if (err) - goto err_unlock; - - *irq = val; - mutex_unlock(&i915->drm.struct_mutex); - - /* Flush idle worker to disarm irq */ - drain_delayed_work(&i915->gt.idle_work); - - return 0; - -err_unlock: - mutex_unlock(&i915->drm.struct_mutex); - return err; -} - -static int -i915_ring_missed_irq_get(void *data, u64 *val) -{ - struct drm_i915_private *dev_priv = data; - - *val = dev_priv->gpu_error.missed_irq_rings; - return 0; -} - -static int -i915_ring_missed_irq_set(void *data, u64 val) -{ - struct drm_i915_private *i915 = data; - - return fault_irq_set(i915, &i915->gpu_error.missed_irq_rings, val); -} - -DEFINE_SIMPLE_ATTRIBUTE(i915_ring_missed_irq_fops, - i915_ring_missed_irq_get, i915_ring_missed_irq_set, - "0x%08llx\n"); - -static int -i915_ring_test_irq_get(void *data, u64 *val) -{ - struct drm_i915_private *dev_priv = data; - - *val = dev_priv->gpu_error.test_irq_rings; - - return 0; -} - -static int -i915_ring_test_irq_set(void *data, u64 val) -{ - struct drm_i915_private *i915 = data; - - /* GuC keeps the user interrupt permanently enabled for submission */ - if (USES_GUC_SUBMISSION(i915)) - return -ENODEV; - - /* - * From icl, we can no longer individually mask interrupt generation - * from each engine. - */ - if (INTEL_GEN(i915) >= 11) - return -ENODEV; - - val &= INTEL_INFO(i915)->ring_mask; - DRM_DEBUG_DRIVER("Masking interrupts on rings 0x%08llx\n", val); - - return fault_irq_set(i915, &i915->gpu_error.test_irq_rings, val); -} - -DEFINE_SIMPLE_ATTRIBUTE(i915_ring_test_irq_fops, - i915_ring_test_irq_get, i915_ring_test_irq_set, - "0x%08llx\n"); - #define DROP_UNBOUND BIT(0) #define DROP_BOUND BIT(1) #define DROP_RETIRE BIT(2) @@ -4168,12 +3927,15 @@ static int i915_drop_caches_set(void *data, u64 val) { struct drm_i915_private *i915 = data; + intel_wakeref_t wakeref; int ret = 0; DRM_DEBUG("Dropping caches: 0x%08llx [0x%08llx]\n", val, val & DROP_ALL); + wakeref = intel_runtime_pm_get(i915); - if (val & DROP_RESET_ACTIVE && !intel_engines_are_idle(i915)) + if (val & DROP_RESET_ACTIVE && + wait_for(intel_engines_are_idle(i915), I915_IDLE_ENGINES_TIMEOUT)) i915_gem_set_wedged(i915); /* No need to check and wait for gpu resets, only libdrm auto-restarts @@ -4181,7 +3943,7 @@ i915_drop_caches_set(void *data, u64 val) if (val & (DROP_ACTIVE | DROP_RETIRE | DROP_RESET_SEQNO)) { ret = mutex_lock_interruptible(&i915->drm.struct_mutex); if (ret) - return ret; + goto out; if (val & DROP_ACTIVE) ret = i915_gem_wait_for_idle(i915, @@ -4189,25 +3951,14 @@ i915_drop_caches_set(void *data, u64 val) I915_WAIT_LOCKED, MAX_SCHEDULE_TIMEOUT); - if (ret == 0 && val & DROP_RESET_SEQNO) { - intel_runtime_pm_get(i915); - ret = i915_gem_set_global_seqno(&i915->drm, 1); - intel_runtime_pm_put(i915); - } - if (val & DROP_RETIRE) i915_retire_requests(i915); mutex_unlock(&i915->drm.struct_mutex); } - if (val & DROP_RESET_ACTIVE && - i915_terminally_wedged(&i915->gpu_error)) { + if (val & DROP_RESET_ACTIVE && i915_terminally_wedged(&i915->gpu_error)) i915_handle_error(i915, ALL_ENGINES, 0, NULL); - wait_on_bit(&i915->gpu_error.flags, - I915_RESET_HANDOFF, - TASK_UNINTERRUPTIBLE); - } fs_reclaim_acquire(GFP_KERNEL); if (val & DROP_BOUND) @@ -4231,6 +3982,9 @@ i915_drop_caches_set(void *data, u64 val) if (val & DROP_FREED) i915_gem_drain_freed_objects(i915); +out: + intel_runtime_pm_put(i915, wakeref); + return ret; } @@ -4242,16 +3996,14 @@ static int i915_cache_sharing_get(void *data, u64 *val) { struct drm_i915_private *dev_priv = data; - u32 snpcr; + intel_wakeref_t wakeref; + u32 snpcr = 0; - if (!(IS_GEN6(dev_priv) || IS_GEN7(dev_priv))) + if (!(IS_GEN_RANGE(dev_priv, 6, 7))) return -ENODEV; - intel_runtime_pm_get(dev_priv); - - snpcr = I915_READ(GEN6_MBCUNIT_SNPCR); - - intel_runtime_pm_put(dev_priv); + with_intel_runtime_pm(dev_priv, wakeref) + snpcr = I915_READ(GEN6_MBCUNIT_SNPCR); *val = (snpcr & GEN6_MBC_SNPCR_MASK) >> GEN6_MBC_SNPCR_SHIFT; @@ -4262,24 +4014,25 @@ static int i915_cache_sharing_set(void *data, u64 val) { struct drm_i915_private *dev_priv = data; - u32 snpcr; + intel_wakeref_t wakeref; - if (!(IS_GEN6(dev_priv) || IS_GEN7(dev_priv))) + if (!(IS_GEN_RANGE(dev_priv, 6, 7))) return -ENODEV; if (val > 3) return -EINVAL; - intel_runtime_pm_get(dev_priv); DRM_DEBUG_DRIVER("Manually setting uncore sharing to %llu\n", val); + with_intel_runtime_pm(dev_priv, wakeref) { + u32 snpcr; + + /* Update the cache sharing policy here as well */ + snpcr = I915_READ(GEN6_MBCUNIT_SNPCR); + snpcr &= ~GEN6_MBC_SNPCR_MASK; + snpcr |= val << GEN6_MBC_SNPCR_SHIFT; + I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr); + } - /* Update the cache sharing policy here as well */ - snpcr = I915_READ(GEN6_MBCUNIT_SNPCR); - snpcr &= ~GEN6_MBC_SNPCR_MASK; - snpcr |= (val << GEN6_MBC_SNPCR_SHIFT); - I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr); - - intel_runtime_pm_put(dev_priv); return 0; } @@ -4324,14 +4077,14 @@ static void gen10_sseu_device_status(struct drm_i915_private *dev_priv, struct sseu_dev_info *sseu) { #define SS_MAX 6 - const struct intel_device_info *info = INTEL_INFO(dev_priv); + const struct intel_runtime_info *info = RUNTIME_INFO(dev_priv); u32 s_reg[SS_MAX], eu_reg[2 * SS_MAX], eu_mask[2]; int s, ss; for (s = 0; s < info->sseu.max_slices; s++) { /* * FIXME: Valid SS Mask respects the spec and read - * only valid bits for those registers, excluding reserverd + * only valid bits for those registers, excluding reserved * although this seems wrong because it would leave many * subslices without ACK. */ @@ -4380,7 +4133,7 @@ static void gen9_sseu_device_status(struct drm_i915_private *dev_priv, struct sseu_dev_info *sseu) { #define SS_MAX 3 - const struct intel_device_info *info = INTEL_INFO(dev_priv); + const struct intel_runtime_info *info = RUNTIME_INFO(dev_priv); u32 s_reg[SS_MAX], eu_reg[2 * SS_MAX], eu_mask[2]; int s, ss; @@ -4408,7 +4161,7 @@ static void gen9_sseu_device_status(struct drm_i915_private *dev_priv, if (IS_GEN9_BC(dev_priv)) sseu->subslice_mask[s] = - INTEL_INFO(dev_priv)->sseu.subslice_mask[s]; + RUNTIME_INFO(dev_priv)->sseu.subslice_mask[s]; for (ss = 0; ss < info->sseu.max_subslices; ss++) { unsigned int eu_cnt; @@ -4442,10 +4195,10 @@ static void broadwell_sseu_device_status(struct drm_i915_private *dev_priv, if (sseu->slice_mask) { sseu->eu_per_subslice = - INTEL_INFO(dev_priv)->sseu.eu_per_subslice; + RUNTIME_INFO(dev_priv)->sseu.eu_per_subslice; for (s = 0; s < fls(sseu->slice_mask); s++) { sseu->subslice_mask[s] = - INTEL_INFO(dev_priv)->sseu.subslice_mask[s]; + RUNTIME_INFO(dev_priv)->sseu.subslice_mask[s]; } sseu->eu_total = sseu->eu_per_subslice * sseu_subslice_total(sseu); @@ -4453,7 +4206,7 @@ static void broadwell_sseu_device_status(struct drm_i915_private *dev_priv, /* subtract fused off EU(s) from enabled slice(s) */ for (s = 0; s < fls(sseu->slice_mask); s++) { u8 subslice_7eu = - INTEL_INFO(dev_priv)->sseu.subslice_7eu[s]; + RUNTIME_INFO(dev_priv)->sseu.subslice_7eu[s]; sseu->eu_total -= hweight8(subslice_7eu); } @@ -4501,34 +4254,32 @@ static int i915_sseu_status(struct seq_file *m, void *unused) { struct drm_i915_private *dev_priv = node_to_i915(m->private); struct sseu_dev_info sseu; + intel_wakeref_t wakeref; if (INTEL_GEN(dev_priv) < 8) return -ENODEV; seq_puts(m, "SSEU Device Info\n"); - i915_print_sseu_info(m, true, &INTEL_INFO(dev_priv)->sseu); + i915_print_sseu_info(m, true, &RUNTIME_INFO(dev_priv)->sseu); seq_puts(m, "SSEU Device Status\n"); memset(&sseu, 0, sizeof(sseu)); - sseu.max_slices = INTEL_INFO(dev_priv)->sseu.max_slices; - sseu.max_subslices = INTEL_INFO(dev_priv)->sseu.max_subslices; + sseu.max_slices = RUNTIME_INFO(dev_priv)->sseu.max_slices; + sseu.max_subslices = RUNTIME_INFO(dev_priv)->sseu.max_subslices; sseu.max_eus_per_subslice = - INTEL_INFO(dev_priv)->sseu.max_eus_per_subslice; - - intel_runtime_pm_get(dev_priv); - - if (IS_CHERRYVIEW(dev_priv)) { - cherryview_sseu_device_status(dev_priv, &sseu); - } else if (IS_BROADWELL(dev_priv)) { - broadwell_sseu_device_status(dev_priv, &sseu); - } else if (IS_GEN9(dev_priv)) { - gen9_sseu_device_status(dev_priv, &sseu); - } else if (INTEL_GEN(dev_priv) >= 10) { - gen10_sseu_device_status(dev_priv, &sseu); + RUNTIME_INFO(dev_priv)->sseu.max_eus_per_subslice; + + with_intel_runtime_pm(dev_priv, wakeref) { + if (IS_CHERRYVIEW(dev_priv)) + cherryview_sseu_device_status(dev_priv, &sseu); + else if (IS_BROADWELL(dev_priv)) + broadwell_sseu_device_status(dev_priv, &sseu); + else if (IS_GEN(dev_priv, 9)) + gen9_sseu_device_status(dev_priv, &sseu); + else if (INTEL_GEN(dev_priv) >= 10) + gen10_sseu_device_status(dev_priv, &sseu); } - intel_runtime_pm_put(dev_priv); - i915_print_sseu_info(m, false, &sseu); return 0; @@ -4541,7 +4292,7 @@ static int i915_forcewake_open(struct inode *inode, struct file *file) if (INTEL_GEN(i915) < 6) return 0; - intel_runtime_pm_get(i915); + file->private_data = (void *)(uintptr_t)intel_runtime_pm_get(i915); intel_uncore_forcewake_user_get(i915); return 0; @@ -4555,7 +4306,8 @@ static int i915_forcewake_release(struct inode *inode, struct file *file) return 0; intel_uncore_forcewake_user_put(i915); - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, + (intel_wakeref_t)(uintptr_t)file->private_data); return 0; } @@ -4571,6 +4323,13 @@ static int i915_hpd_storm_ctl_show(struct seq_file *m, void *data) struct drm_i915_private *dev_priv = m->private; struct i915_hotplug *hotplug = &dev_priv->hotplug; + /* Synchronize with everything first in case there's been an HPD + * storm, but we haven't finished handling it in the kernel yet + */ + synchronize_irq(dev_priv->drm.irq); + flush_work(&dev_priv->hotplug.dig_port_work); + flush_work(&dev_priv->hotplug.hotplug_work); + seq_printf(m, "Threshold: %d\n", hotplug->hpd_storm_threshold); seq_printf(m, "Detected: %s\n", yesno(delayed_work_pending(&hotplug->reenable_work))); @@ -4641,24 +4400,122 @@ static const struct file_operations i915_hpd_storm_ctl_fops = { .write = i915_hpd_storm_ctl_write }; +static int i915_hpd_short_storm_ctl_show(struct seq_file *m, void *data) +{ + struct drm_i915_private *dev_priv = m->private; + + seq_printf(m, "Enabled: %s\n", + yesno(dev_priv->hotplug.hpd_short_storm_enabled)); + + return 0; +} + +static int +i915_hpd_short_storm_ctl_open(struct inode *inode, struct file *file) +{ + return single_open(file, i915_hpd_short_storm_ctl_show, + inode->i_private); +} + +static ssize_t i915_hpd_short_storm_ctl_write(struct file *file, + const char __user *ubuf, + size_t len, loff_t *offp) +{ + struct seq_file *m = file->private_data; + struct drm_i915_private *dev_priv = m->private; + struct i915_hotplug *hotplug = &dev_priv->hotplug; + char *newline; + char tmp[16]; + int i; + bool new_state; + + if (len >= sizeof(tmp)) + return -EINVAL; + + if (copy_from_user(tmp, ubuf, len)) + return -EFAULT; + + tmp[len] = '\0'; + + /* Strip newline, if any */ + newline = strchr(tmp, '\n'); + if (newline) + *newline = '\0'; + + /* Reset to the "default" state for this system */ + if (strcmp(tmp, "reset") == 0) + new_state = !HAS_DP_MST(dev_priv); + else if (kstrtobool(tmp, &new_state) != 0) + return -EINVAL; + + DRM_DEBUG_KMS("%sabling HPD short storm detection\n", + new_state ? "En" : "Dis"); + + spin_lock_irq(&dev_priv->irq_lock); + hotplug->hpd_short_storm_enabled = new_state; + /* Reset the HPD storm stats so we don't accidentally trigger a storm */ + for_each_hpd_pin(i) + hotplug->stats[i].count = 0; + spin_unlock_irq(&dev_priv->irq_lock); + + /* Re-enable hpd immediately if we were in an irq storm */ + flush_delayed_work(&dev_priv->hotplug.reenable_work); + + return len; +} + +static const struct file_operations i915_hpd_short_storm_ctl_fops = { + .owner = THIS_MODULE, + .open = i915_hpd_short_storm_ctl_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, + .write = i915_hpd_short_storm_ctl_write, +}; + static int i915_drrs_ctl_set(void *data, u64 val) { struct drm_i915_private *dev_priv = data; struct drm_device *dev = &dev_priv->drm; - struct intel_crtc *intel_crtc; - struct intel_encoder *encoder; - struct intel_dp *intel_dp; + struct intel_crtc *crtc; if (INTEL_GEN(dev_priv) < 7) return -ENODEV; - drm_modeset_lock_all(dev); - for_each_intel_crtc(dev, intel_crtc) { - if (!intel_crtc->base.state->active || - !intel_crtc->config->has_drrs) - continue; + for_each_intel_crtc(dev, crtc) { + struct drm_connector_list_iter conn_iter; + struct intel_crtc_state *crtc_state; + struct drm_connector *connector; + struct drm_crtc_commit *commit; + int ret; + + ret = drm_modeset_lock_single_interruptible(&crtc->base.mutex); + if (ret) + return ret; + + crtc_state = to_intel_crtc_state(crtc->base.state); + + if (!crtc_state->base.active || + !crtc_state->has_drrs) + goto out; + + commit = crtc_state->base.commit; + if (commit) { + ret = wait_for_completion_interruptible(&commit->hw_done); + if (ret) + goto out; + } + + drm_connector_list_iter_begin(dev, &conn_iter); + drm_for_each_connector_iter(connector, &conn_iter) { + struct intel_encoder *encoder; + struct intel_dp *intel_dp; - for_each_encoder_on_crtc(dev, &intel_crtc->base, encoder) { + if (!(crtc_state->base.connector_mask & + drm_connector_mask(connector))) + continue; + + encoder = intel_attached_encoder(connector); if (encoder->type != INTEL_OUTPUT_EDP) continue; @@ -4668,13 +4525,18 @@ static int i915_drrs_ctl_set(void *data, u64 val) intel_dp = enc_to_intel_dp(&encoder->base); if (val) intel_edp_drrs_enable(intel_dp, - intel_crtc->config); + crtc_state); else intel_edp_drrs_disable(intel_dp, - intel_crtc->config); + crtc_state); } + drm_connector_list_iter_end(&conn_iter); + +out: + drm_modeset_unlock(&crtc->base.mutex); + if (ret) + return ret; } - drm_modeset_unlock_all(dev); return 0; } @@ -4772,7 +4634,6 @@ static const struct drm_info_list i915_debugfs_list[] = { {"i915_context_status", i915_context_status, 0}, {"i915_forcewake_domains", i915_forcewake_domains, 0}, {"i915_swizzle_info", i915_swizzle_info, 0}, - {"i915_ppgtt_info", i915_ppgtt_info, 0}, {"i915_llc", i915_llc, 0}, {"i915_edp_psr_status", i915_edp_psr_status, 0}, {"i915_energy_uJ", i915_energy_uJ, 0}, @@ -4799,15 +4660,12 @@ static const struct i915_debugfs_files { } i915_debugfs_files[] = { {"i915_wedged", &i915_wedged_fops}, {"i915_cache_sharing", &i915_cache_sharing_fops}, - {"i915_ring_missed_irq", &i915_ring_missed_irq_fops}, - {"i915_ring_test_irq", &i915_ring_test_irq_fops}, {"i915_gem_drop_caches", &i915_drop_caches_fops}, #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) {"i915_error_state", &i915_error_state_fops}, {"i915_gpu_info", &i915_gpu_info_fops}, #endif {"i915_fifo_underrun_reset", &i915_fifo_underrun_reset_ops}, - {"i915_next_seqno", &i915_next_seqno_fops}, {"i915_pri_wm_latency", &i915_pri_wm_latency_fops}, {"i915_spr_wm_latency", &i915_spr_wm_latency_fops}, {"i915_cur_wm_latency", &i915_cur_wm_latency_fops}, @@ -4818,6 +4676,7 @@ static const struct i915_debugfs_files { {"i915_guc_log_level", &i915_guc_log_level_fops}, {"i915_guc_log_relay", &i915_guc_log_relay_fops}, {"i915_hpd_storm_ctl", &i915_hpd_storm_ctl_fops}, + {"i915_hpd_short_storm_ctl", &i915_hpd_short_storm_ctl_fops}, {"i915_ipc_status", &i915_ipc_status_fops}, {"i915_drrs_ctl", &i915_drrs_ctl_fops}, {"i915_edp_psr_debug", &i915_edp_psr_debug_fops} @@ -4879,7 +4738,7 @@ static int i915_dpcd_show(struct seq_file *m, void *data) struct drm_connector *connector = m->private; struct intel_dp *intel_dp = enc_to_intel_dp(&intel_attached_encoder(connector)->base); - uint8_t buf[16]; + u8 buf[16]; ssize_t err; int i; @@ -4899,13 +4758,10 @@ static int i915_dpcd_show(struct seq_file *m, void *data) continue; err = drm_dp_dpcd_read(&intel_dp->aux, b->offset, buf, size); - if (err <= 0) { - DRM_ERROR("dpcd read (%zu bytes at %u) failed (%zd)\n", - size, b->offset, err); - continue; - } - - seq_printf(m, "%04x: %*ph\n", b->offset, (int) size, buf); + if (err < 0) + seq_printf(m, "%04x: ERROR %d\n", b->offset, (int)err); + else + seq_printf(m, "%04x: %*ph\n", b->offset, (int)err, buf); } return 0; @@ -4934,6 +4790,130 @@ static int i915_panel_show(struct seq_file *m, void *data) } DEFINE_SHOW_ATTRIBUTE(i915_panel); +static int i915_hdcp_sink_capability_show(struct seq_file *m, void *data) +{ + struct drm_connector *connector = m->private; + struct intel_connector *intel_connector = to_intel_connector(connector); + + if (connector->status != connector_status_connected) + return -ENODEV; + + /* HDCP is supported by connector */ + if (!intel_connector->hdcp.shim) + return -EINVAL; + + seq_printf(m, "%s:%d HDCP version: ", connector->name, + connector->base.id); + seq_printf(m, "%s ", !intel_hdcp_capable(intel_connector) ? + "None" : "HDCP1.4"); + seq_puts(m, "\n"); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(i915_hdcp_sink_capability); + +static int i915_dsc_fec_support_show(struct seq_file *m, void *data) +{ + struct drm_connector *connector = m->private; + struct drm_device *dev = connector->dev; + struct drm_crtc *crtc; + struct intel_dp *intel_dp; + struct drm_modeset_acquire_ctx ctx; + struct intel_crtc_state *crtc_state = NULL; + int ret = 0; + bool try_again = false; + + drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE); + + do { + try_again = false; + ret = drm_modeset_lock(&dev->mode_config.connection_mutex, + &ctx); + if (ret) { + if (ret == -EDEADLK && !drm_modeset_backoff(&ctx)) { + try_again = true; + continue; + } + break; + } + crtc = connector->state->crtc; + if (connector->status != connector_status_connected || !crtc) { + ret = -ENODEV; + break; + } + ret = drm_modeset_lock(&crtc->mutex, &ctx); + if (ret == -EDEADLK) { + ret = drm_modeset_backoff(&ctx); + if (!ret) { + try_again = true; + continue; + } + break; + } else if (ret) { + break; + } + intel_dp = enc_to_intel_dp(&intel_attached_encoder(connector)->base); + crtc_state = to_intel_crtc_state(crtc->state); + seq_printf(m, "DSC_Enabled: %s\n", + yesno(crtc_state->dsc_params.compression_enable)); + seq_printf(m, "DSC_Sink_Support: %s\n", + yesno(drm_dp_sink_supports_dsc(intel_dp->dsc_dpcd))); + if (!intel_dp_is_edp(intel_dp)) + seq_printf(m, "FEC_Sink_Support: %s\n", + yesno(drm_dp_sink_supports_fec(intel_dp->fec_capable))); + } while (try_again); + + drm_modeset_drop_locks(&ctx); + drm_modeset_acquire_fini(&ctx); + + return ret; +} + +static ssize_t i915_dsc_fec_support_write(struct file *file, + const char __user *ubuf, + size_t len, loff_t *offp) +{ + bool dsc_enable = false; + int ret; + struct drm_connector *connector = + ((struct seq_file *)file->private_data)->private; + struct intel_encoder *encoder = intel_attached_encoder(connector); + struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); + + if (len == 0) + return 0; + + DRM_DEBUG_DRIVER("Copied %zu bytes from user to force DSC\n", + len); + + ret = kstrtobool_from_user(ubuf, len, &dsc_enable); + if (ret < 0) + return ret; + + DRM_DEBUG_DRIVER("Got %s for DSC Enable\n", + (dsc_enable) ? "true" : "false"); + intel_dp->force_dsc_en = dsc_enable; + + *offp += len; + return len; +} + +static int i915_dsc_fec_support_open(struct inode *inode, + struct file *file) +{ + return single_open(file, i915_dsc_fec_support_show, + inode->i_private); +} + +static const struct file_operations i915_dsc_fec_support_fops = { + .owner = THIS_MODULE, + .open = i915_dsc_fec_support_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, + .write = i915_dsc_fec_support_write +}; + /** * i915_debugfs_connector_add - add i915 specific connector debugfs files * @connector: pointer to a registered drm_connector @@ -4946,6 +4926,7 @@ DEFINE_SHOW_ATTRIBUTE(i915_panel); int i915_debugfs_connector_add(struct drm_connector *connector) { struct dentry *root = connector->debugfs_entry; + struct drm_i915_private *dev_priv = to_i915(connector->dev); /* The connector must have been registered beforehands. */ if (!root) @@ -4963,5 +4944,18 @@ int i915_debugfs_connector_add(struct drm_connector *connector) connector, &i915_psr_sink_status_fops); } + if (connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort || + connector->connector_type == DRM_MODE_CONNECTOR_HDMIA || + connector->connector_type == DRM_MODE_CONNECTOR_HDMIB) { + debugfs_create_file("i915_hdcp_sink_capability", S_IRUGO, root, + connector, &i915_hdcp_sink_capability_fops); + } + + if (INTEL_GEN(dev_priv) >= 10 && + (connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort || + connector->connector_type == DRM_MODE_CONNECTOR_eDP)) + debugfs_create_file("i915_dsc_fec_support", S_IRUGO, root, + connector, &i915_dsc_fec_support_fops); + return 0; } diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index 47062ee979cf..9df65d386d11 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -41,18 +41,21 @@ #include <linux/vt.h> #include <acpi/video.h> -#include <drm/drmP.h> -#include <drm/drm_crtc_helper.h> #include <drm/drm_atomic_helper.h> +#include <drm/drm_ioctl.h> +#include <drm/drm_irq.h> +#include <drm/drm_probe_helper.h> #include <drm/i915_drm.h> #include "i915_drv.h" #include "i915_trace.h" #include "i915_pmu.h" +#include "i915_reset.h" #include "i915_query.h" #include "i915_vgpu.h" #include "intel_drv.h" #include "intel_uc.h" +#include "intel_workarounds.h" static struct drm_driver driver; @@ -131,15 +134,15 @@ intel_pch_type(const struct drm_i915_private *dev_priv, unsigned short id) switch (id) { case INTEL_PCH_IBX_DEVICE_ID_TYPE: DRM_DEBUG_KMS("Found Ibex Peak PCH\n"); - WARN_ON(!IS_GEN5(dev_priv)); + WARN_ON(!IS_GEN(dev_priv, 5)); return PCH_IBX; case INTEL_PCH_CPT_DEVICE_ID_TYPE: DRM_DEBUG_KMS("Found CougarPoint PCH\n"); - WARN_ON(!IS_GEN6(dev_priv) && !IS_IVYBRIDGE(dev_priv)); + WARN_ON(!IS_GEN(dev_priv, 6) && !IS_IVYBRIDGE(dev_priv)); return PCH_CPT; case INTEL_PCH_PPT_DEVICE_ID_TYPE: DRM_DEBUG_KMS("Found PantherPoint PCH\n"); - WARN_ON(!IS_GEN6(dev_priv) && !IS_IVYBRIDGE(dev_priv)); + WARN_ON(!IS_GEN(dev_priv, 6) && !IS_IVYBRIDGE(dev_priv)); /* PantherPoint is CPT compatible */ return PCH_CPT; case INTEL_PCH_LPT_DEVICE_ID_TYPE: @@ -216,9 +219,9 @@ intel_virt_detect_pch(const struct drm_i915_private *dev_priv) * make an educated guess as to which PCH is really there. */ - if (IS_GEN5(dev_priv)) + if (IS_GEN(dev_priv, 5)) id = INTEL_PCH_IBX_DEVICE_ID_TYPE; - else if (IS_GEN6(dev_priv) || IS_IVYBRIDGE(dev_priv)) + else if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv)) id = INTEL_PCH_CPT_DEVICE_ID_TYPE; else if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv)) id = INTEL_PCH_LPT_LP_DEVICE_ID_TYPE; @@ -287,7 +290,7 @@ static void intel_detect_pch(struct drm_i915_private *dev_priv) * Use PCH_NOP (PCH but no South Display) for PCH platforms without * display. */ - if (pch && INTEL_INFO(dev_priv)->num_pipes == 0) { + if (pch && !HAS_DISPLAY(dev_priv)) { DRM_DEBUG_KMS("Display disabled, reverting to NOP PCH\n"); dev_priv->pch_type = PCH_NOP; dev_priv->pch_id = 0; @@ -345,10 +348,10 @@ static int i915_getparam_ioctl(struct drm_device *dev, void *data, value = HAS_WT(dev_priv); break; case I915_PARAM_HAS_ALIASING_PPGTT: - value = USES_PPGTT(dev_priv); + value = min_t(int, INTEL_PPGTT(dev_priv), I915_GEM_PPGTT_FULL); break; case I915_PARAM_HAS_SEMAPHORES: - value = HAS_LEGACY_SEMAPHORES(dev_priv); + value = 0; break; case I915_PARAM_HAS_SECURE_BATCHES: value = capable(CAP_SYS_ADMIN); @@ -357,12 +360,12 @@ static int i915_getparam_ioctl(struct drm_device *dev, void *data, value = i915_cmd_parser_get_version(dev_priv); break; case I915_PARAM_SUBSLICE_TOTAL: - value = sseu_subslice_total(&INTEL_INFO(dev_priv)->sseu); + value = sseu_subslice_total(&RUNTIME_INFO(dev_priv)->sseu); if (!value) return -ENODEV; break; case I915_PARAM_EU_TOTAL: - value = INTEL_INFO(dev_priv)->sseu.eu_total; + value = RUNTIME_INFO(dev_priv)->sseu.eu_total; if (!value) return -ENODEV; break; @@ -379,7 +382,7 @@ static int i915_getparam_ioctl(struct drm_device *dev, void *data, value = HAS_POOLED_EU(dev_priv); break; case I915_PARAM_MIN_EU_IN_POOL: - value = INTEL_INFO(dev_priv)->sseu.min_eu_in_pool; + value = RUNTIME_INFO(dev_priv)->sseu.min_eu_in_pool; break; case I915_PARAM_HUC_STATUS: value = intel_huc_check_status(&dev_priv->huc); @@ -429,17 +432,17 @@ static int i915_getparam_ioctl(struct drm_device *dev, void *data, value = intel_engines_has_context_isolation(dev_priv); break; case I915_PARAM_SLICE_MASK: - value = INTEL_INFO(dev_priv)->sseu.slice_mask; + value = RUNTIME_INFO(dev_priv)->sseu.slice_mask; if (!value) return -ENODEV; break; case I915_PARAM_SUBSLICE_MASK: - value = INTEL_INFO(dev_priv)->sseu.subslice_mask[0]; + value = RUNTIME_INFO(dev_priv)->sseu.subslice_mask[0]; if (!value) return -ENODEV; break; case I915_PARAM_CS_TIMESTAMP_FREQUENCY: - value = 1000 * INTEL_INFO(dev_priv)->cs_timestamp_frequency_khz; + value = 1000 * RUNTIME_INFO(dev_priv)->cs_timestamp_frequency_khz; break; case I915_PARAM_MMAP_GTT_COHERENT: value = INTEL_INFO(dev_priv)->has_coherent_ggtt; @@ -645,6 +648,13 @@ static int i915_load_modeset_init(struct drm_device *dev) if (i915_inject_load_failure()) return -ENODEV; + if (HAS_DISPLAY(dev_priv)) { + ret = drm_vblank_init(&dev_priv->drm, + INTEL_INFO(dev_priv)->num_pipes); + if (ret) + goto out; + } + intel_bios_init(dev_priv); /* If we have > 1 VGA cards, then we need to arbitrate access @@ -687,9 +697,9 @@ static int i915_load_modeset_init(struct drm_device *dev) if (ret) goto cleanup_modeset; - intel_setup_overlay(dev_priv); + intel_overlay_setup(dev_priv); - if (INTEL_INFO(dev_priv)->num_pipes == 0) + if (!HAS_DISPLAY(dev_priv)) return 0; ret = intel_fbdev_init(dev); @@ -699,6 +709,8 @@ static int i915_load_modeset_init(struct drm_device *dev) /* Only enable hotplug handling once the fbdev is fully set up. */ intel_hpd_init(dev_priv); + intel_init_ipc(dev_priv); + return 0; cleanup_gem: @@ -745,39 +757,6 @@ static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv) return ret; } -#if !defined(CONFIG_VGA_CONSOLE) -static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv) -{ - return 0; -} -#elif !defined(CONFIG_DUMMY_CONSOLE) -static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv) -{ - return -ENODEV; -} -#else -static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv) -{ - int ret = 0; - - DRM_INFO("Replacing VGA console driver\n"); - - console_lock(); - if (con_is_bound(&vga_con)) - ret = do_take_over_console(&dummy_con, 0, MAX_NR_CONSOLES - 1, 1); - if (ret == 0) { - ret = do_unregister_con_driver(&vga_con); - - /* Ignore "already unregistered". */ - if (ret == -ENODEV) - ret = 0; - } - console_unlock(); - - return ret; -} -#endif - static void intel_init_dpio(struct drm_i915_private *dev_priv) { /* @@ -859,6 +838,7 @@ static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv) pre |= IS_HSW_EARLY_SDV(dev_priv); pre |= IS_SKL_REVID(dev_priv, 0, SKL_REVID_F0); pre |= IS_BXT_REVID(dev_priv, 0, BXT_REVID_B_LAST); + pre |= IS_KBL_REVID(dev_priv, 0, KBL_REVID_A0); if (pre) { DRM_ERROR("This is a pre-production stepping. " @@ -895,6 +875,7 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv) mutex_init(&dev_priv->pps_mutex); i915_memcpy_init_early(dev_priv); + intel_runtime_pm_init_early(dev_priv); ret = i915_workqueues_init(dev_priv); if (ret < 0) @@ -955,7 +936,7 @@ static int i915_mmio_setup(struct drm_i915_private *dev_priv) int mmio_bar; int mmio_size; - mmio_bar = IS_GEN2(dev_priv) ? 1 : 0; + mmio_bar = IS_GEN(dev_priv, 2) ? 1 : 0; /* * Before gen4, the registers and the GTT are behind different BARs. * However, from gen4 onwards, the registers and the GTT are shared @@ -1030,6 +1011,7 @@ static int i915_driver_init_mmio(struct drm_i915_private *dev_priv) err_uncore: intel_uncore_fini(dev_priv); + i915_mmio_cleanup(dev_priv); err_bridge: pci_dev_put(dev_priv->bridge_dev); @@ -1049,17 +1031,6 @@ static void i915_driver_cleanup_mmio(struct drm_i915_private *dev_priv) static void intel_sanitize_options(struct drm_i915_private *dev_priv) { - /* - * i915.enable_ppgtt is read-only, so do an early pass to validate the - * user's requested state against the hardware/driver capabilities. We - * do this now so that we can print out any log messages once rather - * than every time we check intel_enable_ppgtt(). - */ - i915_modparams.enable_ppgtt = - intel_sanitize_enable_ppgtt(dev_priv, - i915_modparams.enable_ppgtt); - DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915_modparams.enable_ppgtt); - intel_gvt_sanitize_options(dev_priv); } @@ -1340,7 +1311,7 @@ intel_get_dram_info(struct drm_i915_private *dev_priv) /* Need to calculate bandwidth only for Gen9 */ if (IS_BROXTON(dev_priv)) ret = bxt_get_dram_info(dev_priv); - else if (INTEL_GEN(dev_priv) == 9) + else if (IS_GEN(dev_priv, 9)) ret = skl_get_dram_info(dev_priv); else ret = skl_dram_get_channels_info(dev_priv); @@ -1373,7 +1344,30 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv) if (i915_inject_load_failure()) return -ENODEV; - intel_device_info_runtime_init(mkwrite_device_info(dev_priv)); + intel_device_info_runtime_init(dev_priv); + + if (HAS_PPGTT(dev_priv)) { + if (intel_vgpu_active(dev_priv) && + !intel_vgpu_has_full_48bit_ppgtt(dev_priv)) { + i915_report_error(dev_priv, + "incompatible vGPU found, support for isolated ppGTT required\n"); + return -ENXIO; + } + } + + if (HAS_EXECLISTS(dev_priv)) { + /* + * Older GVT emulation depends upon intercepting CSB mmio, + * which we no longer use, preferring to use the HWSP cache + * instead. + */ + if (intel_vgpu_active(dev_priv) && + !intel_vgpu_has_hwsp_emulation(dev_priv)) { + i915_report_error(dev_priv, + "old vGPU host found, support for HWSP emulation required\n"); + return -ENXIO; + } + } intel_sanitize_options(dev_priv); @@ -1393,7 +1387,7 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv) goto err_ggtt; } - ret = i915_kick_out_vgacon(dev_priv); + ret = vga_remove_vgacon(pdev); if (ret) { DRM_ERROR("failed to remove conflicting VGA console\n"); goto err_ggtt; @@ -1412,7 +1406,7 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv) pci_set_master(pdev); /* overlay on gen2 is broken and can't address above 1G */ - if (IS_GEN2(dev_priv)) { + if (IS_GEN(dev_priv, 2)) { ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(30)); if (ret) { DRM_ERROR("failed to set DMA mask\n"); @@ -1544,13 +1538,13 @@ static void i915_driver_register(struct drm_i915_private *dev_priv) } else DRM_ERROR("Failed to register driver for userspace access!\n"); - if (INTEL_INFO(dev_priv)->num_pipes) { + if (HAS_DISPLAY(dev_priv)) { /* Must be done after probing outputs */ intel_opregion_register(dev_priv); acpi_video_register(); } - if (IS_GEN5(dev_priv)) + if (IS_GEN(dev_priv, 5)) intel_gpu_ips_init(dev_priv); intel_audio_init(dev_priv); @@ -1568,7 +1562,7 @@ static void i915_driver_register(struct drm_i915_private *dev_priv) * We need to coordinate the hotplugs with the asynchronous fbdev * configuration, for which we use the fbdev->async_cookie. */ - if (INTEL_INFO(dev_priv)->num_pipes) + if (HAS_DISPLAY(dev_priv)) drm_kms_helper_poll_init(dev); intel_power_domains_enable(dev_priv); @@ -1612,8 +1606,14 @@ static void i915_welcome_messages(struct drm_i915_private *dev_priv) if (drm_debug & DRM_UT_DRIVER) { struct drm_printer p = drm_debug_printer("i915 device info:"); - intel_device_info_dump(&dev_priv->info, &p); - intel_device_info_dump_runtime(&dev_priv->info, &p); + drm_printf(&p, "pciid=0x%04x rev=0x%02x platform=%s gen=%i\n", + INTEL_DEVID(dev_priv), + INTEL_REVID(dev_priv), + intel_platform_name(INTEL_INFO(dev_priv)->platform), + INTEL_GEN(dev_priv)); + + intel_device_info_dump_flags(INTEL_INFO(dev_priv), &p); + intel_device_info_dump_runtime(RUNTIME_INFO(dev_priv), &p); } if (IS_ENABLED(CONFIG_DRM_I915_DEBUG)) @@ -1631,14 +1631,16 @@ i915_driver_create(struct pci_dev *pdev, const struct pci_device_id *ent) (struct intel_device_info *)ent->driver_data; struct intel_device_info *device_info; struct drm_i915_private *i915; + int err; i915 = kzalloc(sizeof(*i915), GFP_KERNEL); if (!i915) - return NULL; + return ERR_PTR(-ENOMEM); - if (drm_dev_init(&i915->drm, &driver, &pdev->dev)) { + err = drm_dev_init(&i915->drm, &driver, &pdev->dev); + if (err) { kfree(i915); - return NULL; + return ERR_PTR(err); } i915->drm.pdev = pdev; @@ -1648,11 +1650,11 @@ i915_driver_create(struct pci_dev *pdev, const struct pci_device_id *ent) /* Setup the write-once "constant" device info */ device_info = mkwrite_device_info(i915); memcpy(device_info, match_info, sizeof(*device_info)); - device_info->device_id = pdev->device; + RUNTIME_INFO(i915)->device_id = pdev->device; BUILD_BUG_ON(INTEL_MAX_PLATFORMS > - sizeof(device_info->platform_mask) * BITS_PER_BYTE); - BUG_ON(device_info->gen > sizeof(device_info->gen_mask) * BITS_PER_BYTE); + BITS_PER_TYPE(device_info->platform_mask)); + BUG_ON(device_info->gen > BITS_PER_TYPE(device_info->gen_mask)); return i915; } @@ -1687,8 +1689,8 @@ int i915_driver_load(struct pci_dev *pdev, const struct pci_device_id *ent) int ret; dev_priv = i915_driver_create(pdev, ent); - if (!dev_priv) - return -ENOMEM; + if (IS_ERR(dev_priv)) + return PTR_ERR(dev_priv); /* Disable nuclear pageflip by default on pre-ILK */ if (!i915_modparams.nuclear_pageflip && match_info->gen < 5) @@ -1712,26 +1714,12 @@ int i915_driver_load(struct pci_dev *pdev, const struct pci_device_id *ent) if (ret < 0) goto out_cleanup_mmio; - /* - * TODO: move the vblank init and parts of modeset init steps into one - * of the i915_driver_init_/i915_driver_register functions according - * to the role/effect of the given init step. - */ - if (INTEL_INFO(dev_priv)->num_pipes) { - ret = drm_vblank_init(&dev_priv->drm, - INTEL_INFO(dev_priv)->num_pipes); - if (ret) - goto out_cleanup_hw; - } - ret = i915_load_modeset_init(&dev_priv->drm); if (ret < 0) goto out_cleanup_hw; i915_driver_register(dev_priv); - intel_init_ipc(dev_priv); - enable_rpm_wakeref_asserts(dev_priv); i915_welcome_messages(dev_priv); @@ -1762,6 +1750,9 @@ void i915_driver_unload(struct drm_device *dev) i915_driver_unregister(dev_priv); + /* Flush any external code that still may be under the RCU lock */ + synchronize_rcu(); + if (i915_gem_suspend(dev_priv)) DRM_ERROR("failed to idle hardware; continuing to unload!\n"); @@ -1783,7 +1774,6 @@ void i915_driver_unload(struct drm_device *dev) i915_reset_error_state(dev_priv); i915_gem_fini(dev_priv); - intel_fbc_cleanup_cfb(dev_priv); intel_power_domains_fini_hw(dev_priv); @@ -1791,8 +1781,7 @@ void i915_driver_unload(struct drm_device *dev) i915_driver_cleanup_mmio(dev_priv); enable_rpm_wakeref_asserts(dev_priv); - - WARN_ON(atomic_read(&dev_priv->runtime_pm.wakeref_count)); + intel_runtime_pm_cleanup(dev_priv); } static void i915_driver_release(struct drm_device *dev) @@ -1921,9 +1910,7 @@ static int i915_drm_suspend(struct drm_device *dev) i915_save_state(dev_priv); opregion_target_state = suspend_to_idle(dev_priv) ? PCI_D1 : PCI_D3cold; - intel_opregion_notify_adapter(dev_priv, opregion_target_state); - - intel_opregion_unregister(dev_priv); + intel_opregion_suspend(dev_priv, opregion_target_state); intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED, true); @@ -1964,7 +1951,7 @@ static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation) get_suspend_mode(dev_priv, hibernation)); ret = 0; - if (IS_GEN9_LP(dev_priv)) + if (INTEL_GEN(dev_priv) >= 11 || IS_GEN9_LP(dev_priv)) bxt_enable_dc9(dev_priv); else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) hsw_enable_pc8(dev_priv); @@ -1996,6 +1983,8 @@ static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation) out: enable_rpm_wakeref_asserts(dev_priv); + if (!dev_priv->uncore.user_forcewake.count) + intel_runtime_pm_cleanup(dev_priv); return ret; } @@ -2042,7 +2031,6 @@ static int i915_drm_resume(struct drm_device *dev) i915_restore_state(dev_priv); intel_pps_unlock_regs_wa(dev_priv); - intel_opregion_setup(dev_priv); intel_init_pch_refclk(dev_priv); @@ -2084,12 +2072,10 @@ static int i915_drm_resume(struct drm_device *dev) * */ intel_hpd_init(dev_priv); - intel_opregion_register(dev_priv); + intel_opregion_resume(dev_priv); intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING, false); - intel_opregion_notify_adapter(dev_priv, PCI_D0); - intel_power_domains_enable(dev_priv); enable_rpm_wakeref_asserts(dev_priv); @@ -2157,7 +2143,7 @@ static int i915_drm_resume_early(struct drm_device *dev) intel_uncore_resume_early(dev_priv); - if (IS_GEN9_LP(dev_priv)) { + if (INTEL_GEN(dev_priv) >= 11 || IS_GEN9_LP(dev_priv)) { gen9_sanitize_dc_state(dev_priv); bxt_disable_dc9(dev_priv); } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { @@ -2168,7 +2154,7 @@ static int i915_drm_resume_early(struct drm_device *dev) intel_power_domains_resume(dev_priv); - intel_engines_sanitize(dev_priv); + intel_engines_sanitize(dev_priv, true); enable_rpm_wakeref_asserts(dev_priv); @@ -2189,210 +2175,6 @@ static int i915_resume_switcheroo(struct drm_device *dev) return i915_drm_resume(dev); } -/** - * i915_reset - reset chip after a hang - * @i915: #drm_i915_private to reset - * @stalled_mask: mask of the stalled engines with the guilty requests - * @reason: user error message for why we are resetting - * - * Reset the chip. Useful if a hang is detected. Marks the device as wedged - * on failure. - * - * Caller must hold the struct_mutex. - * - * Procedure is fairly simple: - * - reset the chip using the reset reg - * - re-init context state - * - re-init hardware status page - * - re-init ring buffer - * - re-init interrupt state - * - re-init display - */ -void i915_reset(struct drm_i915_private *i915, - unsigned int stalled_mask, - const char *reason) -{ - struct i915_gpu_error *error = &i915->gpu_error; - int ret; - int i; - - GEM_TRACE("flags=%lx\n", error->flags); - - might_sleep(); - lockdep_assert_held(&i915->drm.struct_mutex); - GEM_BUG_ON(!test_bit(I915_RESET_BACKOFF, &error->flags)); - - if (!test_bit(I915_RESET_HANDOFF, &error->flags)) - return; - - /* Clear any previous failed attempts at recovery. Time to try again. */ - if (!i915_gem_unset_wedged(i915)) - goto wakeup; - - if (reason) - dev_notice(i915->drm.dev, "Resetting chip for %s\n", reason); - error->reset_count++; - - ret = i915_gem_reset_prepare(i915); - if (ret) { - dev_err(i915->drm.dev, "GPU recovery failed\n"); - goto taint; - } - - if (!intel_has_gpu_reset(i915)) { - if (i915_modparams.reset) - dev_err(i915->drm.dev, "GPU reset not supported\n"); - else - DRM_DEBUG_DRIVER("GPU reset disabled\n"); - goto error; - } - - for (i = 0; i < 3; i++) { - ret = intel_gpu_reset(i915, ALL_ENGINES); - if (ret == 0) - break; - - msleep(100); - } - if (ret) { - dev_err(i915->drm.dev, "Failed to reset chip\n"); - goto taint; - } - - /* Ok, now get things going again... */ - - /* - * Everything depends on having the GTT running, so we need to start - * there. - */ - ret = i915_ggtt_enable_hw(i915); - if (ret) { - DRM_ERROR("Failed to re-enable GGTT following reset (%d)\n", - ret); - goto error; - } - - i915_gem_reset(i915, stalled_mask); - intel_overlay_reset(i915); - - /* - * Next we need to restore the context, but we don't use those - * yet either... - * - * Ring buffer needs to be re-initialized in the KMS case, or if X - * was running at the time of the reset (i.e. we weren't VT - * switched away). - */ - ret = i915_gem_init_hw(i915); - if (ret) { - DRM_ERROR("Failed to initialise HW following reset (%d)\n", - ret); - goto error; - } - - i915_queue_hangcheck(i915); - -finish: - i915_gem_reset_finish(i915); -wakeup: - clear_bit(I915_RESET_HANDOFF, &error->flags); - wake_up_bit(&error->flags, I915_RESET_HANDOFF); - return; - -taint: - /* - * History tells us that if we cannot reset the GPU now, we - * never will. This then impacts everything that is run - * subsequently. On failing the reset, we mark the driver - * as wedged, preventing further execution on the GPU. - * We also want to go one step further and add a taint to the - * kernel so that any subsequent faults can be traced back to - * this failure. This is important for CI, where if the - * GPU/driver fails we would like to reboot and restart testing - * rather than continue on into oblivion. For everyone else, - * the system should still plod along, but they have been warned! - */ - add_taint(TAINT_WARN, LOCKDEP_STILL_OK); -error: - i915_gem_set_wedged(i915); - i915_retire_requests(i915); - goto finish; -} - -static inline int intel_gt_reset_engine(struct drm_i915_private *dev_priv, - struct intel_engine_cs *engine) -{ - return intel_gpu_reset(dev_priv, intel_engine_flag(engine)); -} - -/** - * i915_reset_engine - reset GPU engine to recover from a hang - * @engine: engine to reset - * @msg: reason for GPU reset; or NULL for no dev_notice() - * - * Reset a specific GPU engine. Useful if a hang is detected. - * Returns zero on successful reset or otherwise an error code. - * - * Procedure is: - * - identifies the request that caused the hang and it is dropped - * - reset engine (which will force the engine to idle) - * - re-init/configure engine - */ -int i915_reset_engine(struct intel_engine_cs *engine, const char *msg) -{ - struct i915_gpu_error *error = &engine->i915->gpu_error; - struct i915_request *active_request; - int ret; - - GEM_TRACE("%s flags=%lx\n", engine->name, error->flags); - GEM_BUG_ON(!test_bit(I915_RESET_ENGINE + engine->id, &error->flags)); - - active_request = i915_gem_reset_prepare_engine(engine); - if (IS_ERR_OR_NULL(active_request)) { - /* Either the previous reset failed, or we pardon the reset. */ - ret = PTR_ERR(active_request); - goto out; - } - - if (msg) - dev_notice(engine->i915->drm.dev, - "Resetting %s for %s\n", engine->name, msg); - error->reset_engine_count[engine->id]++; - - if (!engine->i915->guc.execbuf_client) - ret = intel_gt_reset_engine(engine->i915, engine); - else - ret = intel_guc_reset_engine(&engine->i915->guc, engine); - if (ret) { - /* If we fail here, we expect to fallback to a global reset */ - DRM_DEBUG_DRIVER("%sFailed to reset %s, ret=%d\n", - engine->i915->guc.execbuf_client ? "GuC " : "", - engine->name, ret); - goto out; - } - - /* - * The request that caused the hang is stuck on elsp, we know the - * active request and can drop it, adjust head to skip the offending - * request to resume executing remaining requests in the queue. - */ - i915_gem_reset_engine(engine, active_request, true); - - /* - * The engine and its registers (and workarounds in case of render) - * have been reset to their default values. Follow the init_ring - * process to program RING_MODE, HWSP and re-enable submission. - */ - ret = engine->init_hw(engine); - if (ret) - goto out; - -out: - intel_engine_cancel_stop_cs(engine); - i915_gem_reset_finish_engine(engine); - return ret; -} - static int i915_pm_prepare(struct device *kdev) { struct pci_dev *pdev = to_pci_dev(kdev); @@ -2730,6 +2512,10 @@ static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *dev_priv) static int vlv_wait_for_pw_status(struct drm_i915_private *dev_priv, u32 mask, u32 val) { + i915_reg_t reg = VLV_GTLC_PW_STATUS; + u32 reg_value; + int ret; + /* The HW does not like us polling for PW_STATUS frequently, so * use the sleeping loop rather than risk the busy spin within * intel_wait_for_register(). @@ -2737,8 +2523,12 @@ static int vlv_wait_for_pw_status(struct drm_i915_private *dev_priv, * Transitioning between RC6 states should be at most 2ms (see * valleyview_enable_rps) so use a 3ms timeout. */ - return wait_for((I915_READ_NOTRACE(VLV_GTLC_PW_STATUS) & mask) == val, - 3); + ret = wait_for(((reg_value = I915_READ_NOTRACE(reg)) & mask) == val, 3); + + /* just trace the final value */ + trace_i915_reg_rw(false, reg, reg_value, sizeof(reg_value), true); + + return ret; } int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on) @@ -2924,7 +2714,10 @@ static int intel_runtime_suspend(struct device *kdev) intel_uncore_suspend(dev_priv); ret = 0; - if (IS_GEN9_LP(dev_priv)) { + if (INTEL_GEN(dev_priv) >= 11) { + icl_display_core_uninit(dev_priv); + bxt_enable_dc9(dev_priv); + } else if (IS_GEN9_LP(dev_priv)) { bxt_display_core_uninit(dev_priv); bxt_enable_dc9(dev_priv); } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { @@ -2950,7 +2743,7 @@ static int intel_runtime_suspend(struct device *kdev) } enable_rpm_wakeref_asserts(dev_priv); - WARN_ON_ONCE(atomic_read(&dev_priv->runtime_pm.wakeref_count)); + intel_runtime_pm_cleanup(dev_priv); if (intel_uncore_arm_unclaimed_mmio_detection(dev_priv)) DRM_ERROR("Unclaimed access detected prior to suspending\n"); @@ -3009,7 +2802,18 @@ static int intel_runtime_resume(struct device *kdev) if (intel_uncore_unclaimed_mmio(dev_priv)) DRM_DEBUG_DRIVER("Unclaimed access during suspend, bios?\n"); - if (IS_GEN9_LP(dev_priv)) { + if (INTEL_GEN(dev_priv) >= 11) { + bxt_disable_dc9(dev_priv); + icl_display_core_init(dev_priv, true); + if (dev_priv->csr.dmc_payload) { + if (dev_priv->csr.allowed_dc_mask & + DC_STATE_EN_UPTO_DC6) + skl_enable_dc6(dev_priv); + else if (dev_priv->csr.allowed_dc_mask & + DC_STATE_EN_UPTO_DC5) + gen9_enable_dc5(dev_priv); + } + } else if (IS_GEN9_LP(dev_priv)) { bxt_disable_dc9(dev_priv); bxt_display_core_init(dev_priv, true); if (dev_priv->csr.dmc_payload && @@ -3183,7 +2987,7 @@ static struct drm_driver driver = { * deal with them for Intel hardware. */ .driver_features = - DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM | DRIVER_PRIME | + DRIVER_GEM | DRIVER_PRIME | DRIVER_RENDER | DRIVER_MODESET | DRIVER_ATOMIC | DRIVER_SYNCOBJ, .release = i915_driver_release, .open = i915_driver_open, diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 872a2e159a5f..a67a63b5aa84 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -45,15 +45,18 @@ #include <linux/pm_qos.h> #include <linux/reservation.h> #include <linux/shmem_fs.h> +#include <linux/stackdepot.h> -#include <drm/drmP.h> #include <drm/intel-gtt.h> #include <drm/drm_legacy.h> /* for struct drm_dma_handle */ #include <drm/drm_gem.h> #include <drm/drm_auth.h> #include <drm/drm_cache.h> #include <drm/drm_util.h> +#include <drm/drm_dsc.h> +#include <drm/drm_connector.h> +#include "i915_fixed.h" #include "i915_params.h" #include "i915_reg.h" #include "i915_utils.h" @@ -88,8 +91,8 @@ #define DRIVER_NAME "i915" #define DRIVER_DESC "Intel Graphics" -#define DRIVER_DATE "20180921" -#define DRIVER_TIMESTAMP 1537521997 +#define DRIVER_DATE "20190207" +#define DRIVER_TIMESTAMP 1549572331 /* Use I915_STATE_WARN(x) and I915_STATE_WARN_ON() (rather than WARN() and * WARN_ON()) for hw state sanity checks to check for unexpected conditions @@ -128,143 +131,7 @@ bool i915_error_injected(void); __i915_printk(i915, i915_error_injected() ? KERN_DEBUG : KERN_ERR, \ fmt, ##__VA_ARGS__) -typedef struct { - uint32_t val; -} uint_fixed_16_16_t; - -#define FP_16_16_MAX ({ \ - uint_fixed_16_16_t fp; \ - fp.val = UINT_MAX; \ - fp; \ -}) - -static inline bool is_fixed16_zero(uint_fixed_16_16_t val) -{ - if (val.val == 0) - return true; - return false; -} - -static inline uint_fixed_16_16_t u32_to_fixed16(uint32_t val) -{ - uint_fixed_16_16_t fp; - - WARN_ON(val > U16_MAX); - - fp.val = val << 16; - return fp; -} - -static inline uint32_t fixed16_to_u32_round_up(uint_fixed_16_16_t fp) -{ - return DIV_ROUND_UP(fp.val, 1 << 16); -} - -static inline uint32_t fixed16_to_u32(uint_fixed_16_16_t fp) -{ - return fp.val >> 16; -} - -static inline uint_fixed_16_16_t min_fixed16(uint_fixed_16_16_t min1, - uint_fixed_16_16_t min2) -{ - uint_fixed_16_16_t min; - - min.val = min(min1.val, min2.val); - return min; -} - -static inline uint_fixed_16_16_t max_fixed16(uint_fixed_16_16_t max1, - uint_fixed_16_16_t max2) -{ - uint_fixed_16_16_t max; - - max.val = max(max1.val, max2.val); - return max; -} - -static inline uint_fixed_16_16_t clamp_u64_to_fixed16(uint64_t val) -{ - uint_fixed_16_16_t fp; - WARN_ON(val > U32_MAX); - fp.val = (uint32_t) val; - return fp; -} - -static inline uint32_t div_round_up_fixed16(uint_fixed_16_16_t val, - uint_fixed_16_16_t d) -{ - return DIV_ROUND_UP(val.val, d.val); -} - -static inline uint32_t mul_round_up_u32_fixed16(uint32_t val, - uint_fixed_16_16_t mul) -{ - uint64_t intermediate_val; - - intermediate_val = (uint64_t) val * mul.val; - intermediate_val = DIV_ROUND_UP_ULL(intermediate_val, 1 << 16); - WARN_ON(intermediate_val > U32_MAX); - return (uint32_t) intermediate_val; -} - -static inline uint_fixed_16_16_t mul_fixed16(uint_fixed_16_16_t val, - uint_fixed_16_16_t mul) -{ - uint64_t intermediate_val; - - intermediate_val = (uint64_t) val.val * mul.val; - intermediate_val = intermediate_val >> 16; - return clamp_u64_to_fixed16(intermediate_val); -} - -static inline uint_fixed_16_16_t div_fixed16(uint32_t val, uint32_t d) -{ - uint64_t interm_val; - - interm_val = (uint64_t)val << 16; - interm_val = DIV_ROUND_UP_ULL(interm_val, d); - return clamp_u64_to_fixed16(interm_val); -} - -static inline uint32_t div_round_up_u32_fixed16(uint32_t val, - uint_fixed_16_16_t d) -{ - uint64_t interm_val; - - interm_val = (uint64_t)val << 16; - interm_val = DIV_ROUND_UP_ULL(interm_val, d.val); - WARN_ON(interm_val > U32_MAX); - return (uint32_t) interm_val; -} - -static inline uint_fixed_16_16_t mul_u32_fixed16(uint32_t val, - uint_fixed_16_16_t mul) -{ - uint64_t intermediate_val; - - intermediate_val = (uint64_t) val * mul.val; - return clamp_u64_to_fixed16(intermediate_val); -} - -static inline uint_fixed_16_16_t add_fixed16(uint_fixed_16_16_t add1, - uint_fixed_16_16_t add2) -{ - uint64_t interm_sum; - - interm_sum = (uint64_t) add1.val + add2.val; - return clamp_u64_to_fixed16(interm_sum); -} - -static inline uint_fixed_16_16_t add_fixed16_u32(uint_fixed_16_16_t add1, - uint32_t add2) -{ - uint64_t interm_sum; - uint_fixed_16_16_t interm_add2 = u32_to_fixed16(add2); - - interm_sum = (uint64_t) add1.val + interm_add2.val; - return clamp_u64_to_fixed16(interm_sum); -} +typedef depot_stack_handle_t intel_wakeref_t; enum hpd_pin { HPD_NONE = 0, @@ -284,7 +151,8 @@ enum hpd_pin { #define for_each_hpd_pin(__pin) \ for ((__pin) = (HPD_NONE + 1); (__pin) < HPD_NUM_PINS; (__pin)++) -#define HPD_STORM_DEFAULT_THRESHOLD 5 +/* Threshold == 5 for long IRQs, 50 for short */ +#define HPD_STORM_DEFAULT_THRESHOLD 50 struct i915_hotplug { struct work_struct hotplug_work; @@ -309,6 +177,8 @@ struct i915_hotplug { bool poll_enabled; unsigned int hpd_storm_threshold; + /* Whether or not to count short HPD IRQs in HPD storms */ + u8 hpd_short_storm_enabled; /* * if we get a HPD irq from DP and a HPD irq from non-DP @@ -414,16 +284,14 @@ struct drm_i915_display_funcs { int (*get_fifo_size)(struct drm_i915_private *dev_priv, enum i9xx_plane_id i9xx_plane); int (*compute_pipe_wm)(struct intel_crtc_state *cstate); - int (*compute_intermediate_wm)(struct drm_device *dev, - struct intel_crtc *intel_crtc, - struct intel_crtc_state *newstate); + int (*compute_intermediate_wm)(struct intel_crtc_state *newstate); void (*initial_watermarks)(struct intel_atomic_state *state, struct intel_crtc_state *cstate); void (*atomic_update_watermarks)(struct intel_atomic_state *state, struct intel_crtc_state *cstate); void (*optimize_watermarks)(struct intel_atomic_state *state, struct intel_crtc_state *cstate); - int (*compute_global_watermarks)(struct drm_atomic_state *state); + int (*compute_global_watermarks)(struct intel_atomic_state *state); void (*update_wm)(struct intel_crtc *crtc); int (*modeset_calc_cdclk)(struct drm_atomic_state *state); /* Returns the active state of the crtc, and if the crtc is active, @@ -455,8 +323,20 @@ struct drm_i915_display_funcs { /* display clock increase/decrease */ /* pll clock increase/decrease */ - void (*load_csc_matrix)(struct drm_crtc_state *crtc_state); - void (*load_luts)(struct drm_crtc_state *crtc_state); + /* + * Program double buffered color management registers during + * vblank evasion. The registers should then latch during the + * next vblank start, alongside any other double buffered registers + * involved with the same commit. + */ + void (*color_commit)(const struct intel_crtc_state *crtc_state); + /* + * Load LUTs (and other single buffered color management + * registers). Will (hopefully) be called during the vblank + * following the latching of any double buffered registers + * involved with the same commit. + */ + void (*load_luts)(const struct intel_crtc_state *crtc_state); }; #define CSR_VERSION(major, minor) ((major) << 16 | (minor)) @@ -466,14 +346,17 @@ struct drm_i915_display_funcs { struct intel_csr { struct work_struct work; const char *fw_path; - uint32_t *dmc_payload; - uint32_t dmc_fw_size; - uint32_t version; - uint32_t mmio_count; + u32 required_version; + u32 max_fw_size; /* bytes */ + u32 *dmc_payload; + u32 dmc_fw_size; /* dwords */ + u32 version; + u32 mmio_count; i915_reg_t mmioaddr[8]; - uint32_t mmiodata[8]; - uint32_t dc_state; - uint32_t allowed_dc_mask; + u32 mmiodata[8]; + u32 dc_state; + u32 allowed_dc_mask; + intel_wakeref_t wakeref; }; enum i915_cache_level { @@ -529,7 +412,7 @@ struct intel_fbc { struct { unsigned int mode_flags; - uint32_t hsw_bdw_pixel_rate; + u32 hsw_bdw_pixel_rate; } crtc; struct { @@ -547,6 +430,8 @@ struct intel_fbc { int adjusted_y; int y; + + u16 pixel_blend_mode; } plane; struct { @@ -625,17 +510,20 @@ struct i915_psr { bool sink_support; bool prepared, enabled; struct intel_dp *dp; + enum pipe pipe; bool active; struct work_struct work; unsigned busy_frontbuffer_bits; bool sink_psr2_support; bool link_standby; bool colorimetry_support; - bool alpm; bool psr2_enabled; u8 sink_sync_latency; ktime_t last_entry_attempt; ktime_t last_exit; + bool sink_not_reliable; + bool irq_aux_error; + u16 su_x_granularity; }; enum intel_pch { @@ -683,7 +571,7 @@ struct i915_suspend_saved_registers { u32 saveSWF0[16]; u32 saveSWF1[16]; u32 saveSWF3[3]; - uint64_t saveFENCE[I915_MAX_NUM_FENCES]; + u64 saveFENCE[I915_MAX_NUM_FENCES]; u32 savePCH_PORT_HOTPLUG; u16 saveGCDGMBUS; }; @@ -919,6 +807,11 @@ struct i915_power_well_desc { /* The pw is backing the VGA functionality */ bool has_vga:1; bool has_fuses:1; + /* + * The pw is for an ICL+ TypeC PHY port in + * Thunderbolt mode. + */ + bool is_tc_tbt:1; } hsw; }; const struct i915_power_well_ops *ops; @@ -941,6 +834,8 @@ struct i915_power_domains { bool display_core_suspended; int power_well_count; + intel_wakeref_t wakeref; + struct mutex lock; int domain_use_count[POWER_DOMAIN_NUM]; struct i915_power_well *power_wells; @@ -1023,9 +918,9 @@ struct i915_gem_mm { atomic_t bsd_engine_dispatch_index; /** Bit 6 swizzling required for X tiling */ - uint32_t bit_6_swizzle_x; + u32 bit_6_swizzle_x; /** Bit 6 swizzling required for Y tiling */ - uint32_t bit_6_swizzle_y; + u32 bit_6_swizzle_y; /* accounting, useful for userland debugging */ spinlock_t object_stat_lock; @@ -1043,17 +938,6 @@ struct i915_gem_mm { #define I915_ENGINE_WEDGED_TIMEOUT (60 * HZ) /* Reset but no recovery? */ -#define DP_AUX_A 0x40 -#define DP_AUX_B 0x10 -#define DP_AUX_C 0x20 -#define DP_AUX_D 0x30 -#define DP_AUX_E 0x50 -#define DP_AUX_F 0x60 - -#define DDC_PIN_B 0x05 -#define DDC_PIN_C 0x04 -#define DDC_PIN_D 0x06 - struct ddi_vbt_port_info { int max_tmds_clock; @@ -1063,18 +947,20 @@ struct ddi_vbt_port_info { * populate this field. */ #define HDMI_LEVEL_SHIFT_UNKNOWN 0xff - uint8_t hdmi_level_shift; + u8 hdmi_level_shift; - uint8_t supports_dvi:1; - uint8_t supports_hdmi:1; - uint8_t supports_dp:1; - uint8_t supports_edp:1; + u8 supports_dvi:1; + u8 supports_hdmi:1; + u8 supports_dp:1; + u8 supports_edp:1; + u8 supports_typec_usb:1; + u8 supports_tbt:1; - uint8_t alternate_aux_channel; - uint8_t alternate_ddc_pin; + u8 alternate_aux_channel; + u8 alternate_ddc_pin; - uint8_t dp_boost_level; - uint8_t hdmi_boost_level; + u8 dp_boost_level; + u8 hdmi_boost_level; int dp_max_link_rate; /* 0 for not limited by VBT */ }; @@ -1100,6 +986,7 @@ struct intel_vbt_data { unsigned int panel_type:4; int lvds_ssc_freq; unsigned int bios_lvds_val; /* initial [PCH_]LVDS reg val in VBIOS */ + enum drm_panel_orientation orientation; enum drrs_support_type drrs_type; @@ -1145,6 +1032,7 @@ struct intel_vbt_data { u8 *data; const u8 *sequence[MIPI_SEQ_MAX]; u8 *deassert_seq; /* Used by fixup_mipi_sequences() */ + enum drm_panel_orientation orientation; } dsi; int crt_ddc_pin; @@ -1163,41 +1051,41 @@ enum intel_ddb_partitioning { struct intel_wm_level { bool enable; - uint32_t pri_val; - uint32_t spr_val; - uint32_t cur_val; - uint32_t fbc_val; + u32 pri_val; + u32 spr_val; + u32 cur_val; + u32 fbc_val; }; struct ilk_wm_values { - uint32_t wm_pipe[3]; - uint32_t wm_lp[3]; - uint32_t wm_lp_spr[3]; - uint32_t wm_linetime[3]; + u32 wm_pipe[3]; + u32 wm_lp[3]; + u32 wm_lp_spr[3]; + u32 wm_linetime[3]; bool enable_fbc_wm; enum intel_ddb_partitioning partitioning; }; struct g4x_pipe_wm { - uint16_t plane[I915_MAX_PLANES]; - uint16_t fbc; + u16 plane[I915_MAX_PLANES]; + u16 fbc; }; struct g4x_sr_wm { - uint16_t plane; - uint16_t cursor; - uint16_t fbc; + u16 plane; + u16 cursor; + u16 fbc; }; struct vlv_wm_ddl_values { - uint8_t plane[I915_MAX_PLANES]; + u8 plane[I915_MAX_PLANES]; }; struct vlv_wm_values { struct g4x_pipe_wm pipe[3]; struct g4x_sr_wm sr; struct vlv_wm_ddl_values ddl[3]; - uint8_t level; + u8 level; bool cxsr; }; @@ -1211,10 +1099,10 @@ struct g4x_wm_values { }; struct skl_ddb_entry { - uint16_t start, end; /* in number of blocks, 'end' is exclusive */ + u16 start, end; /* in number of blocks, 'end' is exclusive */ }; -static inline uint16_t skl_ddb_entry_size(const struct skl_ddb_entry *entry) +static inline u16 skl_ddb_entry_size(const struct skl_ddb_entry *entry) { return entry->end - entry->start; } @@ -1229,9 +1117,6 @@ static inline bool skl_ddb_entry_equal(const struct skl_ddb_entry *e1, } struct skl_ddb_allocation { - /* packed/y */ - struct skl_ddb_entry plane[I915_MAX_PIPES][I915_MAX_PLANES]; - struct skl_ddb_entry uv_plane[I915_MAX_PIPES][I915_MAX_PLANES]; u8 enabled_slices; /* GEN11 has configurable 2 slices */ }; @@ -1241,9 +1126,10 @@ struct skl_ddb_values { }; struct skl_wm_level { + u16 min_ddb_alloc; + u16 plane_res_b; + u8 plane_res_l; bool plane_en; - uint16_t plane_res_b; - uint8_t plane_res_l; }; /* Stores plane specific WM parameters */ @@ -1251,15 +1137,15 @@ struct skl_wm_params { bool x_tiled, y_tiled; bool rc_surface; bool is_planar; - uint32_t width; - uint8_t cpp; - uint32_t plane_pixel_rate; - uint32_t y_min_scanlines; - uint32_t plane_bytes_per_line; + u32 width; + u8 cpp; + u32 plane_pixel_rate; + u32 y_min_scanlines; + u32 plane_bytes_per_line; uint_fixed_16_16_t plane_blocks_per_line; uint_fixed_16_16_t y_tile_minimum; - uint32_t linetime_us; - uint32_t dbuf_block_size; + u32 linetime_us; + u32 dbuf_block_size; }; /* @@ -1289,6 +1175,25 @@ struct i915_runtime_pm { atomic_t wakeref_count; bool suspended; bool irqs_enabled; + +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM) + /* + * To aide detection of wakeref leaks and general misuse, we + * track all wakeref holders. With manual markup (i.e. returning + * a cookie to each rpm_get caller which they then supply to their + * paired rpm_put) we can remove corresponding pairs of and keep + * the array trimmed to active wakerefs. + */ + struct intel_runtime_pm_debug { + spinlock_t lock; + + depot_stack_handle_t last_acquire; + depot_stack_handle_t last_release; + + depot_stack_handle_t *owners; + unsigned long count; + } debug; +#endif }; enum intel_pipe_crc_source { @@ -1324,20 +1229,6 @@ struct i915_frontbuffer_tracking { unsigned flip_bits; }; -struct i915_wa_reg { - u32 addr; - u32 value; - /* bitmask representing WA bits */ - u32 mask; -}; - -#define I915_MAX_WA_REGS 16 - -struct i915_workarounds { - struct i915_wa_reg reg[I915_MAX_WA_REGS]; - u32 count; -}; - struct i915_virtual_gpu { bool active; u32 caps; @@ -1459,6 +1350,12 @@ struct i915_perf_stream { struct list_head link; /** + * @wakeref: As we keep the device awake while the perf stream is + * active, we track our runtime pm reference for later release. + */ + intel_wakeref_t wakeref; + + /** * @sample_flags: Flags representing the `DRM_I915_PERF_PROP_SAMPLE_*` * properties given when opening a stream, representing the contents * of a single sample as read() by userspace. @@ -1521,30 +1418,12 @@ struct i915_oa_ops { bool (*is_valid_flex_reg)(struct drm_i915_private *dev_priv, u32 addr); /** - * @init_oa_buffer: Resets the head and tail pointers of the - * circular buffer for periodic OA reports. - * - * Called when first opening a stream for OA metrics, but also may be - * called in response to an OA buffer overflow or other error - * condition. - * - * Note it may be necessary to clear the full OA buffer here as part of - * maintaining the invariable that new reports must be written to - * zeroed memory for us to be able to reliable detect if an expected - * report has not yet landed in memory. (At least on Haswell the OA - * buffer tail pointer is not synchronized with reports being visible - * to the CPU) - */ - void (*init_oa_buffer)(struct drm_i915_private *dev_priv); - - /** * @enable_metric_set: Selects and applies any MUX configuration to set * up the Boolean and Custom (B/C) counters that are part of the * counter reports being sampled. May apply system constraints such as * disabling EU clock gating as required. */ - int (*enable_metric_set)(struct drm_i915_private *dev_priv, - const struct i915_oa_config *oa_config); + int (*enable_metric_set)(struct i915_perf_stream *stream); /** * @disable_metric_set: Remove system constraints associated with using @@ -1555,12 +1434,12 @@ struct i915_oa_ops { /** * @oa_enable: Enable periodic sampling */ - void (*oa_enable)(struct drm_i915_private *dev_priv); + void (*oa_enable)(struct i915_perf_stream *stream); /** * @oa_disable: Disable periodic sampling */ - void (*oa_disable)(struct drm_i915_private *dev_priv); + void (*oa_disable)(struct i915_perf_stream *stream); /** * @read: Copy data from the circular OA buffer into a given userspace @@ -1596,7 +1475,8 @@ struct drm_i915_private { struct kmem_cache *dependencies; struct kmem_cache *priorities; - const struct intel_device_info info; + const struct intel_device_info __info; /* Use INTEL_INFO() to access. */ + struct intel_runtime_info __runtime; /* Use RUNTIME_INFO() to access. */ struct intel_driver_caps caps; /** @@ -1648,14 +1528,14 @@ struct drm_i915_private { * Base address of where the gmbus and gpio blocks are located (either * on PCH or on SoC for platforms without PCH). */ - uint32_t gpio_mmio_base; + u32 gpio_mmio_base; /* MMIO base address for MIPI regs */ - uint32_t mipi_mmio_base; + u32 mipi_mmio_base; - uint32_t psr_mmio_base; + u32 psr_mmio_base; - uint32_t pps_mmio_base; + u32 pps_mmio_base; wait_queue_head_t gmbus_wait_queue; @@ -1805,7 +1685,6 @@ struct drm_i915_private { int dpio_phy_iosf_port[I915_NUM_PHYS_VLV]; - struct i915_workarounds workarounds; struct i915_wa_list gt_wa_list; struct i915_frontbuffer_tracking fb_tracking; @@ -1911,17 +1790,17 @@ struct drm_i915_private { * in 0.5us units for WM1+. */ /* primary */ - uint16_t pri_latency[5]; + u16 pri_latency[5]; /* sprite */ - uint16_t spr_latency[5]; + u16 spr_latency[5]; /* cursor */ - uint16_t cur_latency[5]; + u16 cur_latency[5]; /* * Raw watermark memory latency values * for SKL for all 8 levels * in 1us units. */ - uint16_t skl_latency[8]; + u16 skl_latency[8]; /* current hardware state */ union { @@ -1931,7 +1810,7 @@ struct drm_i915_private { struct g4x_wm_values g4x; }; - uint8_t max_level; + u8 max_level; /* * Should be held around atomic WM register writing; also @@ -2109,12 +1988,18 @@ struct drm_i915_private { void (*resume)(struct drm_i915_private *); void (*cleanup_engine)(struct intel_engine_cs *engine); - struct list_head timelines; + struct i915_gt_timelines { + struct mutex mutex; /* protects list, tainted by GPU */ + struct list_head active_list; + + /* Pack multiple timelines' seqnos into the same page */ + spinlock_t hwsp_lock; + struct list_head hwsp_free_list; + } timelines; struct list_head active_rings; struct list_head closed_vma; u32 active_requests; - u32 request_serial; /** * Is the GPU currently considered idle, or busy executing @@ -2123,7 +2008,7 @@ struct drm_i915_private { * In order to reduce the effect on performance, there * is a slight delay before we do so. */ - bool awake; + intel_wakeref_t awake; /** * The number of times we have woken up. @@ -2326,6 +2211,8 @@ static inline struct scatterlist *__sg_next(struct scatterlist *sg) (((__iter).curr += PAGE_SIZE) >= (__iter).max) ? \ (__iter) = __sgt_iter(__sg_next((__iter).sgp), false), 0 : 0) +bool i915_sg_trim(struct sg_table *orig_st); + static inline unsigned int i915_sg_page_sizes(struct scatterlist *sg) { unsigned int page_sizes; @@ -2356,37 +2243,28 @@ static inline unsigned int i915_sg_segment_size(void) return size; } -static inline const struct intel_device_info * -intel_info(const struct drm_i915_private *dev_priv) -{ - return &dev_priv->info; -} - -#define INTEL_INFO(dev_priv) intel_info((dev_priv)) +#define INTEL_INFO(dev_priv) (&(dev_priv)->__info) +#define RUNTIME_INFO(dev_priv) (&(dev_priv)->__runtime) #define DRIVER_CAPS(dev_priv) (&(dev_priv)->caps) -#define INTEL_GEN(dev_priv) ((dev_priv)->info.gen) -#define INTEL_DEVID(dev_priv) ((dev_priv)->info.device_id) +#define INTEL_GEN(dev_priv) (INTEL_INFO(dev_priv)->gen) +#define INTEL_DEVID(dev_priv) (RUNTIME_INFO(dev_priv)->device_id) #define REVID_FOREVER 0xff #define INTEL_REVID(dev_priv) ((dev_priv)->drm.pdev->revision) -#define GEN_FOREVER (0) - #define INTEL_GEN_MASK(s, e) ( \ BUILD_BUG_ON_ZERO(!__builtin_constant_p(s)) + \ BUILD_BUG_ON_ZERO(!__builtin_constant_p(e)) + \ - GENMASK((e) != GEN_FOREVER ? (e) - 1 : BITS_PER_LONG - 1, \ - (s) != GEN_FOREVER ? (s) - 1 : 0) \ -) + GENMASK((e) - 1, (s) - 1)) -/* - * Returns true if Gen is in inclusive range [Start, End]. - * - * Use GEN_FOREVER for unbound start and or end. - */ -#define IS_GEN(dev_priv, s, e) \ - (!!((dev_priv)->info.gen_mask & INTEL_GEN_MASK((s), (e)))) +/* Returns true if Gen is in inclusive range [Start, End] */ +#define IS_GEN_RANGE(dev_priv, s, e) \ + (!!(INTEL_INFO(dev_priv)->gen_mask & INTEL_GEN_MASK((s), (e)))) + +#define IS_GEN(dev_priv, n) \ + (BUILD_BUG_ON_ZERO(!__builtin_constant_p(n)) + \ + INTEL_INFO(dev_priv)->gen == (n)) /* * Return true if revision is in range [since,until] inclusive. @@ -2396,7 +2274,7 @@ intel_info(const struct drm_i915_private *dev_priv) #define IS_REVID(p, since, until) \ (INTEL_REVID(p) >= (since) && INTEL_REVID(p) <= (until)) -#define IS_PLATFORM(dev_priv, p) ((dev_priv)->info.platform_mask & BIT(p)) +#define IS_PLATFORM(dev_priv, p) (INTEL_INFO(dev_priv)->platform_mask & BIT(p)) #define IS_I830(dev_priv) IS_PLATFORM(dev_priv, INTEL_I830) #define IS_I845G(dev_priv) IS_PLATFORM(dev_priv, INTEL_I845G) @@ -2418,7 +2296,7 @@ intel_info(const struct drm_i915_private *dev_priv) #define IS_IRONLAKE_M(dev_priv) (INTEL_DEVID(dev_priv) == 0x0046) #define IS_IVYBRIDGE(dev_priv) IS_PLATFORM(dev_priv, INTEL_IVYBRIDGE) #define IS_IVB_GT1(dev_priv) (IS_IVYBRIDGE(dev_priv) && \ - (dev_priv)->info.gt == 1) + INTEL_INFO(dev_priv)->gt == 1) #define IS_VALLEYVIEW(dev_priv) IS_PLATFORM(dev_priv, INTEL_VALLEYVIEW) #define IS_CHERRYVIEW(dev_priv) IS_PLATFORM(dev_priv, INTEL_CHERRYVIEW) #define IS_HASWELL(dev_priv) IS_PLATFORM(dev_priv, INTEL_HASWELL) @@ -2430,7 +2308,7 @@ intel_info(const struct drm_i915_private *dev_priv) #define IS_COFFEELAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_COFFEELAKE) #define IS_CANNONLAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_CANNONLAKE) #define IS_ICELAKE(dev_priv) IS_PLATFORM(dev_priv, INTEL_ICELAKE) -#define IS_MOBILE(dev_priv) ((dev_priv)->info.is_mobile) +#define IS_MOBILE(dev_priv) (INTEL_INFO(dev_priv)->is_mobile) #define IS_HSW_EARLY_SDV(dev_priv) (IS_HASWELL(dev_priv) && \ (INTEL_DEVID(dev_priv) & 0xFF00) == 0x0C00) #define IS_BDW_ULT(dev_priv) (IS_BROADWELL(dev_priv) && \ @@ -2441,11 +2319,13 @@ intel_info(const struct drm_i915_private *dev_priv) #define IS_BDW_ULX(dev_priv) (IS_BROADWELL(dev_priv) && \ (INTEL_DEVID(dev_priv) & 0xf) == 0xe) #define IS_BDW_GT3(dev_priv) (IS_BROADWELL(dev_priv) && \ - (dev_priv)->info.gt == 3) + INTEL_INFO(dev_priv)->gt == 3) #define IS_HSW_ULT(dev_priv) (IS_HASWELL(dev_priv) && \ (INTEL_DEVID(dev_priv) & 0xFF00) == 0x0A00) #define IS_HSW_GT3(dev_priv) (IS_HASWELL(dev_priv) && \ - (dev_priv)->info.gt == 3) + INTEL_INFO(dev_priv)->gt == 3) +#define IS_HSW_GT1(dev_priv) (IS_HASWELL(dev_priv) && \ + INTEL_INFO(dev_priv)->gt == 1) /* ULX machines are also considered ULT. */ #define IS_HSW_ULX(dev_priv) (INTEL_DEVID(dev_priv) == 0x0A0E || \ INTEL_DEVID(dev_priv) == 0x0A1E) @@ -2465,24 +2345,29 @@ intel_info(const struct drm_i915_private *dev_priv) #define IS_KBL_ULX(dev_priv) (INTEL_DEVID(dev_priv) == 0x590E || \ INTEL_DEVID(dev_priv) == 0x5915 || \ INTEL_DEVID(dev_priv) == 0x591E) +#define IS_AML_ULX(dev_priv) (INTEL_DEVID(dev_priv) == 0x591C || \ + INTEL_DEVID(dev_priv) == 0x87C0 || \ + INTEL_DEVID(dev_priv) == 0x87CA) #define IS_SKL_GT2(dev_priv) (IS_SKYLAKE(dev_priv) && \ - (dev_priv)->info.gt == 2) + INTEL_INFO(dev_priv)->gt == 2) #define IS_SKL_GT3(dev_priv) (IS_SKYLAKE(dev_priv) && \ - (dev_priv)->info.gt == 3) + INTEL_INFO(dev_priv)->gt == 3) #define IS_SKL_GT4(dev_priv) (IS_SKYLAKE(dev_priv) && \ - (dev_priv)->info.gt == 4) + INTEL_INFO(dev_priv)->gt == 4) #define IS_KBL_GT2(dev_priv) (IS_KABYLAKE(dev_priv) && \ - (dev_priv)->info.gt == 2) + INTEL_INFO(dev_priv)->gt == 2) #define IS_KBL_GT3(dev_priv) (IS_KABYLAKE(dev_priv) && \ - (dev_priv)->info.gt == 3) + INTEL_INFO(dev_priv)->gt == 3) #define IS_CFL_ULT(dev_priv) (IS_COFFEELAKE(dev_priv) && \ (INTEL_DEVID(dev_priv) & 0x00F0) == 0x00A0) #define IS_CFL_GT2(dev_priv) (IS_COFFEELAKE(dev_priv) && \ - (dev_priv)->info.gt == 2) + INTEL_INFO(dev_priv)->gt == 2) #define IS_CFL_GT3(dev_priv) (IS_COFFEELAKE(dev_priv) && \ - (dev_priv)->info.gt == 3) + INTEL_INFO(dev_priv)->gt == 3) #define IS_CNL_WITH_PORT_F(dev_priv) (IS_CANNONLAKE(dev_priv) && \ (INTEL_DEVID(dev_priv) & 0x0004) == 0x0004) +#define IS_ICL_WITH_PORT_F(dev_priv) (IS_ICELAKE(dev_priv) && \ + INTEL_DEVID(dev_priv) != 0x8A51) #define IS_ALPHA_SUPPORT(intel_info) ((intel_info)->is_alpha_support) @@ -2537,26 +2422,9 @@ intel_info(const struct drm_i915_private *dev_priv) #define IS_ICL_REVID(p, since, until) \ (IS_ICELAKE(p) && IS_REVID(p, since, until)) -/* - * The genX designation typically refers to the render engine, so render - * capability related checks should use IS_GEN, while display and other checks - * have their own (e.g. HAS_PCH_SPLIT for ILK+ display, IS_foo for particular - * chips, etc.). - */ -#define IS_GEN2(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(1))) -#define IS_GEN3(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(2))) -#define IS_GEN4(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(3))) -#define IS_GEN5(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(4))) -#define IS_GEN6(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(5))) -#define IS_GEN7(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(6))) -#define IS_GEN8(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(7))) -#define IS_GEN9(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(8))) -#define IS_GEN10(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(9))) -#define IS_GEN11(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(10))) - #define IS_LP(dev_priv) (INTEL_INFO(dev_priv)->is_lp) -#define IS_GEN9_LP(dev_priv) (IS_GEN9(dev_priv) && IS_LP(dev_priv)) -#define IS_GEN9_BC(dev_priv) (IS_GEN9(dev_priv) && !IS_LP(dev_priv)) +#define IS_GEN9_LP(dev_priv) (IS_GEN(dev_priv, 9) && IS_LP(dev_priv)) +#define IS_GEN9_BC(dev_priv) (IS_GEN(dev_priv, 9) && !IS_LP(dev_priv)) #define ENGINE_MASK(id) BIT(id) #define RENDER_RING ENGINE_MASK(RCS) @@ -2570,43 +2438,46 @@ intel_info(const struct drm_i915_private *dev_priv) #define ALL_ENGINES (~0) #define HAS_ENGINE(dev_priv, id) \ - (!!((dev_priv)->info.ring_mask & ENGINE_MASK(id))) + (!!(INTEL_INFO(dev_priv)->ring_mask & ENGINE_MASK(id))) #define HAS_BSD(dev_priv) HAS_ENGINE(dev_priv, VCS) #define HAS_BSD2(dev_priv) HAS_ENGINE(dev_priv, VCS2) #define HAS_BLT(dev_priv) HAS_ENGINE(dev_priv, BCS) #define HAS_VEBOX(dev_priv) HAS_ENGINE(dev_priv, VECS) -#define HAS_LEGACY_SEMAPHORES(dev_priv) IS_GEN7(dev_priv) - -#define HAS_LLC(dev_priv) ((dev_priv)->info.has_llc) -#define HAS_SNOOP(dev_priv) ((dev_priv)->info.has_snoop) +#define HAS_LLC(dev_priv) (INTEL_INFO(dev_priv)->has_llc) +#define HAS_SNOOP(dev_priv) (INTEL_INFO(dev_priv)->has_snoop) #define HAS_EDRAM(dev_priv) (!!((dev_priv)->edram_cap & EDRAM_ENABLED)) #define HAS_WT(dev_priv) ((IS_HASWELL(dev_priv) || \ IS_BROADWELL(dev_priv)) && HAS_EDRAM(dev_priv)) -#define HWS_NEEDS_PHYSICAL(dev_priv) ((dev_priv)->info.hws_needs_physical) +#define HWS_NEEDS_PHYSICAL(dev_priv) (INTEL_INFO(dev_priv)->hws_needs_physical) #define HAS_LOGICAL_RING_CONTEXTS(dev_priv) \ - ((dev_priv)->info.has_logical_ring_contexts) + (INTEL_INFO(dev_priv)->has_logical_ring_contexts) #define HAS_LOGICAL_RING_ELSQ(dev_priv) \ - ((dev_priv)->info.has_logical_ring_elsq) + (INTEL_INFO(dev_priv)->has_logical_ring_elsq) #define HAS_LOGICAL_RING_PREEMPTION(dev_priv) \ - ((dev_priv)->info.has_logical_ring_preemption) + (INTEL_INFO(dev_priv)->has_logical_ring_preemption) #define HAS_EXECLISTS(dev_priv) HAS_LOGICAL_RING_CONTEXTS(dev_priv) -#define USES_PPGTT(dev_priv) (i915_modparams.enable_ppgtt) -#define USES_FULL_PPGTT(dev_priv) (i915_modparams.enable_ppgtt >= 2) -#define USES_FULL_48BIT_PPGTT(dev_priv) (i915_modparams.enable_ppgtt == 3) +#define INTEL_PPGTT(dev_priv) (INTEL_INFO(dev_priv)->ppgtt) +#define HAS_PPGTT(dev_priv) \ + (INTEL_PPGTT(dev_priv) != INTEL_PPGTT_NONE) +#define HAS_FULL_PPGTT(dev_priv) \ + (INTEL_PPGTT(dev_priv) >= INTEL_PPGTT_FULL) +#define HAS_FULL_48BIT_PPGTT(dev_priv) \ + (INTEL_PPGTT(dev_priv) >= INTEL_PPGTT_FULL_4LVL) + #define HAS_PAGE_SIZES(dev_priv, sizes) ({ \ GEM_BUG_ON((sizes) == 0); \ - ((sizes) & ~(dev_priv)->info.page_sizes) == 0; \ + ((sizes) & ~INTEL_INFO(dev_priv)->page_sizes) == 0; \ }) -#define HAS_OVERLAY(dev_priv) ((dev_priv)->info.has_overlay) +#define HAS_OVERLAY(dev_priv) (INTEL_INFO(dev_priv)->display.has_overlay) #define OVERLAY_NEEDS_PHYSICAL(dev_priv) \ - ((dev_priv)->info.overlay_needs_physical) + (INTEL_INFO(dev_priv)->display.overlay_needs_physical) /* Early gen2 have a totally busted CS tlb and require pinned batches. */ #define HAS_BROKEN_CS_TLB(dev_priv) (IS_I830(dev_priv) || IS_I845G(dev_priv)) @@ -2624,42 +2495,42 @@ intel_info(const struct drm_i915_private *dev_priv) /* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte * rows, which changed the alignment requirements and fence programming. */ -#define HAS_128_BYTE_Y_TILING(dev_priv) (!IS_GEN2(dev_priv) && \ +#define HAS_128_BYTE_Y_TILING(dev_priv) (!IS_GEN(dev_priv, 2) && \ !(IS_I915G(dev_priv) || \ IS_I915GM(dev_priv))) -#define SUPPORTS_TV(dev_priv) ((dev_priv)->info.supports_tv) -#define I915_HAS_HOTPLUG(dev_priv) ((dev_priv)->info.has_hotplug) +#define SUPPORTS_TV(dev_priv) (INTEL_INFO(dev_priv)->display.supports_tv) +#define I915_HAS_HOTPLUG(dev_priv) (INTEL_INFO(dev_priv)->display.has_hotplug) #define HAS_FW_BLC(dev_priv) (INTEL_GEN(dev_priv) > 2) -#define HAS_FBC(dev_priv) ((dev_priv)->info.has_fbc) -#define HAS_CUR_FBC(dev_priv) (!HAS_GMCH_DISPLAY(dev_priv) && INTEL_GEN(dev_priv) >= 7) +#define HAS_FBC(dev_priv) (INTEL_INFO(dev_priv)->display.has_fbc) +#define HAS_CUR_FBC(dev_priv) (!HAS_GMCH(dev_priv) && INTEL_GEN(dev_priv) >= 7) #define HAS_IPS(dev_priv) (IS_HSW_ULT(dev_priv) || IS_BROADWELL(dev_priv)) -#define HAS_DP_MST(dev_priv) ((dev_priv)->info.has_dp_mst) +#define HAS_DP_MST(dev_priv) (INTEL_INFO(dev_priv)->display.has_dp_mst) -#define HAS_DDI(dev_priv) ((dev_priv)->info.has_ddi) -#define HAS_FPGA_DBG_UNCLAIMED(dev_priv) ((dev_priv)->info.has_fpga_dbg) -#define HAS_PSR(dev_priv) ((dev_priv)->info.has_psr) +#define HAS_DDI(dev_priv) (INTEL_INFO(dev_priv)->display.has_ddi) +#define HAS_FPGA_DBG_UNCLAIMED(dev_priv) (INTEL_INFO(dev_priv)->has_fpga_dbg) +#define HAS_PSR(dev_priv) (INTEL_INFO(dev_priv)->display.has_psr) -#define HAS_RC6(dev_priv) ((dev_priv)->info.has_rc6) -#define HAS_RC6p(dev_priv) ((dev_priv)->info.has_rc6p) +#define HAS_RC6(dev_priv) (INTEL_INFO(dev_priv)->has_rc6) +#define HAS_RC6p(dev_priv) (INTEL_INFO(dev_priv)->has_rc6p) #define HAS_RC6pp(dev_priv) (false) /* HW was never validated */ -#define HAS_CSR(dev_priv) ((dev_priv)->info.has_csr) +#define HAS_CSR(dev_priv) (INTEL_INFO(dev_priv)->display.has_csr) -#define HAS_RUNTIME_PM(dev_priv) ((dev_priv)->info.has_runtime_pm) -#define HAS_64BIT_RELOC(dev_priv) ((dev_priv)->info.has_64bit_reloc) +#define HAS_RUNTIME_PM(dev_priv) (INTEL_INFO(dev_priv)->has_runtime_pm) +#define HAS_64BIT_RELOC(dev_priv) (INTEL_INFO(dev_priv)->has_64bit_reloc) -#define HAS_IPC(dev_priv) ((dev_priv)->info.has_ipc) +#define HAS_IPC(dev_priv) (INTEL_INFO(dev_priv)->display.has_ipc) /* * For now, anything with a GuC requires uCode loading, and then supports * command submission once loaded. But these are logically independent * properties, so we have separate macros to test them. */ -#define HAS_GUC(dev_priv) ((dev_priv)->info.has_guc) -#define HAS_GUC_CT(dev_priv) ((dev_priv)->info.has_guc_ct) +#define HAS_GUC(dev_priv) (INTEL_INFO(dev_priv)->has_guc) +#define HAS_GUC_CT(dev_priv) (INTEL_INFO(dev_priv)->has_guc_ct) #define HAS_GUC_UCODE(dev_priv) (HAS_GUC(dev_priv)) #define HAS_GUC_SCHED(dev_priv) (HAS_GUC(dev_priv)) @@ -2668,11 +2539,11 @@ intel_info(const struct drm_i915_private *dev_priv) #define HAS_HUC_UCODE(dev_priv) (HAS_GUC(dev_priv)) /* Having a GuC is not the same as using a GuC */ -#define USES_GUC(dev_priv) intel_uc_is_using_guc() -#define USES_GUC_SUBMISSION(dev_priv) intel_uc_is_using_guc_submission() -#define USES_HUC(dev_priv) intel_uc_is_using_huc() +#define USES_GUC(dev_priv) intel_uc_is_using_guc(dev_priv) +#define USES_GUC_SUBMISSION(dev_priv) intel_uc_is_using_guc_submission(dev_priv) +#define USES_HUC(dev_priv) intel_uc_is_using_huc(dev_priv) -#define HAS_POOLED_EU(dev_priv) ((dev_priv)->info.has_pooled_eu) +#define HAS_POOLED_EU(dev_priv) (INTEL_INFO(dev_priv)->has_pooled_eu) #define INTEL_PCH_DEVICE_ID_MASK 0xff80 #define INTEL_PCH_IBX_DEVICE_ID_TYPE 0x3b00 @@ -2712,18 +2583,20 @@ intel_info(const struct drm_i915_private *dev_priv) #define HAS_PCH_NOP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_NOP) #define HAS_PCH_SPLIT(dev_priv) (INTEL_PCH_TYPE(dev_priv) != PCH_NONE) -#define HAS_GMCH_DISPLAY(dev_priv) ((dev_priv)->info.has_gmch_display) +#define HAS_GMCH(dev_priv) (INTEL_INFO(dev_priv)->display.has_gmch) #define HAS_LSPCON(dev_priv) (INTEL_GEN(dev_priv) >= 9) /* DPF == dynamic parity feature */ -#define HAS_L3_DPF(dev_priv) ((dev_priv)->info.has_l3_dpf) +#define HAS_L3_DPF(dev_priv) (INTEL_INFO(dev_priv)->has_l3_dpf) #define NUM_L3_SLICES(dev_priv) (IS_HSW_GT3(dev_priv) ? \ 2 : HAS_L3_DPF(dev_priv)) #define GT_FREQUENCY_MULTIPLIER 50 #define GEN9_FREQ_SCALER 3 +#define HAS_DISPLAY(dev_priv) (INTEL_INFO(dev_priv)->num_pipes > 0) + #include "i915_trace.h" static inline bool intel_vtd_active(void) @@ -2746,9 +2619,6 @@ intel_ggtt_update_needs_vtd_wa(struct drm_i915_private *dev_priv) return IS_BROXTON(dev_priv) && intel_vtd_active(); } -int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv, - int enable_ppgtt); - /* i915_drv.c */ void __printf(3, 4) __i915_printk(struct drm_i915_private *dev_priv, const char *level, @@ -2768,19 +2638,7 @@ extern const struct dev_pm_ops i915_pm_ops; extern int i915_driver_load(struct pci_dev *pdev, const struct pci_device_id *ent); extern void i915_driver_unload(struct drm_device *dev); -extern int intel_gpu_reset(struct drm_i915_private *dev_priv, u32 engine_mask); -extern bool intel_has_gpu_reset(struct drm_i915_private *dev_priv); - -extern void i915_reset(struct drm_i915_private *i915, - unsigned int stalled_mask, - const char *reason); -extern int i915_reset_engine(struct intel_engine_cs *engine, - const char *reason); - -extern bool intel_has_reset_engine(struct drm_i915_private *dev_priv); -extern int intel_reset_guc(struct drm_i915_private *dev_priv); -extern int intel_guc_reset_engine(struct intel_guc *guc, - struct intel_engine_cs *engine); + extern void intel_engine_init_hangcheck(struct intel_engine_cs *engine); extern void intel_hangcheck_init(struct drm_i915_private *dev_priv); extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv); @@ -2823,20 +2681,11 @@ static inline void i915_queue_hangcheck(struct drm_i915_private *dev_priv) &dev_priv->gpu_error.hangcheck_work, delay); } -__printf(4, 5) -void i915_handle_error(struct drm_i915_private *dev_priv, - u32 engine_mask, - unsigned long flags, - const char *fmt, ...); -#define I915_ERROR_CAPTURE BIT(0) - extern void intel_irq_init(struct drm_i915_private *dev_priv); extern void intel_irq_fini(struct drm_i915_private *dev_priv); int intel_irq_install(struct drm_i915_private *dev_priv); void intel_irq_uninstall(struct drm_i915_private *dev_priv); -void i915_clear_error_registers(struct drm_i915_private *dev_priv); - static inline bool intel_gvt_active(struct drm_i915_private *dev_priv) { return dev_priv->gvt; @@ -2860,45 +2709,45 @@ i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv); void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv); void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv, - uint32_t mask, - uint32_t bits); + u32 mask, + u32 bits); void ilk_update_display_irq(struct drm_i915_private *dev_priv, - uint32_t interrupt_mask, - uint32_t enabled_irq_mask); + u32 interrupt_mask, + u32 enabled_irq_mask); static inline void -ilk_enable_display_irq(struct drm_i915_private *dev_priv, uint32_t bits) +ilk_enable_display_irq(struct drm_i915_private *dev_priv, u32 bits) { ilk_update_display_irq(dev_priv, bits, bits); } static inline void -ilk_disable_display_irq(struct drm_i915_private *dev_priv, uint32_t bits) +ilk_disable_display_irq(struct drm_i915_private *dev_priv, u32 bits) { ilk_update_display_irq(dev_priv, bits, 0); } void bdw_update_pipe_irq(struct drm_i915_private *dev_priv, enum pipe pipe, - uint32_t interrupt_mask, - uint32_t enabled_irq_mask); + u32 interrupt_mask, + u32 enabled_irq_mask); static inline void bdw_enable_pipe_irq(struct drm_i915_private *dev_priv, - enum pipe pipe, uint32_t bits) + enum pipe pipe, u32 bits) { bdw_update_pipe_irq(dev_priv, pipe, bits, bits); } static inline void bdw_disable_pipe_irq(struct drm_i915_private *dev_priv, - enum pipe pipe, uint32_t bits) + enum pipe pipe, u32 bits) { bdw_update_pipe_irq(dev_priv, pipe, bits, 0); } void ibx_display_interrupt_update(struct drm_i915_private *dev_priv, - uint32_t interrupt_mask, - uint32_t enabled_irq_mask); + u32 interrupt_mask, + u32 enabled_irq_mask); static inline void -ibx_enable_display_interrupt(struct drm_i915_private *dev_priv, uint32_t bits) +ibx_enable_display_interrupt(struct drm_i915_private *dev_priv, u32 bits) { ibx_display_interrupt_update(dev_priv, bits, bits); } static inline void -ibx_disable_display_interrupt(struct drm_i915_private *dev_priv, uint32_t bits) +ibx_disable_display_interrupt(struct drm_i915_private *dev_priv, u32 bits) { ibx_display_interrupt_update(dev_priv, bits, 0); } @@ -3083,13 +2932,13 @@ i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) __i915_gem_object_unpin_pages(obj); } -enum i915_mm_subclass { /* lockdep subclass for obj->mm.lock */ +enum i915_mm_subclass { /* lockdep subclass for obj->mm.lock/struct_mutex */ I915_MM_NORMAL = 0, - I915_MM_SHRINKER + I915_MM_SHRINKER /* called "recursively" from direct-reclaim-esque */ }; -void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj, - enum i915_mm_subclass subclass); +int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj, + enum i915_mm_subclass subclass); void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj); enum i915_map_type { @@ -3158,7 +3007,7 @@ int i915_gem_dumb_create(struct drm_file *file_priv, struct drm_device *dev, struct drm_mode_create_dumb *args); int i915_gem_mmap_gtt(struct drm_file *file_priv, struct drm_device *dev, - uint32_t handle, uint64_t *offset); + u32 handle, u64 *offset); int i915_gem_mmap_gtt_version(void); void i915_gem_track_fb(struct drm_i915_gem_object *old, @@ -3175,11 +3024,6 @@ static inline bool i915_reset_backoff(struct i915_gpu_error *error) return unlikely(test_bit(I915_RESET_BACKOFF, &error->flags)); } -static inline bool i915_reset_handoff(struct i915_gpu_error *error) -{ - return unlikely(test_bit(I915_RESET_HANDOFF, &error->flags)); -} - static inline bool i915_terminally_wedged(struct i915_gpu_error *error) { return unlikely(test_bit(I915_WEDGED, &error->flags)); @@ -3201,18 +3045,8 @@ static inline u32 i915_reset_engine_count(struct i915_gpu_error *error, return READ_ONCE(error->reset_engine_count[engine->id]); } -struct i915_request * -i915_gem_reset_prepare_engine(struct intel_engine_cs *engine); -int i915_gem_reset_prepare(struct drm_i915_private *dev_priv); -void i915_gem_reset(struct drm_i915_private *dev_priv, - unsigned int stalled_mask); -void i915_gem_reset_finish_engine(struct intel_engine_cs *engine); -void i915_gem_reset_finish(struct drm_i915_private *dev_priv); void i915_gem_set_wedged(struct drm_i915_private *dev_priv); bool i915_gem_unset_wedged(struct drm_i915_private *dev_priv); -void i915_gem_reset_engine(struct intel_engine_cs *engine, - struct i915_request *request, - bool stalled); void i915_gem_init_mmio(struct drm_i915_private *i915); int __must_check i915_gem_init(struct drm_i915_private *dev_priv); @@ -3233,7 +3067,7 @@ int i915_gem_object_wait(struct drm_i915_gem_object *obj, int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj, unsigned int flags, const struct i915_sched_attr *attr); -#define I915_PRIORITY_DISPLAY I915_PRIORITY_MAX +#define I915_PRIORITY_DISPLAY I915_USER_PRIORITY(I915_PRIORITY_MAX) int __must_check i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write); @@ -3309,7 +3143,7 @@ int i915_perf_remove_config_ioctl(struct drm_device *dev, void *data, struct drm_file *file); void i915_oa_init_reg_state(struct intel_engine_cs *engine, struct i915_gem_context *ctx, - uint32_t *reg_state); + u32 *reg_state); /* i915_gem_evict.c */ int __must_check i915_gem_evict_something(struct i915_address_space *vm, @@ -3371,7 +3205,8 @@ unsigned long i915_gem_shrink(struct drm_i915_private *i915, unsigned long i915_gem_shrink_all(struct drm_i915_private *i915); void i915_gem_shrinker_register(struct drm_i915_private *i915); void i915_gem_shrinker_unregister(struct drm_i915_private *i915); -void i915_gem_shrinker_taints_mutex(struct mutex *mutex); +void i915_gem_shrinker_taints_mutex(struct drm_i915_private *i915, + struct mutex *mutex); /* i915_gem_tiling.c */ static inline bool i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) @@ -3465,6 +3300,7 @@ bool intel_bios_is_port_hpd_inverted(struct drm_i915_private *dev_priv, enum port port); bool intel_bios_is_lspcon_present(struct drm_i915_private *dev_priv, enum port port); +enum aux_ch intel_bios_port_aux_ch(struct drm_i915_private *dev_priv, enum port port); /* intel_acpi.c */ #ifdef CONFIG_ACPI @@ -3479,15 +3315,27 @@ static inline void intel_unregister_dsm_handler(void) { return; } static inline struct intel_device_info * mkwrite_device_info(struct drm_i915_private *dev_priv) { - return (struct intel_device_info *)&dev_priv->info; + return (struct intel_device_info *)INTEL_INFO(dev_priv); +} + +static inline struct intel_sseu +intel_device_default_sseu(struct drm_i915_private *i915) +{ + const struct sseu_dev_info *sseu = &RUNTIME_INFO(i915)->sseu; + struct intel_sseu value = { + .slice_mask = sseu->slice_mask, + .subslice_mask = sseu->subslice_mask[0], + .min_eus_per_subslice = sseu->max_eus_per_subslice, + .max_eus_per_subslice = sseu->max_eus_per_subslice, + }; + + return value; } /* modesetting */ extern void intel_modeset_init_hw(struct drm_device *dev); extern int intel_modeset_init(struct drm_device *dev); extern void intel_modeset_cleanup(struct drm_device *dev); -extern int intel_connector_register(struct drm_connector *); -extern void intel_connector_unregister(struct drm_connector *); extern int intel_modeset_vga_set_state(struct drm_i915_private *dev_priv, bool state); extern void intel_display_resume(struct drm_device *dev); @@ -3500,6 +3348,9 @@ extern void intel_rps_mark_interactive(struct drm_i915_private *i915, bool interactive); extern bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable); +void intel_dsc_enable(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state); +void intel_dsc_disable(const struct intel_crtc_state *crtc_state); int i915_reg_read_ioctl(struct drm_device *dev, void *data, struct drm_file *file); @@ -3558,10 +3409,10 @@ bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv, enum dpio_phy phy); bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv, enum dpio_phy phy); -uint8_t bxt_ddi_phy_calc_lane_lat_optim_mask(uint8_t lane_count); +u8 bxt_ddi_phy_calc_lane_lat_optim_mask(u8 lane_count); void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder, - uint8_t lane_lat_optim_mask); -uint8_t bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder); + u8 lane_lat_optim_mask); +u8 bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder); void chv_set_phy_signal_level(struct intel_encoder *encoder, u32 deemph_reg_value, u32 margin_reg_value, @@ -3587,6 +3438,12 @@ void vlv_phy_pre_encoder_enable(struct intel_encoder *encoder, void vlv_phy_reset_lanes(struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state); +/* intel_combo_phy.c */ +void icl_combo_phys_init(struct drm_i915_private *dev_priv); +void icl_combo_phys_uninit(struct drm_i915_private *dev_priv); +void cnl_combo_phys_init(struct drm_i915_private *dev_priv); +void cnl_combo_phys_uninit(struct drm_i915_private *dev_priv); + int intel_gpu_freq(struct drm_i915_private *dev_priv, int val); int intel_freq_opcode(struct drm_i915_private *dev_priv, int val); u64 intel_rc6_residency_ns(struct drm_i915_private *dev_priv, @@ -3758,90 +3615,6 @@ wait_remaining_ms_from_jiffies(unsigned long timestamp_jiffies, int to_wait_ms) } } -static inline bool -__i915_request_irq_complete(const struct i915_request *rq) -{ - struct intel_engine_cs *engine = rq->engine; - u32 seqno; - - /* Note that the engine may have wrapped around the seqno, and - * so our request->global_seqno will be ahead of the hardware, - * even though it completed the request before wrapping. We catch - * this by kicking all the waiters before resetting the seqno - * in hardware, and also signal the fence. - */ - if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags)) - return true; - - /* The request was dequeued before we were awoken. We check after - * inspecting the hw to confirm that this was the same request - * that generated the HWS update. The memory barriers within - * the request execution are sufficient to ensure that a check - * after reading the value from hw matches this request. - */ - seqno = i915_request_global_seqno(rq); - if (!seqno) - return false; - - /* Before we do the heavier coherent read of the seqno, - * check the value (hopefully) in the CPU cacheline. - */ - if (__i915_request_completed(rq, seqno)) - return true; - - /* Ensure our read of the seqno is coherent so that we - * do not "miss an interrupt" (i.e. if this is the last - * request and the seqno write from the GPU is not visible - * by the time the interrupt fires, we will see that the - * request is incomplete and go back to sleep awaiting - * another interrupt that will never come.) - * - * Strictly, we only need to do this once after an interrupt, - * but it is easier and safer to do it every time the waiter - * is woken. - */ - if (engine->irq_seqno_barrier && - test_and_clear_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted)) { - struct intel_breadcrumbs *b = &engine->breadcrumbs; - - /* The ordering of irq_posted versus applying the barrier - * is crucial. The clearing of the current irq_posted must - * be visible before we perform the barrier operation, - * such that if a subsequent interrupt arrives, irq_posted - * is reasserted and our task rewoken (which causes us to - * do another __i915_request_irq_complete() immediately - * and reapply the barrier). Conversely, if the clear - * occurs after the barrier, then an interrupt that arrived - * whilst we waited on the barrier would not trigger a - * barrier on the next pass, and the read may not see the - * seqno update. - */ - engine->irq_seqno_barrier(engine); - - /* If we consume the irq, but we are no longer the bottom-half, - * the real bottom-half may not have serialised their own - * seqno check with the irq-barrier (i.e. may have inspected - * the seqno before we believe it coherent since they see - * irq_posted == false but we are still running). - */ - spin_lock_irq(&b->irq_lock); - if (b->irq_wait && b->irq_wait->tsk != current) - /* Note that if the bottom-half is changed as we - * are sending the wake-up, the new bottom-half will - * be woken by whomever made the change. We only have - * to worry about when we steal the irq-posted for - * ourself. - */ - wake_up_process(b->irq_wait->tsk); - spin_unlock_irq(&b->irq_lock); - - if (__i915_request_completed(rq, seqno)) - return true; - } - - return false; -} - void i915_memcpy_init_early(struct drm_i915_private *dev_priv); bool i915_memcpy_from_wc(void *dst, const void *src, unsigned long len); diff --git a/drivers/gpu/drm/i915/i915_fixed.h b/drivers/gpu/drm/i915/i915_fixed.h new file mode 100644 index 000000000000..591dd89ba7af --- /dev/null +++ b/drivers/gpu/drm/i915/i915_fixed.h @@ -0,0 +1,143 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2018 Intel Corporation + */ + +#ifndef _I915_FIXED_H_ +#define _I915_FIXED_H_ + +typedef struct { + u32 val; +} uint_fixed_16_16_t; + +#define FP_16_16_MAX ((uint_fixed_16_16_t){ .val = UINT_MAX }) + +static inline bool is_fixed16_zero(uint_fixed_16_16_t val) +{ + return val.val == 0; +} + +static inline uint_fixed_16_16_t u32_to_fixed16(u32 val) +{ + uint_fixed_16_16_t fp = { .val = val << 16 }; + + WARN_ON(val > U16_MAX); + + return fp; +} + +static inline u32 fixed16_to_u32_round_up(uint_fixed_16_16_t fp) +{ + return DIV_ROUND_UP(fp.val, 1 << 16); +} + +static inline u32 fixed16_to_u32(uint_fixed_16_16_t fp) +{ + return fp.val >> 16; +} + +static inline uint_fixed_16_16_t min_fixed16(uint_fixed_16_16_t min1, + uint_fixed_16_16_t min2) +{ + uint_fixed_16_16_t min = { .val = min(min1.val, min2.val) }; + + return min; +} + +static inline uint_fixed_16_16_t max_fixed16(uint_fixed_16_16_t max1, + uint_fixed_16_16_t max2) +{ + uint_fixed_16_16_t max = { .val = max(max1.val, max2.val) }; + + return max; +} + +static inline uint_fixed_16_16_t clamp_u64_to_fixed16(u64 val) +{ + uint_fixed_16_16_t fp = { .val = (u32)val }; + + WARN_ON(val > U32_MAX); + + return fp; +} + +static inline u32 div_round_up_fixed16(uint_fixed_16_16_t val, + uint_fixed_16_16_t d) +{ + return DIV_ROUND_UP(val.val, d.val); +} + +static inline u32 mul_round_up_u32_fixed16(u32 val, uint_fixed_16_16_t mul) +{ + u64 tmp; + + tmp = (u64)val * mul.val; + tmp = DIV_ROUND_UP_ULL(tmp, 1 << 16); + WARN_ON(tmp > U32_MAX); + + return (u32)tmp; +} + +static inline uint_fixed_16_16_t mul_fixed16(uint_fixed_16_16_t val, + uint_fixed_16_16_t mul) +{ + u64 tmp; + + tmp = (u64)val.val * mul.val; + tmp = tmp >> 16; + + return clamp_u64_to_fixed16(tmp); +} + +static inline uint_fixed_16_16_t div_fixed16(u32 val, u32 d) +{ + u64 tmp; + + tmp = (u64)val << 16; + tmp = DIV_ROUND_UP_ULL(tmp, d); + + return clamp_u64_to_fixed16(tmp); +} + +static inline u32 div_round_up_u32_fixed16(u32 val, uint_fixed_16_16_t d) +{ + u64 tmp; + + tmp = (u64)val << 16; + tmp = DIV_ROUND_UP_ULL(tmp, d.val); + WARN_ON(tmp > U32_MAX); + + return (u32)tmp; +} + +static inline uint_fixed_16_16_t mul_u32_fixed16(u32 val, uint_fixed_16_16_t mul) +{ + u64 tmp; + + tmp = (u64)val * mul.val; + + return clamp_u64_to_fixed16(tmp); +} + +static inline uint_fixed_16_16_t add_fixed16(uint_fixed_16_16_t add1, + uint_fixed_16_16_t add2) +{ + u64 tmp; + + tmp = (u64)add1.val + add2.val; + + return clamp_u64_to_fixed16(tmp); +} + +static inline uint_fixed_16_16_t add_fixed16_u32(uint_fixed_16_16_t add1, + u32 add2) +{ + uint_fixed_16_16_t tmp_add2 = u32_to_fixed16(add2); + u64 tmp; + + tmp = (u64)add1.val + tmp_add2.val; + + return clamp_u64_to_fixed16(tmp); +} + +#endif /* _I915_FIXED_H_ */ diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 6ae9a6080cc8..8558e81fdc2a 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -25,18 +25,9 @@ * */ -#include <drm/drmP.h> #include <drm/drm_vma_manager.h> +#include <drm/drm_pci.h> #include <drm/i915_drm.h> -#include "i915_drv.h" -#include "i915_gem_clflush.h" -#include "i915_vgpu.h" -#include "i915_trace.h" -#include "intel_drv.h" -#include "intel_frontbuffer.h" -#include "intel_mocs.h" -#include "intel_workarounds.h" -#include "i915_gemfs.h" #include <linux/dma-fence-array.h> #include <linux/kthread.h> #include <linux/reservation.h> @@ -46,6 +37,19 @@ #include <linux/swap.h> #include <linux/pci.h> #include <linux/dma-buf.h> +#include <linux/mman.h> + +#include "i915_drv.h" +#include "i915_gem_clflush.h" +#include "i915_gemfs.h" +#include "i915_reset.h" +#include "i915_trace.h" +#include "i915_vgpu.h" + +#include "intel_drv.h" +#include "intel_frontbuffer.h" +#include "intel_mocs.h" +#include "intel_workarounds.h" static void i915_gem_flush_free_objects(struct drm_i915_private *i915); @@ -139,6 +143,8 @@ int i915_mutex_lock_interruptible(struct drm_device *dev) static u32 __i915_gem_park(struct drm_i915_private *i915) { + intel_wakeref_t wakeref; + GEM_TRACE("\n"); lockdep_assert_held(&i915->drm.struct_mutex); @@ -169,14 +175,13 @@ static u32 __i915_gem_park(struct drm_i915_private *i915) i915_pmu_gt_parked(i915); i915_vma_parked(i915); - i915->gt.awake = false; + wakeref = fetch_and_zero(&i915->gt.awake); + GEM_BUG_ON(!wakeref); if (INTEL_GEN(i915) >= 6) gen6_rps_idle(i915); - intel_display_power_put(i915, POWER_DOMAIN_GT_IRQ); - - intel_runtime_pm_put(i915); + intel_display_power_put(i915, POWER_DOMAIN_GT_IRQ, wakeref); return i915->gt.epoch; } @@ -201,12 +206,11 @@ void i915_gem_unpark(struct drm_i915_private *i915) lockdep_assert_held(&i915->drm.struct_mutex); GEM_BUG_ON(!i915->gt.active_requests); + assert_rpm_wakelock_held(i915); if (i915->gt.awake) return; - intel_runtime_pm_get_noresume(i915); - /* * It seems that the DMC likes to transition between the DC states a lot * when there are no connected displays (no active power domains) during @@ -218,9 +222,9 @@ void i915_gem_unpark(struct drm_i915_private *i915) * Work around it by grabbing a GT IRQ power domain whilst there is any * GT activity, preventing any DC state transitions. */ - intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ); + i915->gt.awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ); + GEM_BUG_ON(!i915->gt.awake); - i915->gt.awake = true; if (unlikely(++i915->gt.epoch == 0)) /* keep 0 as invalid */ i915->gt.epoch = 1; @@ -243,21 +247,19 @@ int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { - struct drm_i915_private *dev_priv = to_i915(dev); - struct i915_ggtt *ggtt = &dev_priv->ggtt; + struct i915_ggtt *ggtt = &to_i915(dev)->ggtt; struct drm_i915_gem_get_aperture *args = data; struct i915_vma *vma; u64 pinned; + mutex_lock(&ggtt->vm.mutex); + pinned = ggtt->vm.reserved; - mutex_lock(&dev->struct_mutex); - list_for_each_entry(vma, &ggtt->vm.active_list, vm_link) + list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link) if (i915_vma_is_pinned(vma)) pinned += vma->node.size; - list_for_each_entry(vma, &ggtt->vm.inactive_list, vm_link) - if (i915_vma_is_pinned(vma)) - pinned += vma->node.size; - mutex_unlock(&dev->struct_mutex); + + mutex_unlock(&ggtt->vm.mutex); args->aper_size = ggtt->vm.total; args->aper_available_size = args->aper_size - pinned; @@ -437,15 +439,19 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj) if (ret) return ret; - while ((vma = list_first_entry_or_null(&obj->vma_list, - struct i915_vma, - obj_link))) { + spin_lock(&obj->vma.lock); + while (!ret && (vma = list_first_entry_or_null(&obj->vma.list, + struct i915_vma, + obj_link))) { list_move_tail(&vma->obj_link, &still_in_list); + spin_unlock(&obj->vma.lock); + ret = i915_vma_unbind(vma); - if (ret) - break; + + spin_lock(&obj->vma.lock); } - list_splice(&still_in_list, &obj->vma_list); + list_splice(&still_in_list, &obj->vma.list); + spin_unlock(&obj->vma.lock); return ret; } @@ -655,11 +661,6 @@ i915_gem_object_wait(struct drm_i915_gem_object *obj, struct intel_rps_client *rps_client) { might_sleep(); -#if IS_ENABLED(CONFIG_LOCKDEP) - GEM_BUG_ON(debug_locks && - !!lockdep_is_held(&obj->base.dev->struct_mutex) != - !!(flags & I915_WAIT_LOCKED)); -#endif GEM_BUG_ON(timeout < 0); timeout = i915_gem_object_wait_reservation(obj->resv, @@ -711,8 +712,8 @@ void i915_gem_object_free(struct drm_i915_gem_object *obj) static int i915_gem_create(struct drm_file *file, struct drm_i915_private *dev_priv, - uint64_t size, - uint32_t *handle_p) + u64 size, + u32 *handle_p) { struct drm_i915_gem_object *obj; int ret; @@ -783,6 +784,8 @@ fb_write_origin(struct drm_i915_gem_object *obj, unsigned int domain) void i915_gem_flush_ggtt_writes(struct drm_i915_private *dev_priv) { + intel_wakeref_t wakeref; + /* * No actual flushing is required for the GTT write domain for reads * from the GTT domain. Writes to it "immediately" go to main memory @@ -809,13 +812,13 @@ void i915_gem_flush_ggtt_writes(struct drm_i915_private *dev_priv) i915_gem_chipset_flush(dev_priv); - intel_runtime_pm_get(dev_priv); - spin_lock_irq(&dev_priv->uncore.lock); + with_intel_runtime_pm(dev_priv, wakeref) { + spin_lock_irq(&dev_priv->uncore.lock); - POSTING_READ_FW(RING_HEAD(RENDER_RING_BASE)); + POSTING_READ_FW(RING_HEAD(RENDER_RING_BASE)); - spin_unlock_irq(&dev_priv->uncore.lock); - intel_runtime_pm_put(dev_priv); + spin_unlock_irq(&dev_priv->uncore.lock); + } } static void @@ -859,58 +862,6 @@ flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains) obj->write_domain = 0; } -static inline int -__copy_to_user_swizzled(char __user *cpu_vaddr, - const char *gpu_vaddr, int gpu_offset, - int length) -{ - int ret, cpu_offset = 0; - - while (length > 0) { - int cacheline_end = ALIGN(gpu_offset + 1, 64); - int this_length = min(cacheline_end - gpu_offset, length); - int swizzled_gpu_offset = gpu_offset ^ 64; - - ret = __copy_to_user(cpu_vaddr + cpu_offset, - gpu_vaddr + swizzled_gpu_offset, - this_length); - if (ret) - return ret + length; - - cpu_offset += this_length; - gpu_offset += this_length; - length -= this_length; - } - - return 0; -} - -static inline int -__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset, - const char __user *cpu_vaddr, - int length) -{ - int ret, cpu_offset = 0; - - while (length > 0) { - int cacheline_end = ALIGN(gpu_offset + 1, 64); - int this_length = min(cacheline_end - gpu_offset, length); - int swizzled_gpu_offset = gpu_offset ^ 64; - - ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset, - cpu_vaddr + cpu_offset, - this_length); - if (ret) - return ret + length; - - cpu_offset += this_length; - gpu_offset += this_length; - length -= this_length; - } - - return 0; -} - /* * Pins the specified object's pages and synchronizes the object with * GPU accesses. Sets needs_clflush to non-zero if the caller should @@ -1030,72 +981,23 @@ err_unpin: return ret; } -static void -shmem_clflush_swizzled_range(char *addr, unsigned long length, - bool swizzled) -{ - if (unlikely(swizzled)) { - unsigned long start = (unsigned long) addr; - unsigned long end = (unsigned long) addr + length; - - /* For swizzling simply ensure that we always flush both - * channels. Lame, but simple and it works. Swizzled - * pwrite/pread is far from a hotpath - current userspace - * doesn't use it at all. */ - start = round_down(start, 128); - end = round_up(end, 128); - - drm_clflush_virt_range((void *)start, end - start); - } else { - drm_clflush_virt_range(addr, length); - } - -} - -/* Only difference to the fast-path function is that this can handle bit17 - * and uses non-atomic copy and kmap functions. */ static int -shmem_pread_slow(struct page *page, int offset, int length, - char __user *user_data, - bool page_do_bit17_swizzling, bool needs_clflush) +shmem_pread(struct page *page, int offset, int len, char __user *user_data, + bool needs_clflush) { char *vaddr; int ret; vaddr = kmap(page); - if (needs_clflush) - shmem_clflush_swizzled_range(vaddr + offset, length, - page_do_bit17_swizzling); - - if (page_do_bit17_swizzling) - ret = __copy_to_user_swizzled(user_data, vaddr, offset, length); - else - ret = __copy_to_user(user_data, vaddr + offset, length); - kunmap(page); - - return ret ? - EFAULT : 0; -} -static int -shmem_pread(struct page *page, int offset, int length, char __user *user_data, - bool page_do_bit17_swizzling, bool needs_clflush) -{ - int ret; + if (needs_clflush) + drm_clflush_virt_range(vaddr + offset, len); - ret = -ENODEV; - if (!page_do_bit17_swizzling) { - char *vaddr = kmap_atomic(page); + ret = __copy_to_user(user_data, vaddr + offset, len); - if (needs_clflush) - drm_clflush_virt_range(vaddr + offset, length); - ret = __copy_to_user_inatomic(user_data, vaddr + offset, length); - kunmap_atomic(vaddr); - } - if (ret == 0) - return 0; + kunmap(page); - return shmem_pread_slow(page, offset, length, user_data, - page_do_bit17_swizzling, needs_clflush); + return ret ? -EFAULT : 0; } static int @@ -1104,15 +1006,10 @@ i915_gem_shmem_pread(struct drm_i915_gem_object *obj, { char __user *user_data; u64 remain; - unsigned int obj_do_bit17_swizzling; unsigned int needs_clflush; unsigned int idx, offset; int ret; - obj_do_bit17_swizzling = 0; - if (i915_gem_object_needs_bit17_swizzle(obj)) - obj_do_bit17_swizzling = BIT(17); - ret = mutex_lock_interruptible(&obj->base.dev->struct_mutex); if (ret) return ret; @@ -1130,7 +1027,6 @@ i915_gem_shmem_pread(struct drm_i915_gem_object *obj, unsigned int length = min_t(u64, remain, PAGE_SIZE - offset); ret = shmem_pread(page, offset, length, user_data, - page_to_phys(page) & obj_do_bit17_swizzling, needs_clflush); if (ret) break; @@ -1174,6 +1070,7 @@ i915_gem_gtt_pread(struct drm_i915_gem_object *obj, { struct drm_i915_private *i915 = to_i915(obj->base.dev); struct i915_ggtt *ggtt = &i915->ggtt; + intel_wakeref_t wakeref; struct drm_mm_node node; struct i915_vma *vma; void __user *user_data; @@ -1184,7 +1081,7 @@ i915_gem_gtt_pread(struct drm_i915_gem_object *obj, if (ret) return ret; - intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(i915); vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE | PIN_NONFAULT | @@ -1257,7 +1154,7 @@ out_unpin: i915_vma_unpin(vma); } out_unlock: - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); mutex_unlock(&i915->drm.struct_mutex); return ret; @@ -1282,8 +1179,7 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data, if (args->size == 0) return 0; - if (!access_ok(VERIFY_WRITE, - u64_to_user_ptr(args->data_ptr), + if (!access_ok(u64_to_user_ptr(args->data_ptr), args->size)) return -EFAULT; @@ -1359,6 +1255,7 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj, { struct drm_i915_private *i915 = to_i915(obj->base.dev); struct i915_ggtt *ggtt = &i915->ggtt; + intel_wakeref_t wakeref; struct drm_mm_node node; struct i915_vma *vma; u64 remain, offset; @@ -1377,13 +1274,14 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj, * This easily dwarfs any performance advantage from * using the cache bypass of indirect GGTT access. */ - if (!intel_runtime_pm_get_if_in_use(i915)) { + wakeref = intel_runtime_pm_get_if_in_use(i915); + if (!wakeref) { ret = -EFAULT; goto out_unlock; } } else { /* No backing pages, no fallback, we must force GGTT access */ - intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(i915); } vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, @@ -1465,39 +1363,12 @@ out_unpin: i915_vma_unpin(vma); } out_rpm: - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); out_unlock: mutex_unlock(&i915->drm.struct_mutex); return ret; } -static int -shmem_pwrite_slow(struct page *page, int offset, int length, - char __user *user_data, - bool page_do_bit17_swizzling, - bool needs_clflush_before, - bool needs_clflush_after) -{ - char *vaddr; - int ret; - - vaddr = kmap(page); - if (unlikely(needs_clflush_before || page_do_bit17_swizzling)) - shmem_clflush_swizzled_range(vaddr + offset, length, - page_do_bit17_swizzling); - if (page_do_bit17_swizzling) - ret = __copy_from_user_swizzled(vaddr, offset, user_data, - length); - else - ret = __copy_from_user(vaddr + offset, user_data, length); - if (needs_clflush_after) - shmem_clflush_swizzled_range(vaddr + offset, length, - page_do_bit17_swizzling); - kunmap(page); - - return ret ? -EFAULT : 0; -} - /* Per-page copy function for the shmem pwrite fastpath. * Flushes invalid cachelines before writing to the target if * needs_clflush_before is set and flushes out any written cachelines after @@ -1505,31 +1376,24 @@ shmem_pwrite_slow(struct page *page, int offset, int length, */ static int shmem_pwrite(struct page *page, int offset, int len, char __user *user_data, - bool page_do_bit17_swizzling, bool needs_clflush_before, bool needs_clflush_after) { + char *vaddr; int ret; - ret = -ENODEV; - if (!page_do_bit17_swizzling) { - char *vaddr = kmap_atomic(page); + vaddr = kmap(page); - if (needs_clflush_before) - drm_clflush_virt_range(vaddr + offset, len); - ret = __copy_from_user_inatomic(vaddr + offset, user_data, len); - if (needs_clflush_after) - drm_clflush_virt_range(vaddr + offset, len); + if (needs_clflush_before) + drm_clflush_virt_range(vaddr + offset, len); - kunmap_atomic(vaddr); - } - if (ret == 0) - return ret; + ret = __copy_from_user(vaddr + offset, user_data, len); + if (!ret && needs_clflush_after) + drm_clflush_virt_range(vaddr + offset, len); + + kunmap(page); - return shmem_pwrite_slow(page, offset, len, user_data, - page_do_bit17_swizzling, - needs_clflush_before, - needs_clflush_after); + return ret ? -EFAULT : 0; } static int @@ -1539,7 +1403,6 @@ i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj, struct drm_i915_private *i915 = to_i915(obj->base.dev); void __user *user_data; u64 remain; - unsigned int obj_do_bit17_swizzling; unsigned int partial_cacheline_write; unsigned int needs_clflush; unsigned int offset, idx; @@ -1554,10 +1417,6 @@ i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj, if (ret) return ret; - obj_do_bit17_swizzling = 0; - if (i915_gem_object_needs_bit17_swizzle(obj)) - obj_do_bit17_swizzling = BIT(17); - /* If we don't overwrite a cacheline completely we need to be * careful to have up-to-date data by first clflushing. Don't * overcomplicate things and flush the entire patch. @@ -1574,7 +1433,6 @@ i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj, unsigned int length = min_t(u64, remain, PAGE_SIZE - offset); ret = shmem_pwrite(page, offset, length, user_data, - page_to_phys(page) & obj_do_bit17_swizzling, (offset | length) & partial_cacheline_write, needs_clflush & CLFLUSH_AFTER); if (ret) @@ -1609,9 +1467,7 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, if (args->size == 0) return 0; - if (!access_ok(VERIFY_READ, - u64_to_user_ptr(args->data_ptr), - args->size)) + if (!access_ok(u64_to_user_ptr(args->data_ptr), args->size)) return -EFAULT; obj = i915_gem_object_lookup(file, args->handle); @@ -1680,23 +1536,21 @@ err: static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj) { - struct drm_i915_private *i915; + struct drm_i915_private *i915 = to_i915(obj->base.dev); struct list_head *list; struct i915_vma *vma; GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); + mutex_lock(&i915->ggtt.vm.mutex); for_each_ggtt_vma(vma, obj) { - if (i915_vma_is_active(vma)) - continue; - if (!drm_mm_node_allocated(&vma->node)) continue; - list_move_tail(&vma->vm_link, &vma->vm->inactive_list); + list_move_tail(&vma->vm_link, &vma->vm->bound_list); } + mutex_unlock(&i915->ggtt.vm.mutex); - i915 = to_i915(obj->base.dev); spin_lock(&i915->mm.obj_lock); list = obj->bind_count ? &i915->mm.bound_list : &i915->mm.unbound_list; list_move_tail(&obj->mm.link, list); @@ -1716,8 +1570,8 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, { struct drm_i915_gem_set_domain *args = data; struct drm_i915_gem_object *obj; - uint32_t read_domains = args->read_domains; - uint32_t write_domain = args->write_domain; + u32 read_domains = args->read_domains; + u32 write_domain = args->write_domain; int err; /* Only handle setting domains to types used by the CPU. */ @@ -1740,6 +1594,7 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, */ err = i915_gem_object_wait(obj, I915_WAIT_INTERRUPTIBLE | + I915_WAIT_PRIORITY | (write_domain ? I915_WAIT_ALL : 0), MAX_SCHEDULE_TIMEOUT, to_rps_client(file)); @@ -1826,6 +1681,17 @@ i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, return 0; } +static inline bool +__vma_matches(struct vm_area_struct *vma, struct file *filp, + unsigned long addr, unsigned long size) +{ + if (vma->vm_file != filp) + return false; + + return vma->vm_start == addr && + (vma->vm_end - vma->vm_start) == PAGE_ALIGN(size); +} + /** * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address * it is mapped to. @@ -1868,39 +1734,50 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data, * pages from. */ if (!obj->base.filp) { - i915_gem_object_put(obj); - return -ENXIO; + addr = -ENXIO; + goto err; + } + + if (range_overflows(args->offset, args->size, (u64)obj->base.size)) { + addr = -EINVAL; + goto err; } addr = vm_mmap(obj->base.filp, 0, args->size, PROT_READ | PROT_WRITE, MAP_SHARED, args->offset); + if (IS_ERR_VALUE(addr)) + goto err; + if (args->flags & I915_MMAP_WC) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma; if (down_write_killable(&mm->mmap_sem)) { - i915_gem_object_put(obj); - return -EINTR; + addr = -EINTR; + goto err; } vma = find_vma(mm, addr); - if (vma) + if (vma && __vma_matches(vma, obj->base.filp, addr, args->size)) vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); else addr = -ENOMEM; up_write(&mm->mmap_sem); + if (IS_ERR_VALUE(addr)) + goto err; /* This may race, but that's ok, it only gets set */ WRITE_ONCE(obj->frontbuffer_ggtt_origin, ORIGIN_CPU); } i915_gem_object_put(obj); - if (IS_ERR((void *)addr)) - return addr; - - args->addr_ptr = (uint64_t) addr; + args->addr_ptr = (u64)addr; return 0; + +err: + i915_gem_object_put(obj); + return addr; } static unsigned int tile_row_pages(const struct drm_i915_gem_object *obj) @@ -2011,6 +1888,7 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf) struct drm_i915_private *dev_priv = to_i915(dev); struct i915_ggtt *ggtt = &dev_priv->ggtt; bool write = area->vm_flags & VM_WRITE; + intel_wakeref_t wakeref; struct i915_vma *vma; pgoff_t page_offset; int ret; @@ -2040,7 +1918,7 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf) if (ret) goto err; - intel_runtime_pm_get(dev_priv); + wakeref = intel_runtime_pm_get(dev_priv); ret = i915_mutex_lock_interruptible(dev); if (ret) @@ -2118,7 +1996,7 @@ err_unpin: err_unlock: mutex_unlock(&dev->struct_mutex); err_rpm: - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, wakeref); i915_gem_object_unpin_pages(obj); err: switch (ret) { @@ -2191,6 +2069,7 @@ void i915_gem_release_mmap(struct drm_i915_gem_object *obj) { struct drm_i915_private *i915 = to_i915(obj->base.dev); + intel_wakeref_t wakeref; /* Serialisation between user GTT access and our code depends upon * revoking the CPU's PTE whilst the mutex is held. The next user @@ -2201,7 +2080,7 @@ i915_gem_release_mmap(struct drm_i915_gem_object *obj) * wakeref. */ lockdep_assert_held(&i915->drm.struct_mutex); - intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(i915); if (!obj->userfault_count) goto out; @@ -2218,7 +2097,7 @@ i915_gem_release_mmap(struct drm_i915_gem_object *obj) wmb(); out: - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); } void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv) @@ -2298,8 +2177,8 @@ static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj) int i915_gem_mmap_gtt(struct drm_file *file, struct drm_device *dev, - uint32_t handle, - uint64_t *offset) + u32 handle, + u64 *offset) { struct drm_i915_gem_object *obj; int ret; @@ -2381,11 +2260,23 @@ void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj) invalidate_mapping_pages(mapping, 0, (loff_t)-1); } +/* + * Move pages to appropriate lru and release the pagevec, decrementing the + * ref count of those pages. + */ +static void check_release_pagevec(struct pagevec *pvec) +{ + check_move_unevictable_pages(pvec); + __pagevec_release(pvec); + cond_resched(); +} + static void i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj, struct sg_table *pages) { struct sgt_iter sgt_iter; + struct pagevec pvec; struct page *page; __i915_gem_object_release_shmem(obj, pages, true); @@ -2395,6 +2286,9 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj, if (i915_gem_object_needs_bit17_swizzle(obj)) i915_gem_object_save_bit_17_swizzle(obj, pages); + mapping_clear_unevictable(file_inode(obj->base.filp)->i_mapping); + + pagevec_init(&pvec); for_each_sgt_page(page, sgt_iter, pages) { if (obj->mm.dirty) set_page_dirty(page); @@ -2402,8 +2296,11 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj, if (obj->mm.madv == I915_MADV_WILLNEED) mark_page_accessed(page); - put_page(page); + if (!pagevec_add(&pvec, page)) + check_release_pagevec(&pvec); } + if (pagevec_count(&pvec)) + check_release_pagevec(&pvec); obj->mm.dirty = false; sg_free_table(pages); @@ -2428,8 +2325,8 @@ __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj) struct sg_table *pages; pages = fetch_and_zero(&obj->mm.pages); - if (!pages) - return NULL; + if (IS_ERR_OR_NULL(pages)) + return pages; spin_lock(&i915->mm.obj_lock); list_del(&obj->mm.link); @@ -2453,22 +2350,23 @@ __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj) return pages; } -void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj, - enum i915_mm_subclass subclass) +int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj, + enum i915_mm_subclass subclass) { struct sg_table *pages; + int ret; if (i915_gem_object_has_pinned_pages(obj)) - return; + return -EBUSY; GEM_BUG_ON(obj->bind_count); - if (!i915_gem_object_has_pages(obj)) - return; /* May be called by shrinker from within get_pages() (on another bo) */ mutex_lock_nested(&obj->mm.lock, subclass); - if (unlikely(atomic_read(&obj->mm.pages_pin_count))) + if (unlikely(atomic_read(&obj->mm.pages_pin_count))) { + ret = -EBUSY; goto unlock; + } /* * ->put_pages might need to allocate memory for the bit17 swizzle @@ -2476,14 +2374,27 @@ void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj, * lists early. */ pages = __i915_gem_object_unset_pages(obj); + + /* + * XXX Temporary hijinx to avoid updating all backends to handle + * NULL pages. In the future, when we have more asynchronous + * get_pages backends we should be better able to handle the + * cancellation of the async task in a more uniform manner. + */ + if (!pages && !i915_gem_object_needs_async_cancel(obj)) + pages = ERR_PTR(-EINVAL); + if (!IS_ERR(pages)) obj->ops->put_pages(obj, pages); + ret = 0; unlock: mutex_unlock(&obj->mm.lock); + + return ret; } -static bool i915_sg_trim(struct sg_table *orig_st) +bool i915_sg_trim(struct sg_table *orig_st) { struct sg_table new_st; struct scatterlist *sg, *new_sg; @@ -2524,6 +2435,7 @@ static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) unsigned long last_pfn = 0; /* suppress gcc warning */ unsigned int max_segment = i915_sg_segment_size(); unsigned int sg_page_sizes; + struct pagevec pvec; gfp_t noreclaim; int ret; @@ -2539,7 +2451,7 @@ static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) * If there's no chance of allocating enough pages for the whole * object, bail early. */ - if (page_count > totalram_pages) + if (page_count > totalram_pages()) return -ENOMEM; st = kmalloc(sizeof(*st), GFP_KERNEL); @@ -2559,6 +2471,7 @@ rebuild_st: * Fail silently without starting the shrinker */ mapping = obj->base.filp->f_mapping; + mapping_set_unevictable(mapping); noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM); noreclaim |= __GFP_NORETRY | __GFP_NOWARN; @@ -2573,6 +2486,7 @@ rebuild_st: gfp_t gfp = noreclaim; do { + cond_resched(); page = shmem_read_mapping_page_gfp(mapping, i, gfp); if (likely(!IS_ERR(page))) break; @@ -2583,7 +2497,6 @@ rebuild_st: } i915_gem_shrink(dev_priv, 2 * page_count, NULL, *s++); - cond_resched(); /* * We've tried hard to allocate the memory by reaping @@ -2673,8 +2586,14 @@ rebuild_st: err_sg: sg_mark_end(sg); err_pages: - for_each_sgt_page(page, sgt_iter, st) - put_page(page); + mapping_clear_unevictable(mapping); + pagevec_init(&pvec); + for_each_sgt_page(page, sgt_iter, st) { + if (!pagevec_add(&pvec, page)) + check_release_pagevec(&pvec); + } + if (pagevec_count(&pvec)) + check_release_pagevec(&pvec); sg_free_table(st); kfree(st); @@ -2976,59 +2895,12 @@ i915_gem_object_pwrite_gtt(struct drm_i915_gem_object *obj, return 0; } -static void i915_gem_client_mark_guilty(struct drm_i915_file_private *file_priv, - const struct i915_gem_context *ctx) +static bool match_ring(struct i915_request *rq) { - unsigned int score; - unsigned long prev_hang; + struct drm_i915_private *dev_priv = rq->i915; + u32 ring = I915_READ(RING_START(rq->engine->mmio_base)); - if (i915_gem_context_is_banned(ctx)) - score = I915_CLIENT_SCORE_CONTEXT_BAN; - else - score = 0; - - prev_hang = xchg(&file_priv->hang_timestamp, jiffies); - if (time_before(jiffies, prev_hang + I915_CLIENT_FAST_HANG_JIFFIES)) - score += I915_CLIENT_SCORE_HANG_FAST; - - if (score) { - atomic_add(score, &file_priv->ban_score); - - DRM_DEBUG_DRIVER("client %s: gained %u ban score, now %u\n", - ctx->name, score, - atomic_read(&file_priv->ban_score)); - } -} - -static void i915_gem_context_mark_guilty(struct i915_gem_context *ctx) -{ - unsigned int score; - bool banned, bannable; - - atomic_inc(&ctx->guilty_count); - - bannable = i915_gem_context_is_bannable(ctx); - score = atomic_add_return(CONTEXT_SCORE_GUILTY, &ctx->ban_score); - banned = score >= CONTEXT_SCORE_BAN_THRESHOLD; - - /* Cool contexts don't accumulate client ban score */ - if (!bannable) - return; - - if (banned) { - DRM_DEBUG_DRIVER("context %s: guilty %d, score %u, banned\n", - ctx->name, atomic_read(&ctx->guilty_count), - score); - i915_gem_context_set_banned(ctx); - } - - if (!IS_ERR_OR_NULL(ctx->file_priv)) - i915_gem_client_mark_guilty(ctx->file_priv, ctx); -} - -static void i915_gem_context_mark_innocent(struct i915_gem_context *ctx) -{ - atomic_inc(&ctx->active_count); + return ring == i915_ggtt_offset(rq->ring->vma); } struct i915_request * @@ -3050,9 +2922,16 @@ i915_gem_find_active_request(struct intel_engine_cs *engine) */ spin_lock_irqsave(&engine->timeline.lock, flags); list_for_each_entry(request, &engine->timeline.requests, link) { - if (__i915_request_completed(request, request->global_seqno)) + if (i915_request_completed(request)) continue; + if (!i915_request_started(request)) + break; + + /* More than one preemptible request may match! */ + if (!match_ring(request)) + break; + active = request; break; } @@ -3061,400 +2940,6 @@ i915_gem_find_active_request(struct intel_engine_cs *engine) return active; } -/* - * Ensure irq handler finishes, and not run again. - * Also return the active request so that we only search for it once. - */ -struct i915_request * -i915_gem_reset_prepare_engine(struct intel_engine_cs *engine) -{ - struct i915_request *request; - - /* - * During the reset sequence, we must prevent the engine from - * entering RC6. As the context state is undefined until we restart - * the engine, if it does enter RC6 during the reset, the state - * written to the powercontext is undefined and so we may lose - * GPU state upon resume, i.e. fail to restart after a reset. - */ - intel_uncore_forcewake_get(engine->i915, FORCEWAKE_ALL); - - request = engine->reset.prepare(engine); - if (request && request->fence.error == -EIO) - request = ERR_PTR(-EIO); /* Previous reset failed! */ - - return request; -} - -int i915_gem_reset_prepare(struct drm_i915_private *dev_priv) -{ - struct intel_engine_cs *engine; - struct i915_request *request; - enum intel_engine_id id; - int err = 0; - - for_each_engine(engine, dev_priv, id) { - request = i915_gem_reset_prepare_engine(engine); - if (IS_ERR(request)) { - err = PTR_ERR(request); - continue; - } - - engine->hangcheck.active_request = request; - } - - i915_gem_revoke_fences(dev_priv); - intel_uc_sanitize(dev_priv); - - return err; -} - -static void engine_skip_context(struct i915_request *request) -{ - struct intel_engine_cs *engine = request->engine; - struct i915_gem_context *hung_ctx = request->gem_context; - struct i915_timeline *timeline = request->timeline; - unsigned long flags; - - GEM_BUG_ON(timeline == &engine->timeline); - - spin_lock_irqsave(&engine->timeline.lock, flags); - spin_lock(&timeline->lock); - - list_for_each_entry_continue(request, &engine->timeline.requests, link) - if (request->gem_context == hung_ctx) - i915_request_skip(request, -EIO); - - list_for_each_entry(request, &timeline->requests, link) - i915_request_skip(request, -EIO); - - spin_unlock(&timeline->lock); - spin_unlock_irqrestore(&engine->timeline.lock, flags); -} - -/* Returns the request if it was guilty of the hang */ -static struct i915_request * -i915_gem_reset_request(struct intel_engine_cs *engine, - struct i915_request *request, - bool stalled) -{ - /* The guilty request will get skipped on a hung engine. - * - * Users of client default contexts do not rely on logical - * state preserved between batches so it is safe to execute - * queued requests following the hang. Non default contexts - * rely on preserved state, so skipping a batch loses the - * evolution of the state and it needs to be considered corrupted. - * Executing more queued batches on top of corrupted state is - * risky. But we take the risk by trying to advance through - * the queued requests in order to make the client behaviour - * more predictable around resets, by not throwing away random - * amount of batches it has prepared for execution. Sophisticated - * clients can use gem_reset_stats_ioctl and dma fence status - * (exported via sync_file info ioctl on explicit fences) to observe - * when it loses the context state and should rebuild accordingly. - * - * The context ban, and ultimately the client ban, mechanism are safety - * valves if client submission ends up resulting in nothing more than - * subsequent hangs. - */ - - if (i915_request_completed(request)) { - GEM_TRACE("%s pardoned global=%d (fence %llx:%d), current %d\n", - engine->name, request->global_seqno, - request->fence.context, request->fence.seqno, - intel_engine_get_seqno(engine)); - stalled = false; - } - - if (stalled) { - i915_gem_context_mark_guilty(request->gem_context); - i915_request_skip(request, -EIO); - - /* If this context is now banned, skip all pending requests. */ - if (i915_gem_context_is_banned(request->gem_context)) - engine_skip_context(request); - } else { - /* - * Since this is not the hung engine, it may have advanced - * since the hang declaration. Double check by refinding - * the active request at the time of the reset. - */ - request = i915_gem_find_active_request(engine); - if (request) { - unsigned long flags; - - i915_gem_context_mark_innocent(request->gem_context); - dma_fence_set_error(&request->fence, -EAGAIN); - - /* Rewind the engine to replay the incomplete rq */ - spin_lock_irqsave(&engine->timeline.lock, flags); - request = list_prev_entry(request, link); - if (&request->link == &engine->timeline.requests) - request = NULL; - spin_unlock_irqrestore(&engine->timeline.lock, flags); - } - } - - return request; -} - -void i915_gem_reset_engine(struct intel_engine_cs *engine, - struct i915_request *request, - bool stalled) -{ - /* - * Make sure this write is visible before we re-enable the interrupt - * handlers on another CPU, as tasklet_enable() resolves to just - * a compiler barrier which is insufficient for our purpose here. - */ - smp_store_mb(engine->irq_posted, 0); - - if (request) - request = i915_gem_reset_request(engine, request, stalled); - - /* Setup the CS to resume from the breadcrumb of the hung request */ - engine->reset.reset(engine, request); -} - -void i915_gem_reset(struct drm_i915_private *dev_priv, - unsigned int stalled_mask) -{ - struct intel_engine_cs *engine; - enum intel_engine_id id; - - lockdep_assert_held(&dev_priv->drm.struct_mutex); - - i915_retire_requests(dev_priv); - - for_each_engine(engine, dev_priv, id) { - struct intel_context *ce; - - i915_gem_reset_engine(engine, - engine->hangcheck.active_request, - stalled_mask & ENGINE_MASK(id)); - ce = fetch_and_zero(&engine->last_retired_context); - if (ce) - intel_context_unpin(ce); - - /* - * Ostensibily, we always want a context loaded for powersaving, - * so if the engine is idle after the reset, send a request - * to load our scratch kernel_context. - * - * More mysteriously, if we leave the engine idle after a reset, - * the next userspace batch may hang, with what appears to be - * an incoherent read by the CS (presumably stale TLB). An - * empty request appears sufficient to paper over the glitch. - */ - if (intel_engine_is_idle(engine)) { - struct i915_request *rq; - - rq = i915_request_alloc(engine, - dev_priv->kernel_context); - if (!IS_ERR(rq)) - i915_request_add(rq); - } - } - - i915_gem_restore_fences(dev_priv); -} - -void i915_gem_reset_finish_engine(struct intel_engine_cs *engine) -{ - engine->reset.finish(engine); - - intel_uncore_forcewake_put(engine->i915, FORCEWAKE_ALL); -} - -void i915_gem_reset_finish(struct drm_i915_private *dev_priv) -{ - struct intel_engine_cs *engine; - enum intel_engine_id id; - - lockdep_assert_held(&dev_priv->drm.struct_mutex); - - for_each_engine(engine, dev_priv, id) { - engine->hangcheck.active_request = NULL; - i915_gem_reset_finish_engine(engine); - } -} - -static void nop_submit_request(struct i915_request *request) -{ - GEM_TRACE("%s fence %llx:%d -> -EIO\n", - request->engine->name, - request->fence.context, request->fence.seqno); - dma_fence_set_error(&request->fence, -EIO); - - i915_request_submit(request); -} - -static void nop_complete_submit_request(struct i915_request *request) -{ - unsigned long flags; - - GEM_TRACE("%s fence %llx:%d -> -EIO\n", - request->engine->name, - request->fence.context, request->fence.seqno); - dma_fence_set_error(&request->fence, -EIO); - - spin_lock_irqsave(&request->engine->timeline.lock, flags); - __i915_request_submit(request); - intel_engine_init_global_seqno(request->engine, request->global_seqno); - spin_unlock_irqrestore(&request->engine->timeline.lock, flags); -} - -void i915_gem_set_wedged(struct drm_i915_private *i915) -{ - struct intel_engine_cs *engine; - enum intel_engine_id id; - - GEM_TRACE("start\n"); - - if (GEM_SHOW_DEBUG()) { - struct drm_printer p = drm_debug_printer(__func__); - - for_each_engine(engine, i915, id) - intel_engine_dump(engine, &p, "%s\n", engine->name); - } - - if (test_and_set_bit(I915_WEDGED, &i915->gpu_error.flags)) - goto out; - - /* - * First, stop submission to hw, but do not yet complete requests by - * rolling the global seqno forward (since this would complete requests - * for which we haven't set the fence error to EIO yet). - */ - for_each_engine(engine, i915, id) { - i915_gem_reset_prepare_engine(engine); - - engine->submit_request = nop_submit_request; - engine->schedule = NULL; - } - i915->caps.scheduler = 0; - - /* Even if the GPU reset fails, it should still stop the engines */ - if (INTEL_GEN(i915) >= 5) - intel_gpu_reset(i915, ALL_ENGINES); - - /* - * Make sure no one is running the old callback before we proceed with - * cancelling requests and resetting the completion tracking. Otherwise - * we might submit a request to the hardware which never completes. - */ - synchronize_rcu(); - - for_each_engine(engine, i915, id) { - /* Mark all executing requests as skipped */ - engine->cancel_requests(engine); - - /* - * Only once we've force-cancelled all in-flight requests can we - * start to complete all requests. - */ - engine->submit_request = nop_complete_submit_request; - } - - /* - * Make sure no request can slip through without getting completed by - * either this call here to intel_engine_init_global_seqno, or the one - * in nop_complete_submit_request. - */ - synchronize_rcu(); - - for_each_engine(engine, i915, id) { - unsigned long flags; - - /* - * Mark all pending requests as complete so that any concurrent - * (lockless) lookup doesn't try and wait upon the request as we - * reset it. - */ - spin_lock_irqsave(&engine->timeline.lock, flags); - intel_engine_init_global_seqno(engine, - intel_engine_last_submit(engine)); - spin_unlock_irqrestore(&engine->timeline.lock, flags); - - i915_gem_reset_finish_engine(engine); - } - -out: - GEM_TRACE("end\n"); - - wake_up_all(&i915->gpu_error.reset_queue); -} - -bool i915_gem_unset_wedged(struct drm_i915_private *i915) -{ - struct i915_timeline *tl; - - lockdep_assert_held(&i915->drm.struct_mutex); - if (!test_bit(I915_WEDGED, &i915->gpu_error.flags)) - return true; - - GEM_TRACE("start\n"); - - /* - * Before unwedging, make sure that all pending operations - * are flushed and errored out - we may have requests waiting upon - * third party fences. We marked all inflight requests as EIO, and - * every execbuf since returned EIO, for consistency we want all - * the currently pending requests to also be marked as EIO, which - * is done inside our nop_submit_request - and so we must wait. - * - * No more can be submitted until we reset the wedged bit. - */ - list_for_each_entry(tl, &i915->gt.timelines, link) { - struct i915_request *rq; - - rq = i915_gem_active_peek(&tl->last_request, - &i915->drm.struct_mutex); - if (!rq) - continue; - - /* - * We can't use our normal waiter as we want to - * avoid recursively trying to handle the current - * reset. The basic dma_fence_default_wait() installs - * a callback for dma_fence_signal(), which is - * triggered by our nop handler (indirectly, the - * callback enables the signaler thread which is - * woken by the nop_submit_request() advancing the seqno - * and when the seqno passes the fence, the signaler - * then signals the fence waking us up). - */ - if (dma_fence_default_wait(&rq->fence, true, - MAX_SCHEDULE_TIMEOUT) < 0) - return false; - } - i915_retire_requests(i915); - GEM_BUG_ON(i915->gt.active_requests); - - if (!intel_gpu_reset(i915, ALL_ENGINES)) - intel_engines_sanitize(i915); - - /* - * Undo nop_submit_request. We prevent all new i915 requests from - * being queued (by disallowing execbuf whilst wedged) so having - * waited for all active requests above, we know the system is idle - * and do not have to worry about a thread being inside - * engine->submit_request() as we swap over. So unlike installing - * the nop_submit_request on reset, we can do this from normal - * context and do not require stop_machine(). - */ - intel_engines_reset_default_submission(i915); - i915_gem_contexts_lost(i915); - - GEM_TRACE("end\n"); - - smp_mb__before_atomic(); /* complete takeover before enabling execbuf */ - clear_bit(I915_WEDGED, &i915->gpu_error.flags); - - return true; -} - static void i915_gem_retire_work_handler(struct work_struct *work) { @@ -3530,6 +3015,8 @@ static void __sleep_rcu(struct rcu_head *rcu) struct sleep_rcu_work *s = container_of(rcu, typeof(*s), rcu); struct drm_i915_private *i915 = s->i915; + destroy_rcu_head(&s->rcu); + if (same_epoch(i915, s->epoch)) { INIT_WORK(&s->work, __sleep_work); queue_work(i915->wq, &s->work); @@ -3555,7 +3042,7 @@ static void assert_kernel_context_is_current(struct drm_i915_private *i915) GEM_BUG_ON(i915->gt.active_requests); for_each_engine(engine, i915, id) { - GEM_BUG_ON(__i915_gem_active_peek(&engine->timeline.last_request)); + GEM_BUG_ON(__i915_active_request_peek(&engine->timeline.last_request)); GEM_BUG_ON(engine->last_retired_context != to_intel_context(i915->kernel_context, engine)); } @@ -3646,6 +3133,7 @@ out_rearm: if (same_epoch(dev_priv, epoch)) { struct sleep_rcu_work *s = kmalloc(sizeof(*s), GFP_KERNEL); if (s) { + init_rcu_head(&s->rcu); s->i915 = dev_priv; s->epoch = epoch; call_rcu(&s->rcu, __sleep_rcu); @@ -3743,7 +3231,9 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) start = ktime_get(); ret = i915_gem_object_wait(obj, - I915_WAIT_INTERRUPTIBLE | I915_WAIT_ALL, + I915_WAIT_INTERRUPTIBLE | + I915_WAIT_PRIORITY | + I915_WAIT_ALL, to_wait_timeout(args->timeout_ns), to_rps_client(file)); @@ -3771,33 +3261,6 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) return ret; } -static long wait_for_timeline(struct i915_timeline *tl, - unsigned int flags, long timeout) -{ - struct i915_request *rq; - - rq = i915_gem_active_get_unlocked(&tl->last_request); - if (!rq) - return timeout; - - /* - * "Race-to-idle". - * - * Switching to the kernel context is often used a synchronous - * step prior to idling, e.g. in suspend for flushing all - * current operations to memory before sleeping. These we - * want to complete as quickly as possible to avoid prolonged - * stalls, so allow the gpu to boost to maximum clocks. - */ - if (flags & I915_WAIT_FOR_IDLE_BOOST) - gen6_rps_boost(rq, NULL); - - timeout = i915_request_wait(rq, flags, timeout); - i915_request_put(rq); - - return timeout; -} - static int wait_for_engines(struct drm_i915_private *i915) { if (wait_for(intel_engines_are_idle(i915), I915_IDLE_ENGINES_TIMEOUT)) { @@ -3811,6 +3274,52 @@ static int wait_for_engines(struct drm_i915_private *i915) return 0; } +static long +wait_for_timelines(struct drm_i915_private *i915, + unsigned int flags, long timeout) +{ + struct i915_gt_timelines *gt = &i915->gt.timelines; + struct i915_timeline *tl; + + if (!READ_ONCE(i915->gt.active_requests)) + return timeout; + + mutex_lock(>->mutex); + list_for_each_entry(tl, >->active_list, link) { + struct i915_request *rq; + + rq = i915_active_request_get_unlocked(&tl->last_request); + if (!rq) + continue; + + mutex_unlock(>->mutex); + + /* + * "Race-to-idle". + * + * Switching to the kernel context is often used a synchronous + * step prior to idling, e.g. in suspend for flushing all + * current operations to memory before sleeping. These we + * want to complete as quickly as possible to avoid prolonged + * stalls, so allow the gpu to boost to maximum clocks. + */ + if (flags & I915_WAIT_FOR_IDLE_BOOST) + gen6_rps_boost(rq, NULL); + + timeout = i915_request_wait(rq, flags, timeout); + i915_request_put(rq); + if (timeout < 0) + return timeout; + + /* restart after reacquiring the lock */ + mutex_lock(>->mutex); + tl = list_entry(>->active_list, typeof(*tl), link); + } + mutex_unlock(>->mutex); + + return timeout; +} + int i915_gem_wait_for_idle(struct drm_i915_private *i915, unsigned int flags, long timeout) { @@ -3822,17 +3331,15 @@ int i915_gem_wait_for_idle(struct drm_i915_private *i915, if (!READ_ONCE(i915->gt.awake)) return 0; + timeout = wait_for_timelines(i915, flags, timeout); + if (timeout < 0) + return timeout; + if (flags & I915_WAIT_LOCKED) { - struct i915_timeline *tl; int err; lockdep_assert_held(&i915->drm.struct_mutex); - list_for_each_entry(tl, &i915->gt.timelines, link) { - timeout = wait_for_timeline(tl, flags, timeout); - if (timeout < 0) - return timeout; - } if (GEM_SHOW_DEBUG() && !timeout) { /* Presume that timeout was non-zero to begin with! */ dev_warn(&i915->drm.pdev->dev, @@ -3846,17 +3353,6 @@ int i915_gem_wait_for_idle(struct drm_i915_private *i915, i915_retire_requests(i915); GEM_BUG_ON(i915->gt.active_requests); - } else { - struct intel_engine_cs *engine; - enum intel_engine_id id; - - for_each_engine(engine, i915, id) { - struct i915_timeline *tl = &engine->timeline; - - timeout = wait_for_timeline(tl, flags, timeout); - if (timeout < 0) - return timeout; - } } return 0; @@ -4042,7 +3538,7 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, * reading an invalid PTE on older architectures. */ restart: - list_for_each_entry(vma, &obj->vma_list, obj_link) { + list_for_each_entry(vma, &obj->vma.list, obj_link) { if (!drm_mm_node_allocated(&vma->node)) continue; @@ -4120,7 +3616,7 @@ restart: */ } - list_for_each_entry(vma, &obj->vma_list, obj_link) { + list_for_each_entry(vma, &obj->vma.list, obj_link) { if (!drm_mm_node_allocated(&vma->node)) continue; @@ -4130,7 +3626,7 @@ restart: } } - list_for_each_entry(vma, &obj->vma_list, obj_link) + list_for_each_entry(vma, &obj->vma.list, obj_link) vma->node.color = cache_level; i915_gem_object_set_cache_coherency(obj, cache_level); obj->cache_dirty = true; /* Always invalidate stale cachelines */ @@ -4693,7 +4189,8 @@ out: } static void -frontbuffer_retire(struct i915_gem_active *active, struct i915_request *request) +frontbuffer_retire(struct i915_active_request *active, + struct i915_request *request) { struct drm_i915_gem_object *obj = container_of(active, typeof(*obj), frontbuffer_write); @@ -4706,17 +4203,22 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj, { mutex_init(&obj->mm.lock); - INIT_LIST_HEAD(&obj->vma_list); + spin_lock_init(&obj->vma.lock); + INIT_LIST_HEAD(&obj->vma.list); + INIT_LIST_HEAD(&obj->lut_list); INIT_LIST_HEAD(&obj->batch_pool_link); + init_rcu_head(&obj->rcu); + obj->ops = ops; reservation_object_init(&obj->__builtin_resv); obj->resv = &obj->__builtin_resv; obj->frontbuffer_ggtt_origin = ORIGIN_GTT; - init_request_active(&obj->frontbuffer_write, frontbuffer_retire); + i915_active_request_init(&obj->frontbuffer_write, + NULL, frontbuffer_retire); obj->mm.madv = I915_MADV_WILLNEED; INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN); @@ -4859,8 +4361,9 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915, struct llist_node *freed) { struct drm_i915_gem_object *obj, *on; + intel_wakeref_t wakeref; - intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(i915); llist_for_each_entry_safe(obj, on, freed, freed) { struct i915_vma *vma, *vn; @@ -4869,14 +4372,13 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915, mutex_lock(&i915->drm.struct_mutex); GEM_BUG_ON(i915_gem_object_is_active(obj)); - list_for_each_entry_safe(vma, vn, - &obj->vma_list, obj_link) { + list_for_each_entry_safe(vma, vn, &obj->vma.list, obj_link) { GEM_BUG_ON(i915_vma_is_active(vma)); vma->flags &= ~I915_VMA_PIN_MASK; i915_vma_destroy(vma); } - GEM_BUG_ON(!list_empty(&obj->vma_list)); - GEM_BUG_ON(!RB_EMPTY_ROOT(&obj->vma_tree)); + GEM_BUG_ON(!list_empty(&obj->vma.list)); + GEM_BUG_ON(!RB_EMPTY_ROOT(&obj->vma.tree)); /* This serializes freeing with the shrinker. Since the free * is delayed, first by RCU then by the workqueue, we want the @@ -4921,7 +4423,7 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915, if (on) cond_resched(); } - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); } static void i915_gem_flush_free_objects(struct drm_i915_private *i915) @@ -4977,6 +4479,13 @@ static void __i915_gem_free_object_rcu(struct rcu_head *head) struct drm_i915_private *i915 = to_i915(obj->base.dev); /* + * We reuse obj->rcu for the freed list, so we had better not treat + * it like a rcu_head from this point forwards. And we expect all + * objects to be freed via this path. + */ + destroy_rcu_head(&obj->rcu); + + /* * Since we require blocking on struct_mutex to unbind the freed * object from the GPU before releasing resources back to the * system, we can not do that directly from the RCU callback (which may @@ -5023,13 +4532,11 @@ void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj) void i915_gem_sanitize(struct drm_i915_private *i915) { - int err; + intel_wakeref_t wakeref; GEM_TRACE("\n"); - mutex_lock(&i915->drm.struct_mutex); - - intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(i915); intel_uncore_forcewake_get(i915, FORCEWAKE_ALL); /* @@ -5049,28 +4556,28 @@ void i915_gem_sanitize(struct drm_i915_private *i915) * it may impact the display and we are uncertain about the stability * of the reset, so this could be applied to even earlier gen. */ - err = -ENODEV; - if (INTEL_GEN(i915) >= 5 && intel_has_gpu_reset(i915)) - err = WARN_ON(intel_gpu_reset(i915, ALL_ENGINES)); - if (!err) - intel_engines_sanitize(i915); + intel_engines_sanitize(i915, false); intel_uncore_forcewake_put(i915, FORCEWAKE_ALL); - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); + mutex_lock(&i915->drm.struct_mutex); i915_gem_contexts_lost(i915); mutex_unlock(&i915->drm.struct_mutex); } int i915_gem_suspend(struct drm_i915_private *i915) { + intel_wakeref_t wakeref; int ret; GEM_TRACE("\n"); - intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(i915); intel_suspend_gt_powersave(i915); + flush_workqueue(i915->wq); + mutex_lock(&i915->drm.struct_mutex); /* @@ -5100,11 +4607,9 @@ int i915_gem_suspend(struct drm_i915_private *i915) i915_retire_requests(i915); /* ensure we flush after wedging */ mutex_unlock(&i915->drm.struct_mutex); + i915_reset_flush(i915); - intel_uc_suspend(i915); - - cancel_delayed_work_sync(&i915->gpu_error.hangcheck_work); - cancel_delayed_work_sync(&i915->gt.retire_work); + drain_delayed_work(&i915->gt.retire_work); /* * As the idle_work is rearming if it detects a race, play safe and @@ -5112,6 +4617,8 @@ int i915_gem_suspend(struct drm_i915_private *i915) */ drain_delayed_work(&i915->gt.idle_work); + intel_uc_suspend(i915); + /* * Assert that we successfully flushed all the work and * reset the GPU back to its idle, low power state. @@ -5120,12 +4627,12 @@ int i915_gem_suspend(struct drm_i915_private *i915) if (WARN_ON(!intel_engines_are_idle(i915))) i915_gem_set_wedged(i915); /* no hope, discard everything */ - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); return 0; err_unlock: mutex_unlock(&i915->drm.struct_mutex); - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); return ret; } @@ -5219,15 +4726,15 @@ void i915_gem_init_swizzling(struct drm_i915_private *dev_priv) I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | DISP_TILE_SURFACE_SWIZZLING); - if (IS_GEN5(dev_priv)) + if (IS_GEN(dev_priv, 5)) return; I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); - if (IS_GEN6(dev_priv)) + if (IS_GEN(dev_priv, 6)) I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB)); - else if (IS_GEN7(dev_priv)) + else if (IS_GEN(dev_priv, 7)) I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB)); - else if (IS_GEN8(dev_priv)) + else if (IS_GEN(dev_priv, 8)) I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW)); else BUG(); @@ -5249,10 +4756,10 @@ static void init_unused_rings(struct drm_i915_private *dev_priv) init_unused_ring(dev_priv, SRB1_BASE); init_unused_ring(dev_priv, SRB2_BASE); init_unused_ring(dev_priv, SRB3_BASE); - } else if (IS_GEN2(dev_priv)) { + } else if (IS_GEN(dev_priv, 2)) { init_unused_ring(dev_priv, SRB0_BASE); init_unused_ring(dev_priv, SRB1_BASE); - } else if (IS_GEN3(dev_priv)) { + } else if (IS_GEN(dev_priv, 3)) { init_unused_ring(dev_priv, PRB1_BASE); init_unused_ring(dev_priv, PRB2_BASE); } @@ -5293,19 +4800,10 @@ int i915_gem_init_hw(struct drm_i915_private *dev_priv) I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev_priv) ? LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED); - if (HAS_PCH_NOP(dev_priv)) { - if (IS_IVYBRIDGE(dev_priv)) { - u32 temp = I915_READ(GEN7_MSG_CTL); - temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK); - I915_WRITE(GEN7_MSG_CTL, temp); - } else if (INTEL_GEN(dev_priv) >= 7) { - u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT); - temp &= ~RESET_PCH_HANDSHAKE_ENABLE; - I915_WRITE(HSW_NDE_RSTWRN_OPT, temp); - } - } - + /* Apply the GT workarounds... */ intel_gt_apply_workarounds(dev_priv); + /* ...and determine whether they are sticking. */ + intel_gt_verify_workarounds(dev_priv, "init"); i915_gem_init_swizzling(dev_priv); @@ -5557,6 +5055,8 @@ int i915_gem_init(struct drm_i915_private *dev_priv) dev_priv->gt.cleanup_engine = intel_engine_cleanup; } + i915_timelines_init(dev_priv); + ret = i915_gem_init_userptr(dev_priv); if (ret) return ret; @@ -5585,7 +5085,7 @@ int i915_gem_init(struct drm_i915_private *dev_priv) } ret = i915_gem_init_scratch(dev_priv, - IS_GEN2(dev_priv) ? SZ_256K : PAGE_SIZE); + IS_GEN(dev_priv, 2) ? SZ_256K : PAGE_SIZE); if (ret) { GEM_BUG_ON(ret == -EIO); goto err_ggtt; @@ -5679,8 +5179,10 @@ err_unlock: err_uc_misc: intel_uc_fini_misc(dev_priv); - if (ret != -EIO) + if (ret != -EIO) { i915_gem_cleanup_userptr(dev_priv); + i915_timelines_fini(dev_priv); + } if (ret == -EIO) { mutex_lock(&dev_priv->drm.struct_mutex); @@ -5731,6 +5233,7 @@ void i915_gem_fini(struct drm_i915_private *dev_priv) intel_uc_fini_misc(dev_priv); i915_gem_cleanup_userptr(dev_priv); + i915_timelines_fini(dev_priv); i915_gem_drain_freed_objects(dev_priv); @@ -5833,7 +5336,6 @@ int i915_gem_init_early(struct drm_i915_private *dev_priv) if (!dev_priv->priorities) goto err_dependencies; - INIT_LIST_HEAD(&dev_priv->gt.timelines); INIT_LIST_HEAD(&dev_priv->gt.active_rings); INIT_LIST_HEAD(&dev_priv->gt.closed_vma); @@ -5845,6 +5347,7 @@ int i915_gem_init_early(struct drm_i915_private *dev_priv) i915_gem_idle_work_handler); init_waitqueue_head(&dev_priv->gpu_error.wait_queue); init_waitqueue_head(&dev_priv->gpu_error.reset_queue); + mutex_init(&dev_priv->gpu_error.wedge_mutex); atomic_set(&dev_priv->mm.bsd_engine_dispatch_index, 0); @@ -5876,7 +5379,6 @@ void i915_gem_cleanup_early(struct drm_i915_private *dev_priv) GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list)); GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count)); WARN_ON(dev_priv->mm.object_count); - WARN_ON(!list_empty(&dev_priv->gt.timelines)); kmem_cache_destroy(dev_priv->priorities); kmem_cache_destroy(dev_priv->dependencies); @@ -6001,7 +5503,7 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old, * the bits. */ BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES > - sizeof(atomic_t) * BITS_PER_BYTE); + BITS_PER_TYPE(atomic_t)); if (old) { WARN_ON(!(atomic_read(&old->frontbuffer_bits) & frontbuffer_bits)); diff --git a/drivers/gpu/drm/i915/i915_gem.h b/drivers/gpu/drm/i915/i915_gem.h index 599c4f6eb1ea..b0e4b976880c 100644 --- a/drivers/gpu/drm/i915/i915_gem.h +++ b/drivers/gpu/drm/i915/i915_gem.h @@ -47,17 +47,19 @@ struct drm_i915_private; #define GEM_DEBUG_DECL(var) var #define GEM_DEBUG_EXEC(expr) expr #define GEM_DEBUG_BUG_ON(expr) GEM_BUG_ON(expr) +#define GEM_DEBUG_WARN_ON(expr) GEM_WARN_ON(expr) #else #define GEM_SHOW_DEBUG() (0) #define GEM_BUG_ON(expr) BUILD_BUG_ON_INVALID(expr) -#define GEM_WARN_ON(expr) (BUILD_BUG_ON_INVALID(expr), 0) +#define GEM_WARN_ON(expr) ({ unlikely(!!(expr)); }) #define GEM_DEBUG_DECL(var) #define GEM_DEBUG_EXEC(expr) do { } while (0) #define GEM_DEBUG_BUG_ON(expr) +#define GEM_DEBUG_WARN_ON(expr) ({ BUILD_BUG_ON_INVALID(expr); 0; }) #endif #if IS_ENABLED(CONFIG_DRM_I915_TRACE_GEM) diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c index f772593b99ab..280813a4bf82 100644 --- a/drivers/gpu/drm/i915/i915_gem_context.c +++ b/drivers/gpu/drm/i915/i915_gem_context.c @@ -86,10 +86,10 @@ */ #include <linux/log2.h> -#include <drm/drmP.h> #include <drm/i915_drm.h> #include "i915_drv.h" #include "i915_trace.h" +#include "intel_lrc_reg.h" #include "intel_workarounds.h" #define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1 @@ -311,7 +311,7 @@ static u32 default_desc_template(const struct drm_i915_private *i915, address_mode = INTEL_LEGACY_64B_CONTEXT; desc |= address_mode << GEN8_CTX_ADDRESSING_MODE_SHIFT; - if (IS_GEN8(i915)) + if (IS_GEN(i915, 8)) desc |= GEN8_CTX_L3LLC_COHERENT; /* TODO: WaDisableLiteRestore when we start using semaphore @@ -322,6 +322,32 @@ static u32 default_desc_template(const struct drm_i915_private *i915, return desc; } +static void intel_context_retire(struct i915_active_request *active, + struct i915_request *rq) +{ + struct intel_context *ce = + container_of(active, typeof(*ce), active_tracker); + + intel_context_unpin(ce); +} + +void +intel_context_init(struct intel_context *ce, + struct i915_gem_context *ctx, + struct intel_engine_cs *engine) +{ + ce->gem_context = ctx; + + INIT_LIST_HEAD(&ce->signal_link); + INIT_LIST_HEAD(&ce->signals); + + /* Use the whole device by default */ + ce->sseu = intel_device_default_sseu(ctx->i915); + + i915_active_request_init(&ce->active_tracker, + NULL, intel_context_retire); +} + static struct i915_gem_context * __create_hw_context(struct drm_i915_private *dev_priv, struct drm_i915_file_private *file_priv) @@ -337,13 +363,10 @@ __create_hw_context(struct drm_i915_private *dev_priv, kref_init(&ctx->ref); list_add_tail(&ctx->link, &dev_priv->contexts.list); ctx->i915 = dev_priv; - ctx->sched.priority = I915_PRIORITY_NORMAL; + ctx->sched.priority = I915_USER_PRIORITY(I915_PRIORITY_NORMAL); - for (n = 0; n < ARRAY_SIZE(ctx->__engine); n++) { - struct intel_context *ce = &ctx->__engine[n]; - - ce->gem_context = ctx; - } + for (n = 0; n < ARRAY_SIZE(ctx->__engine); n++) + intel_context_init(&ctx->__engine[n], ctx, dev_priv->engine[n]); INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL); INIT_LIST_HEAD(&ctx->handles_list); @@ -414,7 +437,7 @@ i915_gem_create_context(struct drm_i915_private *dev_priv, if (IS_ERR(ctx)) return ctx; - if (USES_FULL_PPGTT(dev_priv)) { + if (HAS_FULL_PPGTT(dev_priv)) { struct i915_hw_ppgtt *ppgtt; ppgtt = i915_ppgtt_create(dev_priv, file_priv); @@ -457,7 +480,7 @@ i915_gem_context_create_gvt(struct drm_device *dev) if (ret) return ERR_PTR(ret); - ctx = __create_hw_context(to_i915(dev), NULL); + ctx = i915_gem_create_context(to_i915(dev), NULL); if (IS_ERR(ctx)) goto out; @@ -504,7 +527,7 @@ i915_gem_context_create_kernel(struct drm_i915_private *i915, int prio) } i915_gem_context_clear_bannable(ctx); - ctx->sched.priority = prio; + ctx->sched.priority = I915_USER_PRIORITY(prio); ctx->ring_size = PAGE_SIZE; GEM_BUG_ON(!i915_gem_context_is_kernel(ctx)); @@ -535,16 +558,12 @@ static bool needs_preempt_context(struct drm_i915_private *i915) int i915_gem_contexts_init(struct drm_i915_private *dev_priv) { struct i915_gem_context *ctx; - int ret; /* Reassure ourselves we are only called once */ GEM_BUG_ON(dev_priv->kernel_context); GEM_BUG_ON(dev_priv->preempt_context); - ret = intel_ctx_workarounds_init(dev_priv); - if (ret) - return ret; - + intel_engine_init_ctx_wa(dev_priv->engine[RCS]); init_contexts(dev_priv); /* lowest priority; idle task */ @@ -650,10 +669,10 @@ last_request_on_engine(struct i915_timeline *timeline, GEM_BUG_ON(timeline == &engine->timeline); - rq = i915_gem_active_raw(&timeline->last_request, - &engine->i915->drm.struct_mutex); + rq = i915_active_request_raw(&timeline->last_request, + &engine->i915->drm.struct_mutex); if (rq && rq->engine == engine) { - GEM_TRACE("last request for %s on engine %s: %llx:%d\n", + GEM_TRACE("last request for %s on engine %s: %llx:%llu\n", timeline->name, engine->name, rq->fence.context, rq->fence.seqno); GEM_BUG_ON(rq->timeline != timeline); @@ -690,14 +709,14 @@ static bool engine_has_kernel_context_barrier(struct intel_engine_cs *engine) * switch-to-kernel-context? */ if (!i915_timeline_sync_is_later(barrier, &rq->fence)) { - GEM_TRACE("%s needs barrier for %llx:%d\n", + GEM_TRACE("%s needs barrier for %llx:%lld\n", ring->timeline->name, rq->fence.context, rq->fence.seqno); return false; } - GEM_TRACE("%s has barrier after %llx:%d\n", + GEM_TRACE("%s has barrier after %llx:%lld\n", ring->timeline->name, rq->fence.context, rq->fence.seqno); @@ -753,7 +772,7 @@ int i915_gem_switch_to_kernel_context(struct drm_i915_private *i915) if (prev->gem_context == i915->kernel_context) continue; - GEM_TRACE("add barrier on %s for %llx:%d\n", + GEM_TRACE("add barrier on %s for %llx:%lld\n", engine->name, prev->fence.context, prev->fence.seqno); @@ -844,6 +863,56 @@ out: return 0; } +static int get_sseu(struct i915_gem_context *ctx, + struct drm_i915_gem_context_param *args) +{ + struct drm_i915_gem_context_param_sseu user_sseu; + struct intel_engine_cs *engine; + struct intel_context *ce; + int ret; + + if (args->size == 0) + goto out; + else if (args->size < sizeof(user_sseu)) + return -EINVAL; + + if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value), + sizeof(user_sseu))) + return -EFAULT; + + if (user_sseu.flags || user_sseu.rsvd) + return -EINVAL; + + engine = intel_engine_lookup_user(ctx->i915, + user_sseu.engine_class, + user_sseu.engine_instance); + if (!engine) + return -EINVAL; + + /* Only use for mutex here is to serialize get_param and set_param. */ + ret = mutex_lock_interruptible(&ctx->i915->drm.struct_mutex); + if (ret) + return ret; + + ce = to_intel_context(ctx, engine); + + user_sseu.slice_mask = ce->sseu.slice_mask; + user_sseu.subslice_mask = ce->sseu.subslice_mask; + user_sseu.min_eus_per_subslice = ce->sseu.min_eus_per_subslice; + user_sseu.max_eus_per_subslice = ce->sseu.max_eus_per_subslice; + + mutex_unlock(&ctx->i915->drm.struct_mutex); + + if (copy_to_user(u64_to_user_ptr(args->value), &user_sseu, + sizeof(user_sseu))) + return -EFAULT; + +out: + args->size = sizeof(user_sseu); + + return 0; +} + int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { @@ -856,15 +925,17 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, if (!ctx) return -ENOENT; - args->size = 0; switch (args->param) { case I915_CONTEXT_PARAM_BAN_PERIOD: ret = -EINVAL; break; case I915_CONTEXT_PARAM_NO_ZEROMAP: + args->size = 0; args->value = test_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags); break; case I915_CONTEXT_PARAM_GTT_SIZE: + args->size = 0; + if (ctx->ppgtt) args->value = ctx->ppgtt->vm.total; else if (to_i915(dev)->mm.aliasing_ppgtt) @@ -873,13 +944,19 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, args->value = to_i915(dev)->ggtt.vm.total; break; case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: + args->size = 0; args->value = i915_gem_context_no_error_capture(ctx); break; case I915_CONTEXT_PARAM_BANNABLE: + args->size = 0; args->value = i915_gem_context_is_bannable(ctx); break; case I915_CONTEXT_PARAM_PRIORITY: - args->value = ctx->sched.priority; + args->size = 0; + args->value = ctx->sched.priority >> I915_USER_PRIORITY_SHIFT; + break; + case I915_CONTEXT_PARAM_SSEU: + ret = get_sseu(ctx, args); break; default: ret = -EINVAL; @@ -890,6 +967,281 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, return ret; } +static int gen8_emit_rpcs_config(struct i915_request *rq, + struct intel_context *ce, + struct intel_sseu sseu) +{ + u64 offset; + u32 *cs; + + cs = intel_ring_begin(rq, 4); + if (IS_ERR(cs)) + return PTR_ERR(cs); + + offset = i915_ggtt_offset(ce->state) + + LRC_STATE_PN * PAGE_SIZE + + (CTX_R_PWR_CLK_STATE + 1) * 4; + + *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; + *cs++ = lower_32_bits(offset); + *cs++ = upper_32_bits(offset); + *cs++ = gen8_make_rpcs(rq->i915, &sseu); + + intel_ring_advance(rq, cs); + + return 0; +} + +static int +gen8_modify_rpcs_gpu(struct intel_context *ce, + struct intel_engine_cs *engine, + struct intel_sseu sseu) +{ + struct drm_i915_private *i915 = engine->i915; + struct i915_request *rq, *prev; + intel_wakeref_t wakeref; + int ret; + + GEM_BUG_ON(!ce->pin_count); + + lockdep_assert_held(&i915->drm.struct_mutex); + + /* Submitting requests etc needs the hw awake. */ + wakeref = intel_runtime_pm_get(i915); + + rq = i915_request_alloc(engine, i915->kernel_context); + if (IS_ERR(rq)) { + ret = PTR_ERR(rq); + goto out_put; + } + + /* Queue this switch after all other activity by this context. */ + prev = i915_active_request_raw(&ce->ring->timeline->last_request, + &i915->drm.struct_mutex); + if (prev && !i915_request_completed(prev)) { + ret = i915_request_await_dma_fence(rq, &prev->fence); + if (ret < 0) + goto out_add; + } + + /* Order all following requests to be after. */ + ret = i915_timeline_set_barrier(ce->ring->timeline, rq); + if (ret) + goto out_add; + + ret = gen8_emit_rpcs_config(rq, ce, sseu); + if (ret) + goto out_add; + + /* + * Guarantee context image and the timeline remains pinned until the + * modifying request is retired by setting the ce activity tracker. + * + * But we only need to take one pin on the account of it. Or in other + * words transfer the pinned ce object to tracked active request. + */ + if (!i915_active_request_isset(&ce->active_tracker)) + __intel_context_pin(ce); + __i915_active_request_set(&ce->active_tracker, rq); + +out_add: + i915_request_add(rq); +out_put: + intel_runtime_pm_put(i915, wakeref); + + return ret; +} + +static int +__i915_gem_context_reconfigure_sseu(struct i915_gem_context *ctx, + struct intel_engine_cs *engine, + struct intel_sseu sseu) +{ + struct intel_context *ce = to_intel_context(ctx, engine); + int ret = 0; + + GEM_BUG_ON(INTEL_GEN(ctx->i915) < 8); + GEM_BUG_ON(engine->id != RCS); + + /* Nothing to do if unmodified. */ + if (!memcmp(&ce->sseu, &sseu, sizeof(sseu))) + return 0; + + /* + * If context is not idle we have to submit an ordered request to modify + * its context image via the kernel context. Pristine and idle contexts + * will be configured on pinning. + */ + if (ce->pin_count) + ret = gen8_modify_rpcs_gpu(ce, engine, sseu); + + if (!ret) + ce->sseu = sseu; + + return ret; +} + +static int +i915_gem_context_reconfigure_sseu(struct i915_gem_context *ctx, + struct intel_engine_cs *engine, + struct intel_sseu sseu) +{ + int ret; + + ret = mutex_lock_interruptible(&ctx->i915->drm.struct_mutex); + if (ret) + return ret; + + ret = __i915_gem_context_reconfigure_sseu(ctx, engine, sseu); + + mutex_unlock(&ctx->i915->drm.struct_mutex); + + return ret; +} + +static int +user_to_context_sseu(struct drm_i915_private *i915, + const struct drm_i915_gem_context_param_sseu *user, + struct intel_sseu *context) +{ + const struct sseu_dev_info *device = &RUNTIME_INFO(i915)->sseu; + + /* No zeros in any field. */ + if (!user->slice_mask || !user->subslice_mask || + !user->min_eus_per_subslice || !user->max_eus_per_subslice) + return -EINVAL; + + /* Max > min. */ + if (user->max_eus_per_subslice < user->min_eus_per_subslice) + return -EINVAL; + + /* + * Some future proofing on the types since the uAPI is wider than the + * current internal implementation. + */ + if (overflows_type(user->slice_mask, context->slice_mask) || + overflows_type(user->subslice_mask, context->subslice_mask) || + overflows_type(user->min_eus_per_subslice, + context->min_eus_per_subslice) || + overflows_type(user->max_eus_per_subslice, + context->max_eus_per_subslice)) + return -EINVAL; + + /* Check validity against hardware. */ + if (user->slice_mask & ~device->slice_mask) + return -EINVAL; + + if (user->subslice_mask & ~device->subslice_mask[0]) + return -EINVAL; + + if (user->max_eus_per_subslice > device->max_eus_per_subslice) + return -EINVAL; + + context->slice_mask = user->slice_mask; + context->subslice_mask = user->subslice_mask; + context->min_eus_per_subslice = user->min_eus_per_subslice; + context->max_eus_per_subslice = user->max_eus_per_subslice; + + /* Part specific restrictions. */ + if (IS_GEN(i915, 11)) { + unsigned int hw_s = hweight8(device->slice_mask); + unsigned int hw_ss_per_s = hweight8(device->subslice_mask[0]); + unsigned int req_s = hweight8(context->slice_mask); + unsigned int req_ss = hweight8(context->subslice_mask); + + /* + * Only full subslice enablement is possible if more than one + * slice is turned on. + */ + if (req_s > 1 && req_ss != hw_ss_per_s) + return -EINVAL; + + /* + * If more than four (SScount bitfield limit) subslices are + * requested then the number has to be even. + */ + if (req_ss > 4 && (req_ss & 1)) + return -EINVAL; + + /* + * If only one slice is enabled and subslice count is below the + * device full enablement, it must be at most half of the all + * available subslices. + */ + if (req_s == 1 && req_ss < hw_ss_per_s && + req_ss > (hw_ss_per_s / 2)) + return -EINVAL; + + /* ABI restriction - VME use case only. */ + + /* All slices or one slice only. */ + if (req_s != 1 && req_s != hw_s) + return -EINVAL; + + /* + * Half subslices or full enablement only when one slice is + * enabled. + */ + if (req_s == 1 && + (req_ss != hw_ss_per_s && req_ss != (hw_ss_per_s / 2))) + return -EINVAL; + + /* No EU configuration changes. */ + if ((user->min_eus_per_subslice != + device->max_eus_per_subslice) || + (user->max_eus_per_subslice != + device->max_eus_per_subslice)) + return -EINVAL; + } + + return 0; +} + +static int set_sseu(struct i915_gem_context *ctx, + struct drm_i915_gem_context_param *args) +{ + struct drm_i915_private *i915 = ctx->i915; + struct drm_i915_gem_context_param_sseu user_sseu; + struct intel_engine_cs *engine; + struct intel_sseu sseu; + int ret; + + if (args->size < sizeof(user_sseu)) + return -EINVAL; + + if (!IS_GEN(i915, 11)) + return -ENODEV; + + if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value), + sizeof(user_sseu))) + return -EFAULT; + + if (user_sseu.flags || user_sseu.rsvd) + return -EINVAL; + + engine = intel_engine_lookup_user(i915, + user_sseu.engine_class, + user_sseu.engine_instance); + if (!engine) + return -EINVAL; + + /* Only render engine supports RPCS configuration. */ + if (engine->class != RENDER_CLASS) + return -ENODEV; + + ret = user_to_context_sseu(i915, &user_sseu, &sseu); + if (ret) + return ret; + + ret = i915_gem_context_reconfigure_sseu(ctx, engine, sseu); + if (ret) + return ret; + + args->size = sizeof(user_sseu); + + return 0; +} + int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { @@ -948,10 +1300,13 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data, !capable(CAP_SYS_NICE)) ret = -EPERM; else - ctx->sched.priority = priority; + ctx->sched.priority = + I915_USER_PRIORITY(priority); } break; - + case I915_CONTEXT_PARAM_SSEU: + ret = set_sseu(ctx, args); + break; default: ret = -EINVAL; break; diff --git a/drivers/gpu/drm/i915/i915_gem_context.h b/drivers/gpu/drm/i915/i915_gem_context.h index 08165f6a0a84..ca150a764c24 100644 --- a/drivers/gpu/drm/i915/i915_gem_context.h +++ b/drivers/gpu/drm/i915/i915_gem_context.h @@ -31,6 +31,7 @@ #include "i915_gem.h" #include "i915_scheduler.h" +#include "intel_device_info.h" struct pid; @@ -53,6 +54,16 @@ struct intel_context_ops { void (*destroy)(struct intel_context *ce); }; +/* + * Powergating configuration for a particular (context,engine). + */ +struct intel_sseu { + u8 slice_mask; + u8 subslice_mask; + u8 min_eus_per_subslice; + u8 max_eus_per_subslice; +}; + /** * struct i915_gem_context - client state * @@ -163,13 +174,25 @@ struct i915_gem_context { /** engine: per-engine logical HW state */ struct intel_context { struct i915_gem_context *gem_context; + struct intel_engine_cs *active; + struct list_head signal_link; + struct list_head signals; struct i915_vma *state; struct intel_ring *ring; u32 *lrc_reg_state; u64 lrc_desc; int pin_count; + /** + * active_tracker: Active tracker for the external rq activity + * on this intel_context object. + */ + struct i915_active_request active_tracker; + const struct intel_context_ops *ops; + + /** sseu: Control eu/slice partitioning */ + struct intel_sseu sseu; } __engine[I915_NUM_ENGINES]; /** ring_size: size for allocating the per-engine ring buffer */ @@ -363,4 +386,8 @@ static inline void i915_gem_context_put(struct i915_gem_context *ctx) kref_put(&ctx->ref, i915_gem_context_release); } +void intel_context_init(struct intel_context *ce, + struct i915_gem_context *ctx, + struct intel_engine_cs *engine); + #endif /* !__I915_GEM_CONTEXT_H__ */ diff --git a/drivers/gpu/drm/i915/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/i915_gem_dmabuf.c index 82e2ca17a441..02f7298bfe57 100644 --- a/drivers/gpu/drm/i915/i915_gem_dmabuf.c +++ b/drivers/gpu/drm/i915/i915_gem_dmabuf.c @@ -27,7 +27,6 @@ #include <linux/dma-buf.h> #include <linux/reservation.h> -#include <drm/drmP.h> #include "i915_drv.h" diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c index 02b83a5ed96c..68d74c50ac39 100644 --- a/drivers/gpu/drm/i915/i915_gem_evict.c +++ b/drivers/gpu/drm/i915/i915_gem_evict.c @@ -26,7 +26,6 @@ * */ -#include <drm/drmP.h> #include <drm/i915_drm.h> #include "i915_drv.h" @@ -127,31 +126,25 @@ i915_gem_evict_something(struct i915_address_space *vm, struct drm_i915_private *dev_priv = vm->i915; struct drm_mm_scan scan; struct list_head eviction_list; - struct list_head *phases[] = { - &vm->inactive_list, - &vm->active_list, - NULL, - }, **phase; struct i915_vma *vma, *next; struct drm_mm_node *node; enum drm_mm_insert_mode mode; + struct i915_vma *active; int ret; lockdep_assert_held(&vm->i915->drm.struct_mutex); trace_i915_gem_evict(vm, min_size, alignment, flags); /* - * The goal is to evict objects and amalgamate space in LRU order. - * The oldest idle objects reside on the inactive list, which is in - * retirement order. The next objects to retire are those in flight, - * on the active list, again in retirement order. + * The goal is to evict objects and amalgamate space in rough LRU order. + * Since both active and inactive objects reside on the same list, + * in a mix of creation and last scanned order, as we process the list + * we sort it into inactive/active, which keeps the active portion + * in a rough MRU order. * * The retirement sequence is thus: - * 1. Inactive objects (already retired) - * 2. Active objects (will stall on unbinding) - * - * On each list, the oldest objects lie at the HEAD with the freshest - * object on the TAIL. + * 1. Inactive objects (already retired, random order) + * 2. Active objects (will stall on unbinding, oldest scanned first) */ mode = DRM_MM_INSERT_BEST; if (flags & PIN_HIGH) @@ -170,17 +163,46 @@ i915_gem_evict_something(struct i915_address_space *vm, */ if (!(flags & PIN_NONBLOCK)) i915_retire_requests(dev_priv); - else - phases[1] = NULL; search_again: + active = NULL; INIT_LIST_HEAD(&eviction_list); - phase = phases; - do { - list_for_each_entry(vma, *phase, vm_link) - if (mark_free(&scan, vma, flags, &eviction_list)) - goto found; - } while (*++phase); + list_for_each_entry_safe(vma, next, &vm->bound_list, vm_link) { + /* + * We keep this list in a rough least-recently scanned order + * of active elements (inactive elements are cheap to reap). + * New entries are added to the end, and we move anything we + * scan to the end. The assumption is that the working set + * of applications is either steady state (and thanks to the + * userspace bo cache it almost always is) or volatile and + * frequently replaced after a frame, which are self-evicting! + * Given that assumption, the MRU order of the scan list is + * fairly static, and keeping it in least-recently scan order + * is suitable. + * + * To notice when we complete one full cycle, we record the + * first active element seen, before moving it to the tail. + */ + if (i915_vma_is_active(vma)) { + if (vma == active) { + if (flags & PIN_NONBLOCK) + break; + + active = ERR_PTR(-EAGAIN); + } + + if (active != ERR_PTR(-EAGAIN)) { + if (!active) + active = vma; + + list_move_tail(&vma->vm_link, &vm->bound_list); + continue; + } + } + + if (mark_free(&scan, vma, flags, &eviction_list)) + goto found; + } /* Nothing found, clean up and bail out! */ list_for_each_entry_safe(vma, next, &eviction_list, evict_link) { @@ -389,11 +411,6 @@ int i915_gem_evict_for_node(struct i915_address_space *vm, */ int i915_gem_evict_vm(struct i915_address_space *vm) { - struct list_head *phases[] = { - &vm->inactive_list, - &vm->active_list, - NULL - }, **phase; struct list_head eviction_list; struct i915_vma *vma, *next; int ret; @@ -413,16 +430,15 @@ int i915_gem_evict_vm(struct i915_address_space *vm) } INIT_LIST_HEAD(&eviction_list); - phase = phases; - do { - list_for_each_entry(vma, *phase, vm_link) { - if (i915_vma_is_pinned(vma)) - continue; + mutex_lock(&vm->mutex); + list_for_each_entry(vma, &vm->bound_list, vm_link) { + if (i915_vma_is_pinned(vma)) + continue; - __i915_vma_pin(vma); - list_add(&vma->evict_link, &eviction_list); - } - } while (*++phase); + __i915_vma_pin(vma); + list_add(&vma->evict_link, &eviction_list); + } + mutex_unlock(&vm->mutex); ret = 0; list_for_each_entry_safe(vma, next, &eviction_list, evict_link) { diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c index 1aaccbe7e1de..02adcaf6ebea 100644 --- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c @@ -26,12 +26,11 @@ * */ -#include <linux/dma_remapping.h> +#include <linux/intel-iommu.h> #include <linux/reservation.h> #include <linux/sync_file.h> #include <linux/uaccess.h> -#include <drm/drmP.h> #include <drm/drm_syncobj.h> #include <drm/i915_drm.h> @@ -754,6 +753,68 @@ static int eb_select_context(struct i915_execbuffer *eb) return 0; } +static struct i915_request *__eb_wait_for_ring(struct intel_ring *ring) +{ + struct i915_request *rq; + + /* + * Completely unscientific finger-in-the-air estimates for suitable + * maximum user request size (to avoid blocking) and then backoff. + */ + if (intel_ring_update_space(ring) >= PAGE_SIZE) + return NULL; + + /* + * Find a request that after waiting upon, there will be at least half + * the ring available. The hysteresis allows us to compete for the + * shared ring and should mean that we sleep less often prior to + * claiming our resources, but not so long that the ring completely + * drains before we can submit our next request. + */ + list_for_each_entry(rq, &ring->request_list, ring_link) { + if (__intel_ring_space(rq->postfix, + ring->emit, ring->size) > ring->size / 2) + break; + } + if (&rq->ring_link == &ring->request_list) + return NULL; /* weird, we will check again later for real */ + + return i915_request_get(rq); +} + +static int eb_wait_for_ring(const struct i915_execbuffer *eb) +{ + const struct intel_context *ce; + struct i915_request *rq; + int ret = 0; + + /* + * Apply a light amount of backpressure to prevent excessive hogs + * from blocking waiting for space whilst holding struct_mutex and + * keeping all of their resources pinned. + */ + + ce = to_intel_context(eb->ctx, eb->engine); + if (!ce->ring) /* first use, assume empty! */ + return 0; + + rq = __eb_wait_for_ring(ce->ring); + if (rq) { + mutex_unlock(&eb->i915->drm.struct_mutex); + + if (i915_request_wait(rq, + I915_WAIT_INTERRUPTIBLE, + MAX_SCHEDULE_TIMEOUT) < 0) + ret = -EINTR; + + i915_request_put(rq); + + mutex_lock(&eb->i915->drm.struct_mutex); + } + + return ret; +} + static int eb_lookup_vmas(struct i915_execbuffer *eb) { struct radix_tree_root *handles_vma = &eb->ctx->handles_vma; @@ -1380,7 +1441,7 @@ eb_relocate_entry(struct i915_execbuffer *eb, * batchbuffers. */ if (reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION && - IS_GEN6(eb->i915)) { + IS_GEN(eb->i915, 6)) { err = i915_vma_bind(target, target->obj->cache_level, PIN_GLOBAL); if (WARN_ONCE(err, @@ -1447,7 +1508,7 @@ static int eb_relocate_vma(struct i915_execbuffer *eb, struct i915_vma *vma) * to read. However, if the array is not writable the user loses * the updated relocation values. */ - if (unlikely(!access_ok(VERIFY_READ, urelocs, remain*sizeof(*urelocs)))) + if (unlikely(!access_ok(urelocs, remain*sizeof(*urelocs)))) return -EFAULT; do { @@ -1554,7 +1615,7 @@ static int check_relocations(const struct drm_i915_gem_exec_object2 *entry) addr = u64_to_user_ptr(entry->relocs_ptr); size *= sizeof(struct drm_i915_gem_relocation_entry); - if (!access_ok(VERIFY_READ, addr, size)) + if (!access_ok(addr, size)) return -EFAULT; end = addr + size; @@ -1605,6 +1666,7 @@ static int eb_copy_relocations(const struct i915_execbuffer *eb) (char __user *)urelocs + copied, len)) { end_user: + user_access_end(); kvfree(relocs); err = -EFAULT; goto err; @@ -1623,7 +1685,9 @@ end_user: * happened we would make the mistake of assuming that the * relocations were valid. */ - user_access_begin(); + if (!user_access_begin(urelocs, size)) + goto end_user; + for (copied = 0; copied < nreloc; copied++) unsafe_put_user(-1, &urelocs[copied].presumed_offset, @@ -1893,7 +1957,7 @@ static int i915_reset_gen7_sol_offsets(struct i915_request *rq) u32 *cs; int i; - if (!IS_GEN7(rq->i915) || rq->engine->id != RCS) { + if (!IS_GEN(rq->i915, 7) || rq->engine->id != RCS) { DRM_DEBUG("sol reset is gen7/rcs only\n"); return -EINVAL; } @@ -1974,6 +2038,18 @@ static int eb_submit(struct i915_execbuffer *eb) return err; } + /* + * After we completed waiting for other engines (using HW semaphores) + * then we can signal that this request/batch is ready to run. This + * allows us to determine if the batch is still waiting on the GPU + * or actually running by checking the breadcrumb. + */ + if (eb->engine->emit_init_breadcrumb) { + err = eb->engine->emit_init_breadcrumb(eb->request); + if (err) + return err; + } + err = eb->engine->emit_bb_start(eb->request, eb->batch->node.start + eb->batch_start_offset, @@ -2090,7 +2166,7 @@ get_fence_array(struct drm_i915_gem_execbuffer2 *args, return ERR_PTR(-EINVAL); user = u64_to_user_ptr(args->cliprects_ptr); - if (!access_ok(VERIFY_READ, user, nfences * sizeof(*user))) + if (!access_ok(user, nfences * sizeof(*user))) return ERR_PTR(-EFAULT); fences = kvmalloc_array(nfences, sizeof(*fences), @@ -2186,7 +2262,7 @@ signal_fence_array(struct i915_execbuffer *eb, if (!(flags & I915_EXEC_FENCE_SIGNAL)) continue; - drm_syncobj_replace_fence(syncobj, 0, fence); + drm_syncobj_replace_fence(syncobj, fence); } } @@ -2200,6 +2276,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, struct i915_execbuffer eb; struct dma_fence *in_fence = NULL; struct sync_file *out_fence = NULL; + intel_wakeref_t wakeref; int out_fence_fd = -1; int err; @@ -2270,12 +2347,16 @@ i915_gem_do_execbuffer(struct drm_device *dev, * wakeref that we hold until the GPU has been idle for at least * 100ms. */ - intel_runtime_pm_get(eb.i915); + wakeref = intel_runtime_pm_get(eb.i915); err = i915_mutex_lock_interruptible(dev); if (err) goto err_rpm; + err = eb_wait_for_ring(&eb); /* may temporarily drop struct_mutex */ + if (unlikely(err)) + goto err_unlock; + err = eb_relocate(&eb); if (err) { /* @@ -2420,9 +2501,10 @@ err_batch_unpin: err_vma: if (eb.exec) eb_release_vmas(&eb); +err_unlock: mutex_unlock(&dev->struct_mutex); err_rpm: - intel_runtime_pm_put(eb.i915); + intel_runtime_pm_put(eb.i915, wakeref); i915_gem_context_put(eb.ctx); err_destroy: eb_destroy(&eb); @@ -2605,7 +2687,16 @@ i915_gem_execbuffer2_ioctl(struct drm_device *dev, void *data, unsigned int i; /* Copy the new buffer offsets back to the user's exec list. */ - user_access_begin(); + /* + * Note: count * sizeof(*user_exec_list) does not overflow, + * because we checked 'count' in check_buffer_count(). + * + * And this range already got effectively checked earlier + * when we did the "copy_from_user()" above. + */ + if (!user_access_begin(user_exec_list, count * sizeof(*user_exec_list))) + goto end_user; + for (i = 0; i < args->buffer_count; i++) { if (!(exec2_list[i].offset & UPDATE)) continue; diff --git a/drivers/gpu/drm/i915/i915_gem_fence_reg.c b/drivers/gpu/drm/i915/i915_gem_fence_reg.c index d548ac05ccd7..e037e94792f3 100644 --- a/drivers/gpu/drm/i915/i915_gem_fence_reg.c +++ b/drivers/gpu/drm/i915/i915_gem_fence_reg.c @@ -21,7 +21,6 @@ * IN THE SOFTWARE. */ -#include <drm/drmP.h> #include <drm/i915_drm.h> #include "i915_drv.h" @@ -193,9 +192,9 @@ static void fence_write(struct drm_i915_fence_reg *fence, * and explicitly managed for internal users. */ - if (IS_GEN2(fence->i915)) + if (IS_GEN(fence->i915, 2)) i830_write_fence_reg(fence, vma); - else if (IS_GEN3(fence->i915)) + else if (IS_GEN(fence->i915, 3)) i915_write_fence_reg(fence, vma); else i965_write_fence_reg(fence, vma); @@ -210,6 +209,7 @@ static void fence_write(struct drm_i915_fence_reg *fence, static int fence_update(struct drm_i915_fence_reg *fence, struct i915_vma *vma) { + intel_wakeref_t wakeref; int ret; if (vma) { @@ -223,7 +223,7 @@ static int fence_update(struct drm_i915_fence_reg *fence, i915_gem_object_get_tiling(vma->obj))) return -EINVAL; - ret = i915_gem_active_retire(&vma->last_fence, + ret = i915_active_request_retire(&vma->last_fence, &vma->obj->base.dev->struct_mutex); if (ret) return ret; @@ -232,7 +232,7 @@ static int fence_update(struct drm_i915_fence_reg *fence, if (fence->vma) { struct i915_vma *old = fence->vma; - ret = i915_gem_active_retire(&old->last_fence, + ret = i915_active_request_retire(&old->last_fence, &old->obj->base.dev->struct_mutex); if (ret) return ret; @@ -257,9 +257,10 @@ static int fence_update(struct drm_i915_fence_reg *fence, * If the device is currently powered down, we will defer the write * to the runtime resume, see i915_gem_restore_fences(). */ - if (intel_runtime_pm_get_if_in_use(fence->i915)) { + wakeref = intel_runtime_pm_get_if_in_use(fence->i915); + if (wakeref) { fence_write(fence, vma); - intel_runtime_pm_put(fence->i915); + intel_runtime_pm_put(fence->i915, wakeref); } if (vma) { @@ -554,8 +555,8 @@ void i915_gem_restore_fences(struct drm_i915_private *dev_priv) void i915_gem_detect_bit_6_swizzle(struct drm_i915_private *dev_priv) { - uint32_t swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN; - uint32_t swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN; + u32 swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN; + u32 swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN; if (INTEL_GEN(dev_priv) >= 8 || IS_VALLEYVIEW(dev_priv)) { /* @@ -578,7 +579,7 @@ i915_gem_detect_bit_6_swizzle(struct drm_i915_private *dev_priv) swizzle_y = I915_BIT_6_SWIZZLE_NONE; } } else { - uint32_t dimm_c0, dimm_c1; + u32 dimm_c0, dimm_c1; dimm_c0 = I915_READ(MAD_DIMM_C0); dimm_c1 = I915_READ(MAD_DIMM_C1); dimm_c0 &= MAD_DIMM_A_SIZE_MASK | MAD_DIMM_B_SIZE_MASK; @@ -596,13 +597,13 @@ i915_gem_detect_bit_6_swizzle(struct drm_i915_private *dev_priv) swizzle_y = I915_BIT_6_SWIZZLE_NONE; } } - } else if (IS_GEN5(dev_priv)) { + } else if (IS_GEN(dev_priv, 5)) { /* On Ironlake whatever DRAM config, GPU always do * same swizzling setup. */ swizzle_x = I915_BIT_6_SWIZZLE_9_10; swizzle_y = I915_BIT_6_SWIZZLE_9; - } else if (IS_GEN2(dev_priv)) { + } else if (IS_GEN(dev_priv, 2)) { /* As far as we know, the 865 doesn't have these bit 6 * swizzling issues. */ @@ -610,7 +611,7 @@ i915_gem_detect_bit_6_swizzle(struct drm_i915_private *dev_priv) swizzle_y = I915_BIT_6_SWIZZLE_NONE; } else if (IS_MOBILE(dev_priv) || IS_I915G(dev_priv) || IS_I945G(dev_priv)) { - uint32_t dcc; + u32 dcc; /* On 9xx chipsets, channel interleave by the CPU is * determined by DCC. For single-channel, neither the CPU @@ -647,7 +648,7 @@ i915_gem_detect_bit_6_swizzle(struct drm_i915_private *dev_priv) } /* check for L-shaped memory aka modified enhanced addressing */ - if (IS_GEN4(dev_priv) && + if (IS_GEN(dev_priv, 4) && !(I915_READ(DCC2) & DCC2_MODIFIED_ENHANCED_DISABLE)) { swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN; swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN; diff --git a/drivers/gpu/drm/i915/i915_gem_fence_reg.h b/drivers/gpu/drm/i915/i915_gem_fence_reg.h index 99a31ded4dfd..09dcaf14121b 100644 --- a/drivers/gpu/drm/i915/i915_gem_fence_reg.h +++ b/drivers/gpu/drm/i915/i915_gem_fence_reg.h @@ -50,4 +50,3 @@ struct drm_i915_fence_reg { }; #endif - diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index 07999fe09ad2..d646d37eec2f 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -33,11 +33,11 @@ #include <asm/set_memory.h> -#include <drm/drmP.h> #include <drm/i915_drm.h> #include "i915_drv.h" #include "i915_vgpu.h" +#include "i915_reset.h" #include "i915_trace.h" #include "intel_drv.h" #include "intel_frontbuffer.h" @@ -133,55 +133,6 @@ static inline void i915_ggtt_invalidate(struct drm_i915_private *i915) i915->ggtt.invalidate(i915); } -int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv, - int enable_ppgtt) -{ - bool has_full_ppgtt; - bool has_full_48bit_ppgtt; - - if (!dev_priv->info.has_aliasing_ppgtt) - return 0; - - has_full_ppgtt = dev_priv->info.has_full_ppgtt; - has_full_48bit_ppgtt = dev_priv->info.has_full_48bit_ppgtt; - - if (intel_vgpu_active(dev_priv)) { - /* GVT-g has no support for 32bit ppgtt */ - has_full_ppgtt = false; - has_full_48bit_ppgtt = intel_vgpu_has_full_48bit_ppgtt(dev_priv); - } - - /* - * We don't allow disabling PPGTT for gen9+ as it's a requirement for - * execlists, the sole mechanism available to submit work. - */ - if (enable_ppgtt == 0 && INTEL_GEN(dev_priv) < 9) - return 0; - - if (enable_ppgtt == 1) - return 1; - - if (enable_ppgtt == 2 && has_full_ppgtt) - return 2; - - if (enable_ppgtt == 3 && has_full_48bit_ppgtt) - return 3; - - /* Disable ppgtt on SNB if VT-d is on. */ - if (IS_GEN6(dev_priv) && intel_vtd_active()) { - DRM_INFO("Disabling PPGTT because VT-d is on\n"); - return 0; - } - - if (has_full_48bit_ppgtt) - return 3; - - if (has_full_ppgtt) - return 2; - - return 1; -} - static int ppgtt_bind_vma(struct i915_vma *vma, enum i915_cache_level cache_level, u32 unused) @@ -235,9 +186,9 @@ static void clear_pages(struct i915_vma *vma) memset(&vma->page_sizes, 0, sizeof(vma->page_sizes)); } -static gen8_pte_t gen8_pte_encode(dma_addr_t addr, - enum i915_cache_level level, - u32 flags) +static u64 gen8_pte_encode(dma_addr_t addr, + enum i915_cache_level level, + u32 flags) { gen8_pte_t pte = addr | _PAGE_PRESENT | _PAGE_RW; @@ -274,9 +225,9 @@ static gen8_pde_t gen8_pde_encode(const dma_addr_t addr, #define gen8_pdpe_encode gen8_pde_encode #define gen8_pml4e_encode gen8_pde_encode -static gen6_pte_t snb_pte_encode(dma_addr_t addr, - enum i915_cache_level level, - u32 unused) +static u64 snb_pte_encode(dma_addr_t addr, + enum i915_cache_level level, + u32 flags) { gen6_pte_t pte = GEN6_PTE_VALID; pte |= GEN6_PTE_ADDR_ENCODE(addr); @@ -296,9 +247,9 @@ static gen6_pte_t snb_pte_encode(dma_addr_t addr, return pte; } -static gen6_pte_t ivb_pte_encode(dma_addr_t addr, - enum i915_cache_level level, - u32 unused) +static u64 ivb_pte_encode(dma_addr_t addr, + enum i915_cache_level level, + u32 flags) { gen6_pte_t pte = GEN6_PTE_VALID; pte |= GEN6_PTE_ADDR_ENCODE(addr); @@ -320,9 +271,9 @@ static gen6_pte_t ivb_pte_encode(dma_addr_t addr, return pte; } -static gen6_pte_t byt_pte_encode(dma_addr_t addr, - enum i915_cache_level level, - u32 flags) +static u64 byt_pte_encode(dma_addr_t addr, + enum i915_cache_level level, + u32 flags) { gen6_pte_t pte = GEN6_PTE_VALID; pte |= GEN6_PTE_ADDR_ENCODE(addr); @@ -336,9 +287,9 @@ static gen6_pte_t byt_pte_encode(dma_addr_t addr, return pte; } -static gen6_pte_t hsw_pte_encode(dma_addr_t addr, - enum i915_cache_level level, - u32 unused) +static u64 hsw_pte_encode(dma_addr_t addr, + enum i915_cache_level level, + u32 flags) { gen6_pte_t pte = GEN6_PTE_VALID; pte |= HSW_PTE_ADDR_ENCODE(addr); @@ -349,9 +300,9 @@ static gen6_pte_t hsw_pte_encode(dma_addr_t addr, return pte; } -static gen6_pte_t iris_pte_encode(dma_addr_t addr, - enum i915_cache_level level, - u32 unused) +static u64 iris_pte_encode(dma_addr_t addr, + enum i915_cache_level level, + u32 flags) { gen6_pte_t pte = GEN6_PTE_VALID; pte |= HSW_PTE_ADDR_ENCODE(addr); @@ -523,8 +474,7 @@ static void vm_free_page(struct i915_address_space *vm, struct page *page) spin_unlock(&vm->free_pages.lock); } -static void i915_address_space_init(struct i915_address_space *vm, - struct drm_i915_private *dev_priv) +static void i915_address_space_init(struct i915_address_space *vm, int subclass) { /* * The vm->mutex must be reclaim safe (for use in the shrinker). @@ -532,7 +482,8 @@ static void i915_address_space_init(struct i915_address_space *vm, * attempt holding the lock is immediately reported by lockdep. */ mutex_init(&vm->mutex); - i915_gem_shrinker_taints_mutex(&vm->mutex); + lockdep_set_subclass(&vm->mutex, subclass); + i915_gem_shrinker_taints_mutex(vm->i915, &vm->mutex); GEM_BUG_ON(!vm->total); drm_mm_init(&vm->mm, 0, vm->total); @@ -540,9 +491,8 @@ static void i915_address_space_init(struct i915_address_space *vm, stash_init(&vm->free_pages); - INIT_LIST_HEAD(&vm->active_list); - INIT_LIST_HEAD(&vm->inactive_list); INIT_LIST_HEAD(&vm->unbound_list); + INIT_LIST_HEAD(&vm->bound_list); } static void i915_address_space_fini(struct i915_address_space *vm) @@ -629,10 +579,9 @@ setup_scratch_page(struct i915_address_space *vm, gfp_t gfp) * region, including any PTEs which happen to point to scratch. * * This is only relevant for the 48b PPGTT where we support - * huge-gtt-pages, see also i915_vma_insert(). - * - * TODO: we should really consider write-protecting the scratch-page and - * sharing between ppgtt + * huge-gtt-pages, see also i915_vma_insert(). However, as we share the + * scratch (read-only) between all vm, we create one 64k scratch page + * for all. */ size = I915_GTT_PAGE_SIZE_4K; if (i915_vm_is_48bit(vm) && @@ -715,14 +664,13 @@ static void free_pt(struct i915_address_space *vm, struct i915_page_table *pt) static void gen8_initialize_pt(struct i915_address_space *vm, struct i915_page_table *pt) { - fill_px(vm, pt, - gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC, 0)); + fill_px(vm, pt, vm->scratch_pte); } -static void gen6_initialize_pt(struct gen6_hw_ppgtt *ppgtt, +static void gen6_initialize_pt(struct i915_address_space *vm, struct i915_page_table *pt) { - fill32_px(&ppgtt->base.vm, pt, ppgtt->scratch_pte); + fill32_px(vm, pt, vm->scratch_pte); } static struct i915_page_directory *alloc_pd(struct i915_address_space *vm) @@ -856,15 +804,13 @@ static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt) /* Removes entries from a single page table, releasing it if it's empty. * Caller can use the return value to update higher-level entries. */ -static bool gen8_ppgtt_clear_pt(struct i915_address_space *vm, +static bool gen8_ppgtt_clear_pt(const struct i915_address_space *vm, struct i915_page_table *pt, u64 start, u64 length) { unsigned int num_entries = gen8_pte_count(start, length); unsigned int pte = gen8_pte_index(start); unsigned int pte_end = pte + num_entries; - const gen8_pte_t scratch_pte = - gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC, 0); gen8_pte_t *vaddr; GEM_BUG_ON(num_entries > pt->used_ptes); @@ -875,7 +821,7 @@ static bool gen8_ppgtt_clear_pt(struct i915_address_space *vm, vaddr = kmap_atomic_px(pt); while (pte < pte_end) - vaddr[pte++] = scratch_pte; + vaddr[pte++] = vm->scratch_pte; kunmap_atomic(vaddr); return false; @@ -1208,7 +1154,7 @@ static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma, if (I915_SELFTEST_ONLY(vma->vm->scrub_64K)) { u16 i; - encode = pte_encode | vma->vm->scratch_page.daddr; + encode = vma->vm->scratch_pte; vaddr = kmap_atomic_px(pd->page_table[idx.pde]); for (i = 1; i < index; i += 16) @@ -1261,10 +1207,35 @@ static int gen8_init_scratch(struct i915_address_space *vm) { int ret; + /* + * If everybody agrees to not to write into the scratch page, + * we can reuse it for all vm, keeping contexts and processes separate. + */ + if (vm->has_read_only && + vm->i915->kernel_context && + vm->i915->kernel_context->ppgtt) { + struct i915_address_space *clone = + &vm->i915->kernel_context->ppgtt->vm; + + GEM_BUG_ON(!clone->has_read_only); + + vm->scratch_page.order = clone->scratch_page.order; + vm->scratch_pte = clone->scratch_pte; + vm->scratch_pt = clone->scratch_pt; + vm->scratch_pd = clone->scratch_pd; + vm->scratch_pdp = clone->scratch_pdp; + return 0; + } + ret = setup_scratch_page(vm, __GFP_HIGHMEM); if (ret) return ret; + vm->scratch_pte = + gen8_pte_encode(vm->scratch_page.daddr, + I915_CACHE_LLC, + PTE_READ_ONLY); + vm->scratch_pt = alloc_pt(vm); if (IS_ERR(vm->scratch_pt)) { ret = PTR_ERR(vm->scratch_pt); @@ -1336,6 +1307,9 @@ static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create) static void gen8_free_scratch(struct i915_address_space *vm) { + if (!vm->scratch_page.daddr) + return; + if (use_4lvl(vm)) free_pdp(vm, vm->scratch_pdp); free_pd(vm, vm->scratch_pd); @@ -1448,8 +1422,6 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm, gen8_initialize_pd(vm, pd); gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe); GEM_BUG_ON(pdp->used_pdpes > i915_pdpes_per_pdp(vm)); - - mark_tlbs_dirty(i915_vm_to_ppgtt(vm)); } ret = gen8_ppgtt_alloc_pd(vm, pd, start, length); @@ -1515,85 +1487,6 @@ unwind: return -ENOMEM; } -static void gen8_dump_pdp(struct i915_hw_ppgtt *ppgtt, - struct i915_page_directory_pointer *pdp, - u64 start, u64 length, - gen8_pte_t scratch_pte, - struct seq_file *m) -{ - struct i915_address_space *vm = &ppgtt->vm; - struct i915_page_directory *pd; - u32 pdpe; - - gen8_for_each_pdpe(pd, pdp, start, length, pdpe) { - struct i915_page_table *pt; - u64 pd_len = length; - u64 pd_start = start; - u32 pde; - - if (pdp->page_directory[pdpe] == ppgtt->vm.scratch_pd) - continue; - - seq_printf(m, "\tPDPE #%d\n", pdpe); - gen8_for_each_pde(pt, pd, pd_start, pd_len, pde) { - u32 pte; - gen8_pte_t *pt_vaddr; - - if (pd->page_table[pde] == ppgtt->vm.scratch_pt) - continue; - - pt_vaddr = kmap_atomic_px(pt); - for (pte = 0; pte < GEN8_PTES; pte += 4) { - u64 va = (pdpe << GEN8_PDPE_SHIFT | - pde << GEN8_PDE_SHIFT | - pte << GEN8_PTE_SHIFT); - int i; - bool found = false; - - for (i = 0; i < 4; i++) - if (pt_vaddr[pte + i] != scratch_pte) - found = true; - if (!found) - continue; - - seq_printf(m, "\t\t0x%llx [%03d,%03d,%04d]: =", va, pdpe, pde, pte); - for (i = 0; i < 4; i++) { - if (pt_vaddr[pte + i] != scratch_pte) - seq_printf(m, " %llx", pt_vaddr[pte + i]); - else - seq_puts(m, " SCRATCH "); - } - seq_puts(m, "\n"); - } - kunmap_atomic(pt_vaddr); - } - } -} - -static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) -{ - struct i915_address_space *vm = &ppgtt->vm; - const gen8_pte_t scratch_pte = - gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC, 0); - u64 start = 0, length = ppgtt->vm.total; - - if (use_4lvl(vm)) { - u64 pml4e; - struct i915_pml4 *pml4 = &ppgtt->pml4; - struct i915_page_directory_pointer *pdp; - - gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) { - if (pml4->pdps[pml4e] == ppgtt->vm.scratch_pdp) - continue; - - seq_printf(m, " PML4E #%llu\n", pml4e); - gen8_dump_pdp(ppgtt, pdp, start, length, scratch_pte, m); - } - } else { - gen8_dump_pdp(ppgtt, &ppgtt->pdp, start, length, scratch_pte, m); - } -} - static int gen8_preallocate_top_level_pdp(struct i915_hw_ppgtt *ppgtt) { struct i915_address_space *vm = &ppgtt->vm; @@ -1647,18 +1540,14 @@ static struct i915_hw_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915) ppgtt->vm.i915 = i915; ppgtt->vm.dma = &i915->drm.pdev->dev; - ppgtt->vm.total = USES_FULL_48BIT_PPGTT(i915) ? + ppgtt->vm.total = HAS_FULL_48BIT_PPGTT(i915) ? 1ULL << 48 : 1ULL << 32; - /* - * From bdw, there is support for read-only pages in the PPGTT. - * - * XXX GVT is not honouring the lack of RW in the PTE bits. - */ - ppgtt->vm.has_read_only = !intel_vgpu_active(i915); + /* From bdw, there is support for read-only pages in the PPGTT. */ + ppgtt->vm.has_read_only = true; - i915_address_space_init(&ppgtt->vm, i915); + i915_address_space_init(&ppgtt->vm, VM_CLASS_PPGTT); /* There are only few exceptions for gen >=6. chv and bxt. * And we are not sure about the latter so play safe for now. @@ -1702,7 +1591,6 @@ static struct i915_hw_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915) gen8_ppgtt_notify_vgt(ppgtt, true); ppgtt->vm.cleanup = gen8_ppgtt_cleanup; - ppgtt->debug_dump = gen8_dump_ppgtt; ppgtt->vm.vma_ops.bind_vma = ppgtt_bind_vma; ppgtt->vm.vma_ops.unbind_vma = ppgtt_unbind_vma; @@ -1718,60 +1606,6 @@ err_free: return ERR_PTR(err); } -static void gen6_dump_ppgtt(struct i915_hw_ppgtt *base, struct seq_file *m) -{ - struct gen6_hw_ppgtt *ppgtt = to_gen6_ppgtt(base); - const gen6_pte_t scratch_pte = ppgtt->scratch_pte; - struct i915_page_table *pt; - u32 pte, pde; - - gen6_for_all_pdes(pt, &base->pd, pde) { - gen6_pte_t *vaddr; - - if (pt == base->vm.scratch_pt) - continue; - - if (i915_vma_is_bound(ppgtt->vma, I915_VMA_GLOBAL_BIND)) { - u32 expected = - GEN6_PDE_ADDR_ENCODE(px_dma(pt)) | - GEN6_PDE_VALID; - u32 pd_entry = readl(ppgtt->pd_addr + pde); - - if (pd_entry != expected) - seq_printf(m, - "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n", - pde, - pd_entry, - expected); - - seq_printf(m, "\tPDE: %x\n", pd_entry); - } - - vaddr = kmap_atomic_px(base->pd.page_table[pde]); - for (pte = 0; pte < GEN6_PTES; pte += 4) { - int i; - - for (i = 0; i < 4; i++) - if (vaddr[pte + i] != scratch_pte) - break; - if (i == 4) - continue; - - seq_printf(m, "\t\t(%03d, %04d) %08llx: ", - pde, pte, - (pde * GEN6_PTES + pte) * I915_GTT_PAGE_SIZE); - for (i = 0; i < 4; i++) { - if (vaddr[pte + i] != scratch_pte) - seq_printf(m, " %08x", vaddr[pte + i]); - else - seq_puts(m, " SCRATCH"); - } - seq_puts(m, "\n"); - } - kunmap_atomic(vaddr); - } -} - /* Write pde (index) from the page directory @pd to the page table @pt */ static inline void gen6_write_pde(const struct gen6_hw_ppgtt *ppgtt, const unsigned int pde, @@ -1782,19 +1616,6 @@ static inline void gen6_write_pde(const struct gen6_hw_ppgtt *ppgtt, ppgtt->pd_addr + pde); } -static void gen8_ppgtt_enable(struct drm_i915_private *dev_priv) -{ - struct intel_engine_cs *engine; - enum intel_engine_id id; - - for_each_engine(engine, dev_priv, id) { - u32 four_level = USES_FULL_48BIT_PPGTT(dev_priv) ? - GEN8_GFX_PPGTT_48B : 0; - I915_WRITE(RING_MODE_GEN7(engine), - _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level)); - } -} - static void gen7_ppgtt_enable(struct drm_i915_private *dev_priv) { struct intel_engine_cs *engine; @@ -1834,7 +1655,8 @@ static void gen6_ppgtt_enable(struct drm_i915_private *dev_priv) ecochk = I915_READ(GAM_ECOCHK); I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B); - I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); + if (HAS_PPGTT(dev_priv)) /* may be disabled for VT-d */ + I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); } /* PPGTT support for Sandybdrige/Gen6 and later */ @@ -1846,7 +1668,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm, unsigned int pde = first_entry / GEN6_PTES; unsigned int pte = first_entry % GEN6_PTES; unsigned int num_entries = length / I915_GTT_PAGE_SIZE; - const gen6_pte_t scratch_pte = ppgtt->scratch_pte; + const gen6_pte_t scratch_pte = vm->scratch_pte; while (num_entries) { struct i915_page_table *pt = ppgtt->base.pd.page_table[pde++]; @@ -1937,7 +1759,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, if (IS_ERR(pt)) goto unwind_out; - gen6_initialize_pt(ppgtt, pt); + gen6_initialize_pt(vm, pt); ppgtt->base.pd.page_table[pde] = pt; if (i915_vma_is_bound(ppgtt->vma, @@ -1975,9 +1797,9 @@ static int gen6_ppgtt_init_scratch(struct gen6_hw_ppgtt *ppgtt) if (ret) return ret; - ppgtt->scratch_pte = - vm->pte_encode(vm->scratch_page.daddr, - I915_CACHE_NONE, PTE_READ_ONLY); + vm->scratch_pte = vm->pte_encode(vm->scratch_page.daddr, + I915_CACHE_NONE, + PTE_READ_ONLY); vm->scratch_pt = alloc_pt(vm); if (IS_ERR(vm->scratch_pt)) { @@ -1985,7 +1807,7 @@ static int gen6_ppgtt_init_scratch(struct gen6_hw_ppgtt *ppgtt) return PTR_ERR(vm->scratch_pt); } - gen6_initialize_pt(ppgtt, vm->scratch_pt); + gen6_initialize_pt(vm, vm->scratch_pt); gen6_for_all_pdes(unused, &ppgtt->base.pd, pde) ppgtt->base.pd.page_table[pde] = vm->scratch_pt; @@ -2095,21 +1917,23 @@ static struct i915_vma *pd_vma_create(struct gen6_hw_ppgtt *ppgtt, int size) if (!vma) return ERR_PTR(-ENOMEM); - init_request_active(&vma->last_fence, NULL); + i915_active_init(i915, &vma->active, NULL); + INIT_ACTIVE_REQUEST(&vma->last_fence); vma->vm = &ggtt->vm; vma->ops = &pd_vma_ops; vma->private = ppgtt; - vma->active = RB_ROOT; - vma->size = size; vma->fence_size = size; vma->flags = I915_VMA_GGTT; vma->ggtt_view.type = I915_GGTT_VIEW_ROTATED; /* prevent fencing */ INIT_LIST_HEAD(&vma->obj_link); + + mutex_lock(&vma->vm->mutex); list_add(&vma->vm_link, &vma->vm->unbound_list); + mutex_unlock(&vma->vm->mutex); return vma; } @@ -2117,6 +1941,7 @@ static struct i915_vma *pd_vma_create(struct gen6_hw_ppgtt *ppgtt, int size) int gen6_ppgtt_pin(struct i915_hw_ppgtt *base) { struct gen6_hw_ppgtt *ppgtt = to_gen6_ppgtt(base); + int err; /* * Workaround the limited maximum vma->pin_count and the aliasing_ppgtt @@ -2132,9 +1957,17 @@ int gen6_ppgtt_pin(struct i915_hw_ppgtt *base) * allocator works in address space sizes, so it's multiplied by page * size. We allocate at the top of the GTT to avoid fragmentation. */ - return i915_vma_pin(ppgtt->vma, - 0, GEN6_PD_ALIGN, - PIN_GLOBAL | PIN_HIGH); + err = i915_vma_pin(ppgtt->vma, + 0, GEN6_PD_ALIGN, + PIN_GLOBAL | PIN_HIGH); + if (err) + goto unpin; + + return 0; + +unpin: + ppgtt->pin_count = 0; + return err; } void gen6_ppgtt_unpin(struct i915_hw_ppgtt *base) @@ -2165,13 +1998,12 @@ static struct i915_hw_ppgtt *gen6_ppgtt_create(struct drm_i915_private *i915) ppgtt->base.vm.total = I915_PDES * GEN6_PTES * I915_GTT_PAGE_SIZE; - i915_address_space_init(&ppgtt->base.vm, i915); + i915_address_space_init(&ppgtt->base.vm, VM_CLASS_PPGTT); ppgtt->base.vm.allocate_va_range = gen6_alloc_va_range; ppgtt->base.vm.clear_range = gen6_ppgtt_clear_range; ppgtt->base.vm.insert_entries = gen6_ppgtt_insert_entries; ppgtt->base.vm.cleanup = gen6_ppgtt_cleanup; - ppgtt->base.debug_dump = gen6_dump_ppgtt; ppgtt->base.vm.vma_ops.bind_vma = ppgtt_bind_vma; ppgtt->base.vm.vma_ops.unbind_vma = ppgtt_unbind_vma; @@ -2237,23 +2069,10 @@ int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv) { gtt_write_workarounds(dev_priv); - /* In the case of execlists, PPGTT is enabled by the context descriptor - * and the PDPs are contained within the context itself. We don't - * need to do anything here. */ - if (HAS_LOGICAL_RING_CONTEXTS(dev_priv)) - return 0; - - if (!USES_PPGTT(dev_priv)) - return 0; - - if (IS_GEN6(dev_priv)) + if (IS_GEN(dev_priv, 6)) gen6_ppgtt_enable(dev_priv); - else if (IS_GEN7(dev_priv)) + else if (IS_GEN(dev_priv, 7)) gen7_ppgtt_enable(dev_priv); - else if (INTEL_GEN(dev_priv) >= 8) - gen8_ppgtt_enable(dev_priv); - else - MISSING_CASE(INTEL_GEN(dev_priv)); return 0; } @@ -2293,8 +2112,7 @@ void i915_ppgtt_close(struct i915_address_space *vm) static void ppgtt_destroy_vma(struct i915_address_space *vm) { struct list_head *phases[] = { - &vm->active_list, - &vm->inactive_list, + &vm->bound_list, &vm->unbound_list, NULL, }, **phase; @@ -2317,8 +2135,7 @@ void i915_ppgtt_release(struct kref *kref) ppgtt_destroy_vma(&ppgtt->vm); - GEM_BUG_ON(!list_empty(&ppgtt->vm.active_list)); - GEM_BUG_ON(!list_empty(&ppgtt->vm.inactive_list)); + GEM_BUG_ON(!list_empty(&ppgtt->vm.bound_list)); GEM_BUG_ON(!list_empty(&ppgtt->vm.unbound_list)); ppgtt->vm.cleanup(&ppgtt->vm); @@ -2334,7 +2151,7 @@ static bool needs_idle_maps(struct drm_i915_private *dev_priv) /* Query intel_iommu to see if we need the workaround. Presumably that * was loaded first. */ - return IS_GEN5(dev_priv) && IS_MOBILE(dev_priv) && intel_vtd_active(); + return IS_GEN(dev_priv, 5) && IS_MOBILE(dev_priv) && intel_vtd_active(); } static void gen6_check_faults(struct drm_i915_private *dev_priv) @@ -2427,7 +2244,8 @@ int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj, DMA_ATTR_NO_WARN)) return 0; - /* If the DMA remap fails, one cause can be that we have + /* + * If the DMA remap fails, one cause can be that we have * too many objects pinned in a small remapping table, * such as swiotlb. Incrementally purge all other objects and * try again - if there are no more pages to remove from @@ -2437,8 +2255,7 @@ int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj, } while (i915_gem_shrink(to_i915(obj->base.dev), obj->base.size >> PAGE_SHIFT, NULL, I915_SHRINK_BOUND | - I915_SHRINK_UNBOUND | - I915_SHRINK_ACTIVE)); + I915_SHRINK_UNBOUND)); return -ENOSPC; } @@ -2543,8 +2360,7 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm, struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); unsigned first_entry = start / I915_GTT_PAGE_SIZE; unsigned num_entries = length / I915_GTT_PAGE_SIZE; - const gen8_pte_t scratch_pte = - gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC, 0); + const gen8_pte_t scratch_pte = vm->scratch_pte; gen8_pte_t __iomem *gtt_base = (gen8_pte_t __iomem *)ggtt->gsm + first_entry; const int max_entries = ggtt_total_entries(ggtt) - first_entry; @@ -2669,8 +2485,7 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm, first_entry, num_entries, max_entries)) num_entries = max_entries; - scratch_pte = vm->pte_encode(vm->scratch_page.daddr, - I915_CACHE_LLC, 0); + scratch_pte = vm->scratch_pte; for (i = 0; i < num_entries; i++) iowrite32(scratch_pte, >t_base[i]); @@ -2712,6 +2527,7 @@ static int ggtt_bind_vma(struct i915_vma *vma, { struct drm_i915_private *i915 = vma->vm->i915; struct drm_i915_gem_object *obj = vma->obj; + intel_wakeref_t wakeref; u32 pte_flags; /* Applicable to VLV (gen8+ do not support RO in the GGTT) */ @@ -2719,9 +2535,8 @@ static int ggtt_bind_vma(struct i915_vma *vma, if (i915_gem_object_is_readonly(obj)) pte_flags |= PTE_READ_ONLY; - intel_runtime_pm_get(i915); - vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags); - intel_runtime_pm_put(i915); + with_intel_runtime_pm(i915, wakeref) + vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags); vma->page_sizes.gtt = I915_GTT_PAGE_SIZE; @@ -2738,10 +2553,10 @@ static int ggtt_bind_vma(struct i915_vma *vma, static void ggtt_unbind_vma(struct i915_vma *vma) { struct drm_i915_private *i915 = vma->vm->i915; + intel_wakeref_t wakeref; - intel_runtime_pm_get(i915); - vma->vm->clear_range(vma->vm, vma->node.start, vma->size); - intel_runtime_pm_put(i915); + with_intel_runtime_pm(i915, wakeref) + vma->vm->clear_range(vma->vm, vma->node.start, vma->size); } static int aliasing_gtt_bind_vma(struct i915_vma *vma, @@ -2773,9 +2588,12 @@ static int aliasing_gtt_bind_vma(struct i915_vma *vma, } if (flags & I915_VMA_GLOBAL_BIND) { - intel_runtime_pm_get(i915); - vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags); - intel_runtime_pm_put(i915); + intel_wakeref_t wakeref; + + with_intel_runtime_pm(i915, wakeref) { + vma->vm->insert_entries(vma->vm, vma, + cache_level, pte_flags); + } } return 0; @@ -2786,9 +2604,11 @@ static void aliasing_gtt_unbind_vma(struct i915_vma *vma) struct drm_i915_private *i915 = vma->vm->i915; if (vma->flags & I915_VMA_GLOBAL_BIND) { - intel_runtime_pm_get(i915); - vma->vm->clear_range(vma->vm, vma->node.start, vma->size); - intel_runtime_pm_put(i915); + struct i915_address_space *vm = vma->vm; + intel_wakeref_t wakeref; + + with_intel_runtime_pm(i915, wakeref) + vm->clear_range(vm, vma->node.start, vma->size); } if (vma->flags & I915_VMA_LOCAL_BIND) { @@ -2952,7 +2772,7 @@ int i915_gem_init_ggtt(struct drm_i915_private *dev_priv) /* And finally clear the reserved guard page */ ggtt->vm.clear_range(&ggtt->vm, ggtt->vm.total - PAGE_SIZE, PAGE_SIZE); - if (USES_PPGTT(dev_priv) && !USES_FULL_PPGTT(dev_priv)) { + if (INTEL_PPGTT(dev_priv) == INTEL_PPGTT_ALIASING) { ret = i915_gem_init_aliasing_ppgtt(dev_priv); if (ret) goto err; @@ -2980,8 +2800,7 @@ void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv) mutex_lock(&dev_priv->drm.struct_mutex); i915_gem_fini_aliasing_ppgtt(dev_priv); - GEM_BUG_ON(!list_empty(&ggtt->vm.active_list)); - list_for_each_entry_safe(vma, vn, &ggtt->vm.inactive_list, vm_link) + list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) WARN_ON(i915_vma_unbind(vma)); if (drm_mm_node_allocated(&ggtt->error_capture)) @@ -3076,6 +2895,10 @@ static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) return ret; } + ggtt->vm.scratch_pte = + ggtt->vm.pte_encode(ggtt->vm.scratch_page.daddr, + I915_CACHE_NONE, 0); + return 0; } @@ -3275,7 +3098,7 @@ static void bdw_setup_private_ppat(struct intel_ppat *ppat) ppat->match = bdw_private_pat_match; ppat->clear_value = GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3); - if (!USES_PPGTT(ppat->i915)) { + if (!HAS_PPGTT(ppat->i915)) { /* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry, * so RTL will always use the value corresponding to * pat_sel = 000". @@ -3402,13 +3225,14 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt) ggtt->vm.cleanup = gen6_gmch_remove; ggtt->vm.insert_page = gen8_ggtt_insert_page; ggtt->vm.clear_range = nop_clear_range; - if (!USES_FULL_PPGTT(dev_priv) || intel_scanout_needs_vtd_wa(dev_priv)) + if (intel_scanout_needs_vtd_wa(dev_priv)) ggtt->vm.clear_range = gen8_ggtt_clear_range; ggtt->vm.insert_entries = gen8_ggtt_insert_entries; /* Serialize GTT updates with aperture access on BXT if VT-d is on. */ - if (intel_ggtt_update_needs_vtd_wa(dev_priv)) { + if (intel_ggtt_update_needs_vtd_wa(dev_priv) || + IS_CHERRYVIEW(dev_priv) /* fails with concurrent use/update */) { ggtt->vm.insert_entries = bxt_vtd_ggtt_insert_entries__BKL; ggtt->vm.insert_page = bxt_vtd_ggtt_insert_page__BKL; if (ggtt->vm.clear_range != nop_clear_range) @@ -3427,6 +3251,8 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt) ggtt->vm.vma_ops.set_pages = ggtt_set_pages; ggtt->vm.vma_ops.clear_pages = clear_pages; + ggtt->vm.pte_encode = gen8_pte_encode; + setup_private_pat(dev_priv); return ggtt_probe_common(ggtt, size); @@ -3607,14 +3433,14 @@ int i915_ggtt_init_hw(struct drm_i915_private *dev_priv) * and beyond the end of the GTT if we do not provide a guard. */ mutex_lock(&dev_priv->drm.struct_mutex); - i915_address_space_init(&ggtt->vm, dev_priv); + i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT); ggtt->vm.is_ggtt = true; /* Only VLV supports read-only GGTT mappings */ ggtt->vm.has_read_only = IS_VALLEYVIEW(dev_priv); - if (!HAS_LLC(dev_priv) && !USES_PPGTT(dev_priv)) + if (!HAS_LLC(dev_priv) && !HAS_PPGTT(dev_priv)) ggtt->vm.mm.color_adjust = i915_gtt_color_adjust; mutex_unlock(&dev_priv->drm.struct_mutex); @@ -3680,32 +3506,39 @@ void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv) i915_check_and_clear_faults(dev_priv); + mutex_lock(&ggtt->vm.mutex); + /* First fill our portion of the GTT with scratch pages */ ggtt->vm.clear_range(&ggtt->vm, 0, ggtt->vm.total); - ggtt->vm.closed = true; /* skip rewriting PTE on VMA unbind */ /* clflush objects bound into the GGTT and rebind them. */ - GEM_BUG_ON(!list_empty(&ggtt->vm.active_list)); - list_for_each_entry_safe(vma, vn, &ggtt->vm.inactive_list, vm_link) { + list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) { struct drm_i915_gem_object *obj = vma->obj; if (!(vma->flags & I915_VMA_GLOBAL_BIND)) continue; + mutex_unlock(&ggtt->vm.mutex); + if (!i915_vma_unbind(vma)) - continue; + goto lock; WARN_ON(i915_vma_bind(vma, obj ? obj->cache_level : 0, PIN_UPDATE)); if (obj) WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false)); + +lock: + mutex_lock(&ggtt->vm.mutex); } ggtt->vm.closed = false; i915_ggtt_invalidate(dev_priv); + mutex_unlock(&ggtt->vm.mutex); + if (INTEL_GEN(dev_priv) >= 8) { struct intel_ppat *ppat = &dev_priv->ppat; @@ -3716,7 +3549,7 @@ void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv) } static struct scatterlist * -rotate_pages(const dma_addr_t *in, unsigned int offset, +rotate_pages(struct drm_i915_gem_object *obj, unsigned int offset, unsigned int width, unsigned int height, unsigned int stride, struct sg_table *st, struct scatterlist *sg) @@ -3725,7 +3558,7 @@ rotate_pages(const dma_addr_t *in, unsigned int offset, unsigned int src_idx; for (column = 0; column < width; column++) { - src_idx = stride * (height - 1) + column; + src_idx = stride * (height - 1) + column + offset; for (row = 0; row < height; row++) { st->nents++; /* We don't need the pages, but need to initialize @@ -3733,7 +3566,8 @@ rotate_pages(const dma_addr_t *in, unsigned int offset, * The only thing we need are DMA addresses. */ sg_set_page(sg, NULL, I915_GTT_PAGE_SIZE, 0); - sg_dma_address(sg) = in[offset + src_idx]; + sg_dma_address(sg) = + i915_gem_object_get_dma_address(obj, src_idx); sg_dma_len(sg) = I915_GTT_PAGE_SIZE; sg = sg_next(sg); src_idx -= stride; @@ -3747,22 +3581,11 @@ static noinline struct sg_table * intel_rotate_pages(struct intel_rotation_info *rot_info, struct drm_i915_gem_object *obj) { - const unsigned long n_pages = obj->base.size / I915_GTT_PAGE_SIZE; unsigned int size = intel_rotation_info_size(rot_info); - struct sgt_iter sgt_iter; - dma_addr_t dma_addr; - unsigned long i; - dma_addr_t *page_addr_list; struct sg_table *st; struct scatterlist *sg; int ret = -ENOMEM; - - /* Allocate a temporary list of source pages for random access. */ - page_addr_list = kvmalloc_array(n_pages, - sizeof(dma_addr_t), - GFP_KERNEL); - if (!page_addr_list) - return ERR_PTR(ret); + int i; /* Allocate target SG list. */ st = kmalloc(sizeof(*st), GFP_KERNEL); @@ -3773,29 +3596,20 @@ intel_rotate_pages(struct intel_rotation_info *rot_info, if (ret) goto err_sg_alloc; - /* Populate source page list from the object. */ - i = 0; - for_each_sgt_dma(dma_addr, sgt_iter, obj->mm.pages) - page_addr_list[i++] = dma_addr; - - GEM_BUG_ON(i != n_pages); st->nents = 0; sg = st->sgl; for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++) { - sg = rotate_pages(page_addr_list, rot_info->plane[i].offset, + sg = rotate_pages(obj, rot_info->plane[i].offset, rot_info->plane[i].width, rot_info->plane[i].height, rot_info->plane[i].stride, st, sg); } - kvfree(page_addr_list); - return st; err_sg_alloc: kfree(st); err_st_alloc: - kvfree(page_addr_list); DRM_DEBUG_DRIVER("Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n", obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size); @@ -3840,6 +3654,8 @@ intel_partial_pages(const struct i915_ggtt_view *view, count -= len >> PAGE_SHIFT; if (count == 0) { sg_mark_end(sg); + i915_sg_trim(st); /* Drop any unused tail entries. */ + return st; } diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h index 28039290655c..03ade71b8d9a 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.h +++ b/drivers/gpu/drm/i915/i915_gem_gtt.h @@ -39,6 +39,7 @@ #include <linux/pagevec.h> #include "i915_request.h" +#include "i915_reset.h" #include "i915_selftest.h" #include "i915_timeline.h" @@ -288,39 +289,22 @@ struct i915_address_space { bool closed; struct mutex mutex; /* protects vma and our lists */ +#define VM_CLASS_GGTT 0 +#define VM_CLASS_PPGTT 1 + u64 scratch_pte; struct i915_page_dma scratch_page; struct i915_page_table *scratch_pt; struct i915_page_directory *scratch_pd; struct i915_page_directory_pointer *scratch_pdp; /* GEN8+ & 48b PPGTT */ /** - * List of objects currently involved in rendering. - * - * Includes buffers having the contents of their GPU caches - * flushed, not necessarily primitives. last_read_req - * represents when the rendering involved will be completed. - * - * A reference is held on the buffer while on this list. + * List of vma currently bound. */ - struct list_head active_list; + struct list_head bound_list; /** - * LRU list of objects which are not in the ringbuffer and - * are ready to unbind, but are still in the GTT. - * - * last_read_req is NULL while an object is in this list. - * - * A reference is not held on the buffer while on this list, - * as merely being GTT-bound shouldn't prevent its being - * freed, and we'll pull it off the list in the free path. - */ - struct list_head inactive_list; - - /** - * List of vma that have been unbound. - * - * A reference is not held on the buffer while on this list. + * List of vma that are not unbound. */ struct list_head unbound_list; @@ -335,12 +319,11 @@ struct i915_address_space { /* Some systems support read-only mappings for GGTT and/or PPGTT */ bool has_read_only:1; - /* FIXME: Need a more generic return type */ - gen6_pte_t (*pte_encode)(dma_addr_t addr, - enum i915_cache_level level, - u32 flags); /* Create a valid PTE */ - /* flags for pte_encode */ + u64 (*pte_encode)(dma_addr_t addr, + enum i915_cache_level level, + u32 flags); /* Create a valid PTE */ #define PTE_READ_ONLY (1<<0) + int (*allocate_va_range)(struct i915_address_space *vm, u64 start, u64 length); void (*clear_range)(struct i915_address_space *vm, @@ -413,8 +396,6 @@ struct i915_hw_ppgtt { struct i915_page_directory_pointer pdp; /* GEN8+ */ struct i915_page_directory pd; /* GEN6-7 */ }; - - void (*debug_dump)(struct i915_hw_ppgtt *ppgtt, struct seq_file *m); }; struct gen6_hw_ppgtt { @@ -422,7 +403,6 @@ struct gen6_hw_ppgtt { struct i915_vma *vma; gen6_pte_t __iomem *pd_addr; - gen6_pte_t scratch_pte; unsigned int pin_count; bool scan_for_unused_pt; @@ -662,19 +642,19 @@ int i915_gem_gtt_insert(struct i915_address_space *vm, /* Flags used by pin/bind&friends. */ #define PIN_NONBLOCK BIT_ULL(0) -#define PIN_MAPPABLE BIT_ULL(1) -#define PIN_ZONE_4G BIT_ULL(2) -#define PIN_NONFAULT BIT_ULL(3) -#define PIN_NOEVICT BIT_ULL(4) - -#define PIN_MBZ BIT_ULL(5) /* I915_VMA_PIN_OVERFLOW */ -#define PIN_GLOBAL BIT_ULL(6) /* I915_VMA_GLOBAL_BIND */ -#define PIN_USER BIT_ULL(7) /* I915_VMA_LOCAL_BIND */ -#define PIN_UPDATE BIT_ULL(8) - -#define PIN_HIGH BIT_ULL(9) -#define PIN_OFFSET_BIAS BIT_ULL(10) -#define PIN_OFFSET_FIXED BIT_ULL(11) +#define PIN_NONFAULT BIT_ULL(1) +#define PIN_NOEVICT BIT_ULL(2) +#define PIN_MAPPABLE BIT_ULL(3) +#define PIN_ZONE_4G BIT_ULL(4) +#define PIN_HIGH BIT_ULL(5) +#define PIN_OFFSET_BIAS BIT_ULL(6) +#define PIN_OFFSET_FIXED BIT_ULL(7) + +#define PIN_MBZ BIT_ULL(8) /* I915_VMA_PIN_OVERFLOW */ +#define PIN_GLOBAL BIT_ULL(9) /* I915_VMA_GLOBAL_BIND */ +#define PIN_USER BIT_ULL(10) /* I915_VMA_LOCAL_BIND */ +#define PIN_UPDATE BIT_ULL(11) + #define PIN_OFFSET_MASK (-I915_GTT_PAGE_SIZE) #endif diff --git a/drivers/gpu/drm/i915/i915_gem_internal.c b/drivers/gpu/drm/i915/i915_gem_internal.c index 0d0144b2104c..fddde1033e74 100644 --- a/drivers/gpu/drm/i915/i915_gem_internal.c +++ b/drivers/gpu/drm/i915/i915_gem_internal.c @@ -22,7 +22,6 @@ * */ -#include <drm/drmP.h> #include <drm/i915_drm.h> #include "i915_drv.h" diff --git a/drivers/gpu/drm/i915/i915_gem_object.h b/drivers/gpu/drm/i915/i915_gem_object.h index a6dd7c46de0d..fab040331cdb 100644 --- a/drivers/gpu/drm/i915/i915_gem_object.h +++ b/drivers/gpu/drm/i915/i915_gem_object.h @@ -29,7 +29,8 @@ #include <drm/drm_vma_manager.h> #include <drm/drm_gem.h> -#include <drm/drmP.h> +#include <drm/drm_file.h> +#include <drm/drm_device.h> #include <drm/i915_drm.h> @@ -56,6 +57,7 @@ struct drm_i915_gem_object_ops { #define I915_GEM_OBJECT_HAS_STRUCT_PAGE BIT(0) #define I915_GEM_OBJECT_IS_SHRINKABLE BIT(1) #define I915_GEM_OBJECT_IS_PROXY BIT(2) +#define I915_GEM_OBJECT_ASYNC_CANCEL BIT(3) /* Interface between the GEM object and its backing storage. * get_pages() is called once prior to the use of the associated set @@ -85,24 +87,33 @@ struct drm_i915_gem_object { const struct drm_i915_gem_object_ops *ops; - /** - * @vma_list: List of VMAs backed by this object - * - * The VMA on this list are ordered by type, all GGTT vma are placed - * at the head and all ppGTT vma are placed at the tail. The different - * types of GGTT vma are unordered between themselves, use the - * @vma_tree (which has a defined order between all VMA) to find an - * exact match. - */ - struct list_head vma_list; - /** - * @vma_tree: Ordered tree of VMAs backed by this object - * - * All VMA created for this object are placed in the @vma_tree for - * fast retrieval via a binary search in i915_vma_instance(). - * They are also added to @vma_list for easy iteration. - */ - struct rb_root vma_tree; + struct { + /** + * @vma.lock: protect the list/tree of vmas + */ + spinlock_t lock; + + /** + * @vma.list: List of VMAs backed by this object + * + * The VMA on this list are ordered by type, all GGTT vma are + * placed at the head and all ppGTT vma are placed at the tail. + * The different types of GGTT vma are unordered between + * themselves, use the @vma.tree (which has a defined order + * between all VMA) to quickly find an exact match. + */ + struct list_head list; + + /** + * @vma.tree: Ordered tree of VMAs backed by this object + * + * All VMA created for this object are placed in the @vma.tree + * for fast retrieval via a binary search in + * i915_vma_instance(). They are also added to @vma.list for + * easy iteration. + */ + struct rb_root tree; + } vma; /** * @lut_list: List of vma lookup entries in use for this object. @@ -164,7 +175,7 @@ struct drm_i915_gem_object { atomic_t frontbuffer_bits; unsigned int frontbuffer_ggtt_origin; /* write once */ - struct i915_gem_active frontbuffer_write; + struct i915_active_request frontbuffer_write; /** Current tiling stride for the object, if it's tiled. */ unsigned int tiling_and_stride; @@ -387,6 +398,12 @@ i915_gem_object_is_proxy(const struct drm_i915_gem_object *obj) } static inline bool +i915_gem_object_needs_async_cancel(const struct drm_i915_gem_object *obj) +{ + return obj->ops->flags & I915_GEM_OBJECT_ASYNC_CANCEL; +} + +static inline bool i915_gem_object_is_active(const struct drm_i915_gem_object *obj) { return obj->active_count; diff --git a/drivers/gpu/drm/i915/i915_gem_shrinker.c b/drivers/gpu/drm/i915/i915_gem_shrinker.c index ea90d3a0d511..6da795c7e62e 100644 --- a/drivers/gpu/drm/i915/i915_gem_shrinker.c +++ b/drivers/gpu/drm/i915/i915_gem_shrinker.c @@ -30,30 +30,27 @@ #include <linux/pci.h> #include <linux/dma-buf.h> #include <linux/vmalloc.h> -#include <drm/drmP.h> #include <drm/i915_drm.h> #include "i915_drv.h" #include "i915_trace.h" -static bool shrinker_lock(struct drm_i915_private *i915, bool *unlock) +static bool shrinker_lock(struct drm_i915_private *i915, + unsigned int flags, + bool *unlock) { - switch (mutex_trylock_recursive(&i915->drm.struct_mutex)) { + struct mutex *m = &i915->drm.struct_mutex; + + switch (mutex_trylock_recursive(m)) { case MUTEX_TRYLOCK_RECURSIVE: *unlock = false; return true; case MUTEX_TRYLOCK_FAILED: *unlock = false; - preempt_disable(); - do { - cpu_relax(); - if (mutex_trylock(&i915->drm.struct_mutex)) { - *unlock = true; - break; - } - } while (!need_resched()); - preempt_enable(); + if (flags & I915_SHRINK_ACTIVE && + mutex_lock_killable_nested(m, I915_MM_SHRINKER) == 0) + *unlock = true; return *unlock; case MUTEX_TRYLOCK_SUCCESS: @@ -156,11 +153,12 @@ i915_gem_shrink(struct drm_i915_private *i915, { &i915->mm.bound_list, I915_SHRINK_BOUND }, { NULL, 0 }, }, *phase; + intel_wakeref_t wakeref = 0; unsigned long count = 0; unsigned long scanned = 0; bool unlock; - if (!shrinker_lock(i915, &unlock)) + if (!shrinker_lock(i915, flags, &unlock)) return 0; /* @@ -185,9 +183,11 @@ i915_gem_shrink(struct drm_i915_private *i915, * device just to recover a little memory. If absolutely necessary, * we will force the wake during oom-notifier. */ - if ((flags & I915_SHRINK_BOUND) && - !intel_runtime_pm_get_if_in_use(i915)) - flags &= ~I915_SHRINK_BOUND; + if (flags & I915_SHRINK_BOUND) { + wakeref = intel_runtime_pm_get_if_in_use(i915); + if (!wakeref) + flags &= ~I915_SHRINK_BOUND; + } /* * As we may completely rewrite the (un)bound list whilst unbinding @@ -268,7 +268,7 @@ i915_gem_shrink(struct drm_i915_private *i915, } if (flags & I915_SHRINK_BOUND) - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); i915_retire_requests(i915); @@ -295,14 +295,15 @@ i915_gem_shrink(struct drm_i915_private *i915, */ unsigned long i915_gem_shrink_all(struct drm_i915_private *i915) { - unsigned long freed; - - intel_runtime_pm_get(i915); - freed = i915_gem_shrink(i915, -1UL, NULL, - I915_SHRINK_BOUND | - I915_SHRINK_UNBOUND | - I915_SHRINK_ACTIVE); - intel_runtime_pm_put(i915); + intel_wakeref_t wakeref; + unsigned long freed = 0; + + with_intel_runtime_pm(i915, wakeref) { + freed = i915_gem_shrink(i915, -1UL, NULL, + I915_SHRINK_BOUND | + I915_SHRINK_UNBOUND | + I915_SHRINK_ACTIVE); + } return freed; } @@ -357,7 +358,7 @@ i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) sc->nr_scanned = 0; - if (!shrinker_lock(i915, &unlock)) + if (!shrinker_lock(i915, 0, &unlock)) return SHRINK_STOP; freed = i915_gem_shrink(i915, @@ -373,14 +374,16 @@ i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) I915_SHRINK_BOUND | I915_SHRINK_UNBOUND); if (sc->nr_scanned < sc->nr_to_scan && current_is_kswapd()) { - intel_runtime_pm_get(i915); - freed += i915_gem_shrink(i915, - sc->nr_to_scan - sc->nr_scanned, - &sc->nr_scanned, - I915_SHRINK_ACTIVE | - I915_SHRINK_BOUND | - I915_SHRINK_UNBOUND); - intel_runtime_pm_put(i915); + intel_wakeref_t wakeref; + + with_intel_runtime_pm(i915, wakeref) { + freed += i915_gem_shrink(i915, + sc->nr_to_scan - sc->nr_scanned, + &sc->nr_scanned, + I915_SHRINK_ACTIVE | + I915_SHRINK_BOUND | + I915_SHRINK_UNBOUND); + } } shrinker_unlock(i915, unlock); @@ -388,31 +391,6 @@ i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) return sc->nr_scanned ? freed : SHRINK_STOP; } -static bool -shrinker_lock_uninterruptible(struct drm_i915_private *i915, bool *unlock, - int timeout_ms) -{ - unsigned long timeout = jiffies + msecs_to_jiffies_timeout(timeout_ms); - - do { - if (i915_gem_wait_for_idle(i915, - 0, MAX_SCHEDULE_TIMEOUT) == 0 && - shrinker_lock(i915, unlock)) - break; - - schedule_timeout_killable(1); - if (fatal_signal_pending(current)) - return false; - - if (time_after(jiffies, timeout)) { - pr_err("Unable to lock GPU to purge memory.\n"); - return false; - } - } while (1); - - return true; -} - static int i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr) { @@ -420,8 +398,13 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr) container_of(nb, struct drm_i915_private, mm.oom_notifier); struct drm_i915_gem_object *obj; unsigned long unevictable, bound, unbound, freed_pages; + intel_wakeref_t wakeref; - freed_pages = i915_gem_shrink_all(i915); + freed_pages = 0; + with_intel_runtime_pm(i915, wakeref) + freed_pages += i915_gem_shrink(i915, -1UL, NULL, + I915_SHRINK_BOUND | + I915_SHRINK_UNBOUND); /* Because we may be allocating inside our own driver, we cannot * assert that there are no objects with pinned pages that are not @@ -447,10 +430,6 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr) pr_info("Purging GPU memory, %lu pages freed, " "%lu pages still pinned.\n", freed_pages, unevictable); - if (unbound || bound) - pr_err("%lu and %lu pages still available in the " - "bound and unbound GPU page lists.\n", - bound, unbound); *(unsigned long *)ptr += freed_pages; return NOTIFY_DONE; @@ -463,34 +442,39 @@ i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr container_of(nb, struct drm_i915_private, mm.vmap_notifier); struct i915_vma *vma, *next; unsigned long freed_pages = 0; + intel_wakeref_t wakeref; bool unlock; - int ret; - if (!shrinker_lock_uninterruptible(i915, &unlock, 5000)) + if (!shrinker_lock(i915, 0, &unlock)) return NOTIFY_DONE; /* Force everything onto the inactive lists */ - ret = i915_gem_wait_for_idle(i915, - I915_WAIT_LOCKED, - MAX_SCHEDULE_TIMEOUT); - if (ret) + if (i915_gem_wait_for_idle(i915, + I915_WAIT_LOCKED, + MAX_SCHEDULE_TIMEOUT)) goto out; - intel_runtime_pm_get(i915); - freed_pages += i915_gem_shrink(i915, -1UL, NULL, - I915_SHRINK_BOUND | - I915_SHRINK_UNBOUND | - I915_SHRINK_ACTIVE | - I915_SHRINK_VMAPS); - intel_runtime_pm_put(i915); + with_intel_runtime_pm(i915, wakeref) + freed_pages += i915_gem_shrink(i915, -1UL, NULL, + I915_SHRINK_BOUND | + I915_SHRINK_UNBOUND | + I915_SHRINK_VMAPS); /* We also want to clear any cached iomaps as they wrap vmap */ + mutex_lock(&i915->ggtt.vm.mutex); list_for_each_entry_safe(vma, next, - &i915->ggtt.vm.inactive_list, vm_link) { + &i915->ggtt.vm.bound_list, vm_link) { unsigned long count = vma->node.size >> PAGE_SHIFT; - if (vma->iomap && i915_vma_unbind(vma) == 0) + + if (!vma->iomap || i915_vma_is_active(vma)) + continue; + + mutex_unlock(&i915->ggtt.vm.mutex); + if (i915_vma_unbind(vma) == 0) freed_pages += count; + mutex_lock(&i915->ggtt.vm.mutex); } + mutex_unlock(&i915->ggtt.vm.mutex); out: shrinker_unlock(i915, unlock); @@ -533,13 +517,40 @@ void i915_gem_shrinker_unregister(struct drm_i915_private *i915) unregister_shrinker(&i915->mm.shrinker); } -void i915_gem_shrinker_taints_mutex(struct mutex *mutex) +void i915_gem_shrinker_taints_mutex(struct drm_i915_private *i915, + struct mutex *mutex) { + bool unlock = false; + if (!IS_ENABLED(CONFIG_LOCKDEP)) return; + if (!lockdep_is_held_type(&i915->drm.struct_mutex, -1)) { + mutex_acquire(&i915->drm.struct_mutex.dep_map, + I915_MM_NORMAL, 0, _RET_IP_); + unlock = true; + } + fs_reclaim_acquire(GFP_KERNEL); - mutex_lock(mutex); - mutex_unlock(mutex); + + /* + * As we invariably rely on the struct_mutex within the shrinker, + * but have a complicated recursion dance, taint all the mutexes used + * within the shrinker with the struct_mutex. For completeness, we + * taint with all subclass of struct_mutex, even though we should + * only need tainting by I915_MM_NORMAL to catch possible ABBA + * deadlocks from using struct_mutex inside @mutex. + */ + mutex_acquire(&i915->drm.struct_mutex.dep_map, + I915_MM_SHRINKER, 0, _RET_IP_); + + mutex_acquire(&mutex->dep_map, 0, 0, _RET_IP_); + mutex_release(&mutex->dep_map, 0, _RET_IP_); + + mutex_release(&i915->drm.struct_mutex.dep_map, 0, _RET_IP_); + fs_reclaim_release(GFP_KERNEL); + + if (unlock) + mutex_release(&i915->drm.struct_mutex.dep_map, 0, _RET_IP_); } diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c index f29a7ff7c362..74a9661479ca 100644 --- a/drivers/gpu/drm/i915/i915_gem_stolen.c +++ b/drivers/gpu/drm/i915/i915_gem_stolen.c @@ -26,7 +26,6 @@ * */ -#include <drm/drmP.h> #include <drm/i915_drm.h> #include "i915_drv.h" @@ -102,7 +101,7 @@ static int i915_adjust_stolen(struct drm_i915_private *dev_priv, resource_size_t ggtt_start; ggtt_start = I915_READ(PGTBL_CTL); - if (IS_GEN4(dev_priv)) + if (IS_GEN(dev_priv, 4)) ggtt_start = (ggtt_start & PGTBL_ADDRESS_LO_MASK) | (ggtt_start & PGTBL_ADDRESS_HI_MASK) << 28; else @@ -156,7 +155,7 @@ static int i915_adjust_stolen(struct drm_i915_private *dev_priv, * GEN3 firmware likes to smash pci bridges into the stolen * range. Apparently this works. */ - if (r == NULL && !IS_GEN3(dev_priv)) { + if (r == NULL && !IS_GEN(dev_priv, 3)) { DRM_ERROR("conflict detected with stolen region: %pR\n", dsm); @@ -194,7 +193,8 @@ static void g4x_get_stolen_reserved(struct drm_i915_private *dev_priv, * Whether ILK really reuses the ELK register for this is unclear. * Let's see if we catch anyone with this supposedly enabled on ILK. */ - WARN(IS_GEN5(dev_priv), "ILK stolen reserved found? 0x%08x\n", reg_val); + WARN(IS_GEN(dev_priv, 5), "ILK stolen reserved found? 0x%08x\n", + reg_val); if (!(reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK)) return; @@ -701,7 +701,10 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv vma->pages = obj->mm.pages; vma->flags |= I915_VMA_GLOBAL_BIND; __i915_vma_set_map_and_fenceable(vma); - list_move_tail(&vma->vm_link, &ggtt->vm.inactive_list); + + mutex_lock(&ggtt->vm.mutex); + list_move_tail(&vma->vm_link, &ggtt->vm.bound_list); + mutex_unlock(&ggtt->vm.mutex); spin_lock(&dev_priv->mm.obj_lock); list_move_tail(&obj->mm.link, &dev_priv->mm.bound_list); diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c index d9dc9df523b5..16cc9ddbce34 100644 --- a/drivers/gpu/drm/i915/i915_gem_tiling.c +++ b/drivers/gpu/drm/i915/i915_gem_tiling.c @@ -27,7 +27,6 @@ #include <linux/string.h> #include <linux/bitops.h> -#include <drm/drmP.h> #include <drm/i915_drm.h> #include "i915_drv.h" @@ -87,7 +86,7 @@ u32 i915_gem_fence_size(struct drm_i915_private *i915, } /* Previous chips need a power-of-two fence region when tiling */ - if (IS_GEN3(i915)) + if (IS_GEN(i915, 3)) ggtt_size = 1024*1024; else ggtt_size = 512*1024; @@ -162,7 +161,7 @@ i915_tiling_ok(struct drm_i915_gem_object *obj, return false; } - if (IS_GEN2(i915) || + if (IS_GEN(i915, 2) || (tiling == I915_TILING_Y && HAS_128_BYTE_Y_TILING(i915))) tile_width = 128; else diff --git a/drivers/gpu/drm/i915/i915_gem_userptr.c b/drivers/gpu/drm/i915/i915_gem_userptr.c index 2c9b284036d1..1d3f9a31ad61 100644 --- a/drivers/gpu/drm/i915/i915_gem_userptr.c +++ b/drivers/gpu/drm/i915/i915_gem_userptr.c @@ -22,7 +22,6 @@ * */ -#include <drm/drmP.h> #include <drm/i915_drm.h> #include "i915_drv.h" #include "i915_trace.h" @@ -50,94 +49,86 @@ struct i915_mmu_notifier { struct hlist_node node; struct mmu_notifier mn; struct rb_root_cached objects; - struct workqueue_struct *wq; + struct i915_mm_struct *mm; }; struct i915_mmu_object { struct i915_mmu_notifier *mn; struct drm_i915_gem_object *obj; struct interval_tree_node it; - struct list_head link; - struct work_struct work; - bool attached; }; -static void cancel_userptr(struct work_struct *work) +static void add_object(struct i915_mmu_object *mo) { - struct i915_mmu_object *mo = container_of(work, typeof(*mo), work); - struct drm_i915_gem_object *obj = mo->obj; - struct work_struct *active; - - /* Cancel any active worker and force us to re-evaluate gup */ - mutex_lock(&obj->mm.lock); - active = fetch_and_zero(&obj->userptr.work); - mutex_unlock(&obj->mm.lock); - if (active) - goto out; - - i915_gem_object_wait(obj, I915_WAIT_ALL, MAX_SCHEDULE_TIMEOUT, NULL); - - mutex_lock(&obj->base.dev->struct_mutex); - - /* We are inside a kthread context and can't be interrupted */ - if (i915_gem_object_unbind(obj) == 0) - __i915_gem_object_put_pages(obj, I915_MM_NORMAL); - WARN_ONCE(i915_gem_object_has_pages(obj), - "Failed to release pages: bind_count=%d, pages_pin_count=%d, pin_global=%d\n", - obj->bind_count, - atomic_read(&obj->mm.pages_pin_count), - obj->pin_global); - - mutex_unlock(&obj->base.dev->struct_mutex); - -out: - i915_gem_object_put(obj); + GEM_BUG_ON(!RB_EMPTY_NODE(&mo->it.rb)); + interval_tree_insert(&mo->it, &mo->mn->objects); } -static void add_object(struct i915_mmu_object *mo) +static void del_object(struct i915_mmu_object *mo) { - if (mo->attached) + if (RB_EMPTY_NODE(&mo->it.rb)) return; - interval_tree_insert(&mo->it, &mo->mn->objects); - mo->attached = true; + interval_tree_remove(&mo->it, &mo->mn->objects); + RB_CLEAR_NODE(&mo->it.rb); } -static void del_object(struct i915_mmu_object *mo) +static void +__i915_gem_userptr_set_active(struct drm_i915_gem_object *obj, bool value) { - if (!mo->attached) + struct i915_mmu_object *mo = obj->userptr.mmu_object; + + /* + * During mm_invalidate_range we need to cancel any userptr that + * overlaps the range being invalidated. Doing so requires the + * struct_mutex, and that risks recursion. In order to cause + * recursion, the user must alias the userptr address space with + * a GTT mmapping (possible with a MAP_FIXED) - then when we have + * to invalidate that mmaping, mm_invalidate_range is called with + * the userptr address *and* the struct_mutex held. To prevent that + * we set a flag under the i915_mmu_notifier spinlock to indicate + * whether this object is valid. + */ + if (!mo) return; - interval_tree_remove(&mo->it, &mo->mn->objects); - mo->attached = false; + spin_lock(&mo->mn->lock); + if (value) + add_object(mo); + else + del_object(mo); + spin_unlock(&mo->mn->lock); } -static int i915_gem_userptr_mn_invalidate_range_start(struct mmu_notifier *_mn, - struct mm_struct *mm, - unsigned long start, - unsigned long end, - bool blockable) +static int +userptr_mn_invalidate_range_start(struct mmu_notifier *_mn, + const struct mmu_notifier_range *range) { struct i915_mmu_notifier *mn = container_of(_mn, struct i915_mmu_notifier, mn); - struct i915_mmu_object *mo; struct interval_tree_node *it; - LIST_HEAD(cancelled); + struct mutex *unlock = NULL; + unsigned long end; + int ret = 0; if (RB_EMPTY_ROOT(&mn->objects.rb_root)) return 0; /* interval ranges are inclusive, but invalidate range is exclusive */ - end--; + end = range->end - 1; spin_lock(&mn->lock); - it = interval_tree_iter_first(&mn->objects, start, end); + it = interval_tree_iter_first(&mn->objects, range->start, end); while (it) { - if (!blockable) { - spin_unlock(&mn->lock); - return -EAGAIN; + struct drm_i915_gem_object *obj; + + if (!range->blockable) { + ret = -EAGAIN; + break; } - /* The mmu_object is released late when destroying the + + /* + * The mmu_object is released late when destroying the * GEM object so it is entirely possible to gain a * reference on an object in the process of being freed * since our serialisation is via the spinlock and not @@ -146,29 +137,65 @@ static int i915_gem_userptr_mn_invalidate_range_start(struct mmu_notifier *_mn, * use-after-free we only acquire a reference on the * object if it is not in the process of being destroyed. */ - mo = container_of(it, struct i915_mmu_object, it); - if (kref_get_unless_zero(&mo->obj->base.refcount)) - queue_work(mn->wq, &mo->work); + obj = container_of(it, struct i915_mmu_object, it)->obj; + if (!kref_get_unless_zero(&obj->base.refcount)) { + it = interval_tree_iter_next(it, range->start, end); + continue; + } + spin_unlock(&mn->lock); + + if (!unlock) { + unlock = &mn->mm->i915->drm.struct_mutex; + + switch (mutex_trylock_recursive(unlock)) { + default: + case MUTEX_TRYLOCK_FAILED: + if (mutex_lock_killable_nested(unlock, I915_MM_SHRINKER)) { + i915_gem_object_put(obj); + return -EINTR; + } + /* fall through */ + case MUTEX_TRYLOCK_SUCCESS: + break; + + case MUTEX_TRYLOCK_RECURSIVE: + unlock = ERR_PTR(-EEXIST); + break; + } + } + + ret = i915_gem_object_unbind(obj); + if (ret == 0) + ret = __i915_gem_object_put_pages(obj, I915_MM_SHRINKER); + i915_gem_object_put(obj); + if (ret) + goto unlock; - list_add(&mo->link, &cancelled); - it = interval_tree_iter_next(it, start, end); + spin_lock(&mn->lock); + + /* + * As we do not (yet) protect the mmu from concurrent insertion + * over this range, there is no guarantee that this search will + * terminate given a pathologic workload. + */ + it = interval_tree_iter_first(&mn->objects, range->start, end); } - list_for_each_entry(mo, &cancelled, link) - del_object(mo); spin_unlock(&mn->lock); - if (!list_empty(&cancelled)) - flush_workqueue(mn->wq); +unlock: + if (!IS_ERR_OR_NULL(unlock)) + mutex_unlock(unlock); + + return ret; - return 0; } static const struct mmu_notifier_ops i915_gem_userptr_notifier = { - .invalidate_range_start = i915_gem_userptr_mn_invalidate_range_start, + .invalidate_range_start = userptr_mn_invalidate_range_start, }; static struct i915_mmu_notifier * -i915_mmu_notifier_create(struct mm_struct *mm) +i915_mmu_notifier_create(struct i915_mm_struct *mm) { struct i915_mmu_notifier *mn; @@ -179,13 +206,7 @@ i915_mmu_notifier_create(struct mm_struct *mm) spin_lock_init(&mn->lock); mn->mn.ops = &i915_gem_userptr_notifier; mn->objects = RB_ROOT_CACHED; - mn->wq = alloc_workqueue("i915-userptr-release", - WQ_UNBOUND | WQ_MEM_RECLAIM, - 0); - if (mn->wq == NULL) { - kfree(mn); - return ERR_PTR(-ENOMEM); - } + mn->mm = mm; return mn; } @@ -195,16 +216,14 @@ i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj) { struct i915_mmu_object *mo; - mo = obj->userptr.mmu_object; - if (mo == NULL) + mo = fetch_and_zero(&obj->userptr.mmu_object); + if (!mo) return; spin_lock(&mo->mn->lock); del_object(mo); spin_unlock(&mo->mn->lock); kfree(mo); - - obj->userptr.mmu_object = NULL; } static struct i915_mmu_notifier * @@ -217,7 +236,7 @@ i915_mmu_notifier_find(struct i915_mm_struct *mm) if (mn) return mn; - mn = i915_mmu_notifier_create(mm->mm); + mn = i915_mmu_notifier_create(mm); if (IS_ERR(mn)) err = PTR_ERR(mn); @@ -240,10 +259,8 @@ i915_mmu_notifier_find(struct i915_mm_struct *mm) mutex_unlock(&mm->i915->mm_lock); up_write(&mm->mm->mmap_sem); - if (mn && !IS_ERR(mn)) { - destroy_workqueue(mn->wq); + if (mn && !IS_ERR(mn)) kfree(mn); - } return err ? ERR_PTR(err) : mm->mn; } @@ -266,14 +283,14 @@ i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj, return PTR_ERR(mn); mo = kzalloc(sizeof(*mo), GFP_KERNEL); - if (mo == NULL) + if (!mo) return -ENOMEM; mo->mn = mn; mo->obj = obj; mo->it.start = obj->userptr.ptr; mo->it.last = obj->userptr.ptr + obj->base.size - 1; - INIT_WORK(&mo->work, cancel_userptr); + RB_CLEAR_NODE(&mo->it.rb); obj->userptr.mmu_object = mo; return 0; @@ -287,13 +304,17 @@ i915_mmu_notifier_free(struct i915_mmu_notifier *mn, return; mmu_notifier_unregister(&mn->mn, mm); - destroy_workqueue(mn->wq); kfree(mn); } #else static void +__i915_gem_userptr_set_active(struct drm_i915_gem_object *obj, bool value) +{ +} + +static void i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj) { } @@ -461,42 +482,6 @@ alloc_table: return st; } -static int -__i915_gem_userptr_set_active(struct drm_i915_gem_object *obj, - bool value) -{ - int ret = 0; - - /* During mm_invalidate_range we need to cancel any userptr that - * overlaps the range being invalidated. Doing so requires the - * struct_mutex, and that risks recursion. In order to cause - * recursion, the user must alias the userptr address space with - * a GTT mmapping (possible with a MAP_FIXED) - then when we have - * to invalidate that mmaping, mm_invalidate_range is called with - * the userptr address *and* the struct_mutex held. To prevent that - * we set a flag under the i915_mmu_notifier spinlock to indicate - * whether this object is valid. - */ -#if defined(CONFIG_MMU_NOTIFIER) - if (obj->userptr.mmu_object == NULL) - return 0; - - spin_lock(&obj->userptr.mmu_object->mn->lock); - /* In order to serialise get_pages with an outstanding - * cancel_userptr, we must drop the struct_mutex and try again. - */ - if (!value) - del_object(obj->userptr.mmu_object); - else if (!work_pending(&obj->userptr.mmu_object->work)) - add_object(obj->userptr.mmu_object); - else - ret = -EAGAIN; - spin_unlock(&obj->userptr.mmu_object->mn->lock); -#endif - - return ret; -} - static void __i915_gem_userptr_get_pages_worker(struct work_struct *_work) { @@ -682,8 +667,11 @@ i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj, struct sgt_iter sgt_iter; struct page *page; - BUG_ON(obj->userptr.work != NULL); + /* Cancel any inflight work and force them to restart their gup */ + obj->userptr.work = NULL; __i915_gem_userptr_set_active(obj, false); + if (!pages) + return; if (obj->mm.madv != I915_MADV_WILLNEED) obj->mm.dirty = false; @@ -721,7 +709,8 @@ i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj) static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = { .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | - I915_GEM_OBJECT_IS_SHRINKABLE, + I915_GEM_OBJECT_IS_SHRINKABLE | + I915_GEM_OBJECT_ASYNC_CANCEL, .get_pages = i915_gem_userptr_get_pages, .put_pages = i915_gem_userptr_put_pages, .dmabuf_export = i915_gem_userptr_dmabuf_export, @@ -791,8 +780,7 @@ i915_gem_userptr_ioctl(struct drm_device *dev, if (offset_in_page(args->user_ptr | args->user_size)) return -EINVAL; - if (!access_ok(args->flags & I915_USERPTR_READ_ONLY ? VERIFY_READ : VERIFY_WRITE, - (char __user *)(unsigned long)args->user_ptr, args->user_size)) + if (!access_ok((char __user *)(unsigned long)args->user_ptr, args->user_size)) return -EFAULT; if (args->flags & I915_USERPTR_READ_ONLY) { diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index db4128d6c09b..aa6791255252 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -27,11 +27,14 @@ * */ -#include <generated/utsrelease.h> +#include <linux/ascii85.h> +#include <linux/nmi.h> +#include <linux/scatterlist.h> #include <linux/stop_machine.h> +#include <linux/utsname.h> #include <linux/zlib.h> + #include <drm/drm_print.h> -#include <linux/ascii85.h> #include "i915_gpu_error.h" #include "i915_drv.h" @@ -77,112 +80,110 @@ static const char *purgeable_flag(int purgeable) return purgeable ? " purgeable" : ""; } -static bool __i915_error_ok(struct drm_i915_error_state_buf *e) +static void __sg_set_buf(struct scatterlist *sg, + void *addr, unsigned int len, loff_t it) { - - if (!e->err && WARN(e->bytes > (e->size - 1), "overflow")) { - e->err = -ENOSPC; - return false; - } - - if (e->bytes == e->size - 1 || e->err) - return false; - - return true; + sg->page_link = (unsigned long)virt_to_page(addr); + sg->offset = offset_in_page(addr); + sg->length = len; + sg->dma_address = it; } -static bool __i915_error_seek(struct drm_i915_error_state_buf *e, - unsigned len) +static bool __i915_error_grow(struct drm_i915_error_state_buf *e, size_t len) { - if (e->pos + len <= e->start) { - e->pos += len; + if (!len) return false; - } - /* First vsnprintf needs to fit in its entirety for memmove */ - if (len >= e->size) { - e->err = -EIO; - return false; - } + if (e->bytes + len + 1 <= e->size) + return true; - return true; -} + if (e->bytes) { + __sg_set_buf(e->cur++, e->buf, e->bytes, e->iter); + e->iter += e->bytes; + e->buf = NULL; + e->bytes = 0; + } -static void __i915_error_advance(struct drm_i915_error_state_buf *e, - unsigned len) -{ - /* If this is first printf in this window, adjust it so that - * start position matches start of the buffer - */ + if (e->cur == e->end) { + struct scatterlist *sgl; - if (e->pos < e->start) { - const size_t off = e->start - e->pos; + sgl = (typeof(sgl))__get_free_page(GFP_KERNEL); + if (!sgl) { + e->err = -ENOMEM; + return false; + } - /* Should not happen but be paranoid */ - if (off > len || e->bytes) { - e->err = -EIO; - return; + if (e->cur) { + e->cur->offset = 0; + e->cur->length = 0; + e->cur->page_link = + (unsigned long)sgl | SG_CHAIN; + } else { + e->sgl = sgl; } - memmove(e->buf, e->buf + off, len - off); - e->bytes = len - off; - e->pos = e->start; - return; + e->cur = sgl; + e->end = sgl + SG_MAX_SINGLE_ALLOC - 1; } - e->bytes += len; - e->pos += len; + e->size = ALIGN(len + 1, SZ_64K); + e->buf = kmalloc(e->size, GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY); + if (!e->buf) { + e->size = PAGE_ALIGN(len + 1); + e->buf = kmalloc(e->size, GFP_KERNEL); + } + if (!e->buf) { + e->err = -ENOMEM; + return false; + } + + return true; } __printf(2, 0) static void i915_error_vprintf(struct drm_i915_error_state_buf *e, - const char *f, va_list args) + const char *fmt, va_list args) { - unsigned len; + va_list ap; + int len; - if (!__i915_error_ok(e)) + if (e->err) return; - /* Seek the first printf which is hits start position */ - if (e->pos < e->start) { - va_list tmp; - - va_copy(tmp, args); - len = vsnprintf(NULL, 0, f, tmp); - va_end(tmp); - - if (!__i915_error_seek(e, len)) - return; + va_copy(ap, args); + len = vsnprintf(NULL, 0, fmt, ap); + va_end(ap); + if (len <= 0) { + e->err = len; + return; } - len = vsnprintf(e->buf + e->bytes, e->size - e->bytes, f, args); - if (len >= e->size - e->bytes) - len = e->size - e->bytes - 1; + if (!__i915_error_grow(e, len)) + return; - __i915_error_advance(e, len); + GEM_BUG_ON(e->bytes >= e->size); + len = vscnprintf(e->buf + e->bytes, e->size - e->bytes, fmt, args); + if (len < 0) { + e->err = len; + return; + } + e->bytes += len; } -static void i915_error_puts(struct drm_i915_error_state_buf *e, - const char *str) +static void i915_error_puts(struct drm_i915_error_state_buf *e, const char *str) { unsigned len; - if (!__i915_error_ok(e)) + if (e->err || !str) return; len = strlen(str); + if (!__i915_error_grow(e, len)) + return; - /* Seek the first printf which is hits start position */ - if (e->pos < e->start) { - if (!__i915_error_seek(e, len)) - return; - } - - if (len >= e->size - e->bytes) - len = e->size - e->bytes - 1; + GEM_BUG_ON(e->bytes + len > e->size); memcpy(e->buf + e->bytes, str, len); - - __i915_error_advance(e, len); + e->bytes += len; } #define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__) @@ -268,6 +269,8 @@ static int compress_page(struct compress *c, if (zlib_deflate(zstream, Z_NO_FLUSH) != Z_OK) return -EIO; + + touch_nmi_watchdog(); } while (zstream->avail_in); /* Fallback to uncompressed if we increase size? */ @@ -444,9 +447,14 @@ static void error_print_request(struct drm_i915_error_state_buf *m, if (!erq->seqno) return; - err_printf(m, "%s pid %d, ban score %d, seqno %8x:%08x, prio %d, emitted %dms, start %08x, head %08x, tail %08x\n", + err_printf(m, "%s pid %d, ban score %d, seqno %8x:%08x%s%s, prio %d, emitted %dms, start %08x, head %08x, tail %08x\n", prefix, erq->pid, erq->ban_score, - erq->context, erq->seqno, erq->sched_attr.priority, + erq->context, erq->seqno, + test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, + &erq->flags) ? "!" : "", + test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, + &erq->flags) ? "+" : "", + erq->sched_attr.priority, jiffies_to_msecs(erq->jiffies - epoch), erq->start, erq->head, erq->tail); } @@ -512,7 +520,7 @@ static void error_print_engine(struct drm_i915_error_state_buf *m, err_printf(m, " SYNC_2: 0x%08x\n", ee->semaphore_mboxes[2]); } - if (USES_PPGTT(m->i915)) { + if (HAS_PPGTT(m->i915)) { err_printf(m, " GFX_MODE: 0x%08x\n", ee->vm_info.gfx_mode); if (INTEL_GEN(m->i915) >= 8) { @@ -527,13 +535,9 @@ static void error_print_engine(struct drm_i915_error_state_buf *m, } err_printf(m, " seqno: 0x%08x\n", ee->seqno); err_printf(m, " last_seqno: 0x%08x\n", ee->last_seqno); - err_printf(m, " waiting: %s\n", yesno(ee->waiting)); err_printf(m, " ring->head: 0x%08x\n", ee->cpu_ring_head); err_printf(m, " ring->tail: 0x%08x\n", ee->cpu_ring_tail); - err_printf(m, " hangcheck stall: %s\n", yesno(ee->hangcheck_stalled)); - err_printf(m, " hangcheck action: %s\n", - hangcheck_action_to_str(ee->hangcheck_action)); - err_printf(m, " hangcheck action timestamp: %dms (%lu%s)\n", + err_printf(m, " hangcheck timestamp: %dms (%lu%s)\n", jiffies_to_msecs(ee->hangcheck_timestamp - epoch), ee->hangcheck_timestamp, ee->hangcheck_timestamp == epoch ? "; epoch" : ""); @@ -591,13 +595,14 @@ static void print_error_obj(struct drm_i915_error_state_buf *m, static void err_print_capabilities(struct drm_i915_error_state_buf *m, const struct intel_device_info *info, + const struct intel_runtime_info *runtime, const struct intel_driver_caps *caps) { struct drm_printer p = i915_error_printer(m); intel_device_info_dump_flags(info, &p); intel_driver_caps_print(caps, &p); - intel_device_info_dump_topology(&info->sseu, &p); + intel_device_info_dump_topology(&runtime->sseu, &p); } static void err_print_params(struct drm_i915_error_state_buf *m, @@ -635,25 +640,35 @@ static void err_print_uc(struct drm_i915_error_state_buf *m, print_error_obj(m, NULL, "GuC log buffer", error_uc->guc_log); } -int i915_error_state_to_str(struct drm_i915_error_state_buf *m, - const struct i915_gpu_state *error) +static void err_free_sgl(struct scatterlist *sgl) { - struct drm_i915_private *dev_priv = m->i915; - struct drm_i915_error_object *obj; - struct timespec64 ts; - int i, j; + while (sgl) { + struct scatterlist *sg; - if (!error) { - err_printf(m, "No error state collected\n"); - return 0; + for (sg = sgl; !sg_is_chain(sg); sg++) { + kfree(sg_virt(sg)); + if (sg_is_last(sg)) + break; + } + + sg = sg_is_last(sg) ? NULL : sg_chain_ptr(sg); + free_page((unsigned long)sgl); + sgl = sg; } +} - if (IS_ERR(error)) - return PTR_ERR(error); +static void __err_print_to_sgl(struct drm_i915_error_state_buf *m, + struct i915_gpu_state *error) +{ + struct drm_i915_error_object *obj; + struct timespec64 ts; + int i, j; if (*error->error_msg) err_printf(m, "%s\n", error->error_msg); - err_printf(m, "Kernel: " UTS_RELEASE "\n"); + err_printf(m, "Kernel: %s %s\n", + init_utsname()->release, + init_utsname()->machine); ts = ktime_to_timespec64(error->time); err_printf(m, "Time: %lld s %ld us\n", (s64)ts.tv_sec, ts.tv_nsec / NSEC_PER_USEC); @@ -670,25 +685,25 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, jiffies_to_msecs(error->capture - error->epoch)); for (i = 0; i < ARRAY_SIZE(error->engine); i++) { - if (error->engine[i].hangcheck_stalled && - error->engine[i].context.pid) { - err_printf(m, "Active process (on ring %s): %s [%d], score %d%s\n", - engine_name(m->i915, i), - error->engine[i].context.comm, - error->engine[i].context.pid, - error->engine[i].context.ban_score, - bannable(&error->engine[i].context)); - } + if (!error->engine[i].context.pid) + continue; + + err_printf(m, "Active process (on ring %s): %s [%d], score %d%s\n", + engine_name(m->i915, i), + error->engine[i].context.comm, + error->engine[i].context.pid, + error->engine[i].context.ban_score, + bannable(&error->engine[i].context)); } err_printf(m, "Reset count: %u\n", error->reset_count); err_printf(m, "Suspend count: %u\n", error->suspend_count); err_printf(m, "Platform: %s\n", intel_platform_name(error->device_info.platform)); - err_print_pciid(m, error->i915); + err_print_pciid(m, m->i915); err_printf(m, "IOMMU enabled?: %d\n", error->iommu); - if (HAS_CSR(dev_priv)) { - struct intel_csr *csr = &dev_priv->csr; + if (HAS_CSR(m->i915)) { + struct intel_csr *csr = &m->i915->csr; err_printf(m, "DMC loaded: %s\n", yesno(csr->dmc_payload != NULL)); @@ -708,22 +723,21 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, err_printf(m, "FORCEWAKE: 0x%08x\n", error->forcewake); err_printf(m, "DERRMR: 0x%08x\n", error->derrmr); err_printf(m, "CCID: 0x%08x\n", error->ccid); - err_printf(m, "Missed interrupts: 0x%08lx\n", dev_priv->gpu_error.missed_irq_rings); for (i = 0; i < error->nfence; i++) err_printf(m, " fence[%d] = %08llx\n", i, error->fence[i]); - if (INTEL_GEN(dev_priv) >= 6) { + if (INTEL_GEN(m->i915) >= 6) { err_printf(m, "ERROR: 0x%08x\n", error->error); - if (INTEL_GEN(dev_priv) >= 8) + if (INTEL_GEN(m->i915) >= 8) err_printf(m, "FAULT_TLB_DATA: 0x%08x 0x%08x\n", error->fault_data1, error->fault_data0); err_printf(m, "DONE_REG: 0x%08x\n", error->done_reg); } - if (IS_GEN7(dev_priv)) + if (IS_GEN(m->i915, 7)) err_printf(m, "ERR_INT: 0x%08x\n", error->err_int); for (i = 0; i < ARRAY_SIZE(error->engine); i++) { @@ -745,7 +759,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, len += scnprintf(buf + len, sizeof(buf), "%s%s", first ? "" : ", ", - dev_priv->engine[j]->name); + m->i915->engine[j]->name); first = 0; } scnprintf(buf + len, sizeof(buf), ")"); @@ -763,7 +777,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, obj = ee->batchbuffer; if (obj) { - err_puts(m, dev_priv->engine[i]->name); + err_puts(m, m->i915->engine[i]->name); if (ee->context.pid) err_printf(m, " (submitted by %s [%d], ctx %d [%d], score %d%s)", ee->context.comm, @@ -775,16 +789,16 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, err_printf(m, " --- gtt_offset = 0x%08x %08x\n", upper_32_bits(obj->gtt_offset), lower_32_bits(obj->gtt_offset)); - print_error_obj(m, dev_priv->engine[i], NULL, obj); + print_error_obj(m, m->i915->engine[i], NULL, obj); } for (j = 0; j < ee->user_bo_count; j++) - print_error_obj(m, dev_priv->engine[i], + print_error_obj(m, m->i915->engine[i], "user", ee->user_bo[j]); if (ee->num_requests) { err_printf(m, "%s --- %d requests\n", - dev_priv->engine[i]->name, + m->i915->engine[i]->name, ee->num_requests); for (j = 0; j < ee->num_requests; j++) error_print_request(m, " ", @@ -792,37 +806,22 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, error->epoch); } - if (IS_ERR(ee->waiters)) { - err_printf(m, "%s --- ? waiters [unable to acquire spinlock]\n", - dev_priv->engine[i]->name); - } else if (ee->num_waiters) { - err_printf(m, "%s --- %d waiters\n", - dev_priv->engine[i]->name, - ee->num_waiters); - for (j = 0; j < ee->num_waiters; j++) { - err_printf(m, " seqno 0x%08x for %s [%d]\n", - ee->waiters[j].seqno, - ee->waiters[j].comm, - ee->waiters[j].pid); - } - } - - print_error_obj(m, dev_priv->engine[i], + print_error_obj(m, m->i915->engine[i], "ringbuffer", ee->ringbuffer); - print_error_obj(m, dev_priv->engine[i], + print_error_obj(m, m->i915->engine[i], "HW Status", ee->hws_page); - print_error_obj(m, dev_priv->engine[i], + print_error_obj(m, m->i915->engine[i], "HW context", ee->ctx); - print_error_obj(m, dev_priv->engine[i], + print_error_obj(m, m->i915->engine[i], "WA context", ee->wa_ctx); - print_error_obj(m, dev_priv->engine[i], + print_error_obj(m, m->i915->engine[i], "WA batchbuffer", ee->wa_batchbuffer); - print_error_obj(m, dev_priv->engine[i], + print_error_obj(m, m->i915->engine[i], "NULL context", ee->default_state); } @@ -832,46 +831,111 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, if (error->display) intel_display_print_error_state(m, error->display); - err_print_capabilities(m, &error->device_info, &error->driver_caps); + err_print_capabilities(m, &error->device_info, &error->runtime_info, + &error->driver_caps); err_print_params(m, &error->params); err_print_uc(m, &error->uc); +} + +static int err_print_to_sgl(struct i915_gpu_state *error) +{ + struct drm_i915_error_state_buf m; - if (m->bytes == 0 && m->err) - return m->err; + if (IS_ERR(error)) + return PTR_ERR(error); + + if (READ_ONCE(error->sgl)) + return 0; + + memset(&m, 0, sizeof(m)); + m.i915 = error->i915; + + __err_print_to_sgl(&m, error); + + if (m.buf) { + __sg_set_buf(m.cur++, m.buf, m.bytes, m.iter); + m.bytes = 0; + m.buf = NULL; + } + if (m.cur) { + GEM_BUG_ON(m.end < m.cur); + sg_mark_end(m.cur - 1); + } + GEM_BUG_ON(m.sgl && !m.cur); + + if (m.err) { + err_free_sgl(m.sgl); + return m.err; + } + + if (cmpxchg(&error->sgl, NULL, m.sgl)) + err_free_sgl(m.sgl); return 0; } -int i915_error_state_buf_init(struct drm_i915_error_state_buf *ebuf, - struct drm_i915_private *i915, - size_t count, loff_t pos) +ssize_t i915_gpu_state_copy_to_buffer(struct i915_gpu_state *error, + char *buf, loff_t off, size_t rem) { - memset(ebuf, 0, sizeof(*ebuf)); - ebuf->i915 = i915; + struct scatterlist *sg; + size_t count; + loff_t pos; + int err; - /* We need to have enough room to store any i915_error_state printf - * so that we can move it to start position. - */ - ebuf->size = count + 1 > PAGE_SIZE ? count + 1 : PAGE_SIZE; - ebuf->buf = kmalloc(ebuf->size, - GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN); + if (!error || !rem) + return 0; - if (ebuf->buf == NULL) { - ebuf->size = PAGE_SIZE; - ebuf->buf = kmalloc(ebuf->size, GFP_KERNEL); - } + err = err_print_to_sgl(error); + if (err) + return err; - if (ebuf->buf == NULL) { - ebuf->size = 128; - ebuf->buf = kmalloc(ebuf->size, GFP_KERNEL); - } + sg = READ_ONCE(error->fit); + if (!sg || off < sg->dma_address) + sg = error->sgl; + if (!sg) + return 0; - if (ebuf->buf == NULL) - return -ENOMEM; + pos = sg->dma_address; + count = 0; + do { + size_t len, start; - ebuf->start = pos; + if (sg_is_chain(sg)) { + sg = sg_chain_ptr(sg); + GEM_BUG_ON(sg_is_chain(sg)); + } - return 0; + len = sg->length; + if (pos + len <= off) { + pos += len; + continue; + } + + start = sg->offset; + if (pos < off) { + GEM_BUG_ON(off - pos > len); + len -= off - pos; + start += off - pos; + pos = off; + } + + len = min(len, rem); + GEM_BUG_ON(!len || len > sg->length); + + memcpy(buf, page_address(sg_page(sg)) + start, len); + + count += len; + pos += len; + + buf += len; + rem -= len; + if (!rem) { + WRITE_ONCE(error->fit, sg); + break; + } + } while (!sg_is_last(sg++)); + + return count; } static void i915_error_object_free(struct drm_i915_error_object *obj) @@ -887,17 +951,10 @@ static void i915_error_object_free(struct drm_i915_error_object *obj) kfree(obj); } -static __always_inline void free_param(const char *type, void *x) -{ - if (!__builtin_strcmp(type, "char *")) - kfree(*(void **)x); -} static void cleanup_params(struct i915_gpu_state *error) { -#define FREE(T, x, ...) free_param(#T, &error->params.x); - I915_PARAMS_FOR_EACH(FREE); -#undef FREE + i915_params_free(&error->params); } static void cleanup_uc_state(struct i915_gpu_state *error) @@ -930,8 +987,6 @@ void __i915_gpu_state_free(struct kref *error_ref) i915_error_object_free(ee->wa_ctx); kfree(ee->requests); - if (!IS_ERR_OR_NULL(ee->waiters)) - kfree(ee->waiters); } for (i = 0; i < ARRAY_SIZE(error->active_bo); i++) @@ -944,6 +999,7 @@ void __i915_gpu_state_free(struct kref *error_ref) cleanup_params(error); cleanup_uc_state(error); + err_free_sgl(error->sgl); kfree(error); } @@ -960,7 +1016,7 @@ i915_error_object_create(struct drm_i915_private *i915, dma_addr_t dma; int ret; - if (!vma) + if (!vma || !vma->pages) return NULL; num_pages = min_t(u64, vma->size, vma->obj->base.size) >> PAGE_SHIFT; @@ -1002,28 +1058,27 @@ i915_error_object_create(struct drm_i915_private *i915, } compress_fini(&compress, dst); - ggtt->vm.clear_range(&ggtt->vm, slot, PAGE_SIZE); return dst; } /* The error capture is special as tries to run underneath the normal - * locking rules - so we use the raw version of the i915_gem_active lookup. + * locking rules - so we use the raw version of the i915_active_request lookup. */ -static inline uint32_t -__active_get_seqno(struct i915_gem_active *active) +static inline u32 +__active_get_seqno(struct i915_active_request *active) { struct i915_request *request; - request = __i915_gem_active_peek(active); + request = __i915_active_request_peek(active); return request ? request->global_seqno : 0; } static inline int -__active_get_engine_id(struct i915_gem_active *active) +__active_get_engine_id(struct i915_active_request *active) { struct i915_request *request; - request = __i915_gem_active_peek(active); + request = __i915_active_request_peek(active); return request ? request->engine->id : -1; } @@ -1051,7 +1106,9 @@ static void capture_bo(struct drm_i915_error_buffer *err, static u32 capture_error_bo(struct drm_i915_error_buffer *err, int count, struct list_head *head, - bool pinned_only) + unsigned int flags) +#define ACTIVE_ONLY BIT(0) +#define PINNED_ONLY BIT(1) { struct i915_vma *vma; int i = 0; @@ -1060,7 +1117,10 @@ static u32 capture_error_bo(struct drm_i915_error_buffer *err, if (!vma->obj) continue; - if (pinned_only && !i915_vma_is_pinned(vma)) + if (flags & ACTIVE_ONLY && !i915_vma_is_active(vma)) + continue; + + if (flags & PINNED_ONLY && !i915_vma_is_pinned(vma)) continue; capture_bo(err++, vma); @@ -1071,7 +1131,8 @@ static u32 capture_error_bo(struct drm_i915_error_buffer *err, return i; } -/* Generate a semi-unique error code. The code is not meant to have meaning, The +/* + * Generate a semi-unique error code. The code is not meant to have meaning, The * code's only purpose is to try to prevent false duplicated bug reports by * grossly estimating a GPU error state. * @@ -1080,29 +1141,23 @@ static u32 capture_error_bo(struct drm_i915_error_buffer *err, * * It's only a small step better than a random number in its current form. */ -static uint32_t i915_error_generate_code(struct drm_i915_private *dev_priv, - struct i915_gpu_state *error, - int *engine_id) +static u32 i915_error_generate_code(struct i915_gpu_state *error, + unsigned long engine_mask) { - uint32_t error_code = 0; - int i; - - /* IPEHR would be an ideal way to detect errors, as it's the gross + /* + * IPEHR would be an ideal way to detect errors, as it's the gross * measure of "the command that hung." However, has some very common * synchronization commands which almost always appear in the case * strictly a client bug. Use instdone to differentiate those some. */ - for (i = 0; i < I915_NUM_ENGINES; i++) { - if (error->engine[i].hangcheck_stalled) { - if (engine_id) - *engine_id = i; + if (engine_mask) { + struct drm_i915_error_engine *ee = + &error->engine[ffs(engine_mask)]; - return error->engine[i].ipehr ^ - error->engine[i].instdone.instdone; - } + return ee->ipehr ^ ee->instdone.instdone; } - return error_code; + return 0; } static void gem_record_fences(struct i915_gpu_state *error) @@ -1135,59 +1190,6 @@ static void gen6_record_semaphore_state(struct intel_engine_cs *engine, I915_READ(RING_SYNC_2(engine->mmio_base)); } -static void error_record_engine_waiters(struct intel_engine_cs *engine, - struct drm_i915_error_engine *ee) -{ - struct intel_breadcrumbs *b = &engine->breadcrumbs; - struct drm_i915_error_waiter *waiter; - struct rb_node *rb; - int count; - - ee->num_waiters = 0; - ee->waiters = NULL; - - if (RB_EMPTY_ROOT(&b->waiters)) - return; - - if (!spin_trylock_irq(&b->rb_lock)) { - ee->waiters = ERR_PTR(-EDEADLK); - return; - } - - count = 0; - for (rb = rb_first(&b->waiters); rb != NULL; rb = rb_next(rb)) - count++; - spin_unlock_irq(&b->rb_lock); - - waiter = NULL; - if (count) - waiter = kmalloc_array(count, - sizeof(struct drm_i915_error_waiter), - GFP_ATOMIC); - if (!waiter) - return; - - if (!spin_trylock_irq(&b->rb_lock)) { - kfree(waiter); - ee->waiters = ERR_PTR(-EDEADLK); - return; - } - - ee->waiters = waiter; - for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) { - struct intel_wait *w = rb_entry(rb, typeof(*w), node); - - strcpy(waiter->comm, w->tsk->comm); - waiter->pid = w->tsk->pid; - waiter->seqno = w->seqno; - waiter++; - - if (++ee->num_waiters == count) - break; - } - spin_unlock_irq(&b->rb_lock); -} - static void error_record_engine_registers(struct i915_gpu_state *error, struct intel_engine_cs *engine, struct drm_i915_error_engine *ee) @@ -1223,7 +1225,6 @@ static void error_record_engine_registers(struct i915_gpu_state *error, intel_engine_get_instdone(engine, &ee->instdone); - ee->waiting = intel_engine_has_waiter(engine); ee->instpm = I915_READ(RING_INSTPM(engine->mmio_base)); ee->acthd = intel_engine_get_active_head(engine); ee->seqno = intel_engine_get_seqno(engine); @@ -1238,7 +1239,7 @@ static void error_record_engine_registers(struct i915_gpu_state *error, if (!HWS_NEEDS_PHYSICAL(dev_priv)) { i915_reg_t mmio; - if (IS_GEN7(dev_priv)) { + if (IS_GEN(dev_priv, 7)) { switch (engine->id) { default: case RCS: @@ -1254,7 +1255,7 @@ static void error_record_engine_registers(struct i915_gpu_state *error, mmio = VEBOX_HWS_PGA_GEN7; break; } - } else if (IS_GEN6(engine->i915)) { + } else if (IS_GEN(engine->i915, 6)) { mmio = RING_HWS_PGA_GEN6(engine->mmio_base); } else { /* XXX: gen8 returns to sanity */ @@ -1265,21 +1266,20 @@ static void error_record_engine_registers(struct i915_gpu_state *error, } ee->idle = intel_engine_is_idle(engine); - ee->hangcheck_timestamp = engine->hangcheck.action_timestamp; - ee->hangcheck_action = engine->hangcheck.action; - ee->hangcheck_stalled = engine->hangcheck.stalled; + if (!ee->idle) + ee->hangcheck_timestamp = engine->hangcheck.action_timestamp; ee->reset_count = i915_reset_engine_count(&dev_priv->gpu_error, engine); - if (USES_PPGTT(dev_priv)) { + if (HAS_PPGTT(dev_priv)) { int i; ee->vm_info.gfx_mode = I915_READ(RING_MODE_GEN7(engine)); - if (IS_GEN6(dev_priv)) + if (IS_GEN(dev_priv, 6)) ee->vm_info.pp_dir_base = I915_READ(RING_PP_DIR_BASE_READ(engine)); - else if (IS_GEN7(dev_priv)) + else if (IS_GEN(dev_priv, 7)) ee->vm_info.pp_dir_base = I915_READ(RING_PP_DIR_BASE(engine)); else if (INTEL_GEN(dev_priv) >= 8) @@ -1298,6 +1298,7 @@ static void record_request(struct i915_request *request, { struct i915_gem_context *ctx = request->gem_context; + erq->flags = request->fence.flags; erq->context = ctx->hw_id; erq->sched_attr = request->sched.attr; erq->ban_score = atomic_read(&ctx->ban_score); @@ -1473,7 +1474,6 @@ static void gem_record_rings(struct i915_gpu_state *error) ee->engine_id = i; error_record_engine_registers(error, engine, ee); - error_record_engine_waiters(engine, ee); error_record_engine_execlists(engine, ee); request = i915_gem_find_active_request(engine); @@ -1537,14 +1537,17 @@ static void gem_capture_vm(struct i915_gpu_state *error, int count; count = 0; - list_for_each_entry(vma, &vm->active_list, vm_link) - count++; + list_for_each_entry(vma, &vm->bound_list, vm_link) + if (i915_vma_is_active(vma)) + count++; active_bo = NULL; if (count) active_bo = kcalloc(count, sizeof(*active_bo), GFP_ATOMIC); if (active_bo) - count = capture_error_bo(active_bo, count, &vm->active_list, false); + count = capture_error_bo(active_bo, + count, &vm->bound_list, + ACTIVE_ONLY); else count = 0; @@ -1582,28 +1585,20 @@ static void capture_pinned_buffers(struct i915_gpu_state *error) struct i915_address_space *vm = &error->i915->ggtt.vm; struct drm_i915_error_buffer *bo; struct i915_vma *vma; - int count_inactive, count_active; - - count_inactive = 0; - list_for_each_entry(vma, &vm->inactive_list, vm_link) - count_inactive++; + int count; - count_active = 0; - list_for_each_entry(vma, &vm->active_list, vm_link) - count_active++; + count = 0; + list_for_each_entry(vma, &vm->bound_list, vm_link) + count++; bo = NULL; - if (count_inactive + count_active) - bo = kcalloc(count_inactive + count_active, - sizeof(*bo), GFP_ATOMIC); + if (count) + bo = kcalloc(count, sizeof(*bo), GFP_ATOMIC); if (!bo) return; - count_inactive = capture_error_bo(bo, count_inactive, - &vm->active_list, true); - count_active = capture_error_bo(bo + count_inactive, count_active, - &vm->inactive_list, true); - error->pinned_bo_count = count_inactive + count_active; + error->pinned_bo_count = + capture_error_bo(bo, count, &vm->bound_list, PINNED_ONLY); error->pinned_bo = bo; } @@ -1649,7 +1644,7 @@ static void capture_reg_state(struct i915_gpu_state *error) error->forcewake = I915_READ_FW(FORCEWAKE_VLV); } - if (IS_GEN7(dev_priv)) + if (IS_GEN(dev_priv, 7)) error->err_int = I915_READ(GEN7_ERR_INT); if (INTEL_GEN(dev_priv) >= 8) { @@ -1657,7 +1652,7 @@ static void capture_reg_state(struct i915_gpu_state *error) error->fault_data1 = I915_READ(GEN8_FAULT_TLB_DATA1); } - if (IS_GEN6(dev_priv)) { + if (IS_GEN(dev_priv, 6)) { error->forcewake = I915_READ_FW(FORCEWAKE); error->gab_ctl = I915_READ(GAB_CTL); error->gfx_mode = I915_READ(GFX_MODE); @@ -1677,7 +1672,7 @@ static void capture_reg_state(struct i915_gpu_state *error) error->ccid = I915_READ(CCID); /* 3: Feature specific registers */ - if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) { + if (IS_GEN_RANGE(dev_priv, 6, 7)) { error->gam_ecochk = I915_READ(GAM_ECOCHK); error->gac_eco = I915_READ(GAC_ECO_BITS); } @@ -1701,7 +1696,7 @@ static void capture_reg_state(struct i915_gpu_state *error) error->ier = I915_READ(DEIER); error->gtier[0] = I915_READ(GTIER); error->ngtier = 1; - } else if (IS_GEN2(dev_priv)) { + } else if (IS_GEN(dev_priv, 2)) { error->ier = I915_READ16(IER); } else if (!IS_VALLEYVIEW(dev_priv)) { error->ier = I915_READ(IER); @@ -1710,31 +1705,35 @@ static void capture_reg_state(struct i915_gpu_state *error) error->pgtbl_er = I915_READ(PGTBL_ER); } -static void i915_error_capture_msg(struct drm_i915_private *dev_priv, - struct i915_gpu_state *error, - u32 engine_mask, - const char *error_msg) +static const char * +error_msg(struct i915_gpu_state *error, unsigned long engines, const char *msg) { - u32 ecode; - int engine_id = -1, len; + int len; + int i; - ecode = i915_error_generate_code(dev_priv, error, &engine_id); + for (i = 0; i < ARRAY_SIZE(error->engine); i++) + if (!error->engine[i].context.pid) + engines &= ~BIT(i); len = scnprintf(error->error_msg, sizeof(error->error_msg), - "GPU HANG: ecode %d:%d:0x%08x", - INTEL_GEN(dev_priv), engine_id, ecode); - - if (engine_id != -1 && error->engine[engine_id].context.pid) + "GPU HANG: ecode %d:%lx:0x%08x", + INTEL_GEN(error->i915), engines, + i915_error_generate_code(error, engines)); + if (engines) { + /* Just show the first executing process, more is confusing */ + i = __ffs(engines); len += scnprintf(error->error_msg + len, sizeof(error->error_msg) - len, ", in %s [%d]", - error->engine[engine_id].context.comm, - error->engine[engine_id].context.pid); + error->engine[i].context.comm, + error->engine[i].context.pid); + } + if (msg) + len += scnprintf(error->error_msg + len, + sizeof(error->error_msg) - len, + ", %s", msg); - scnprintf(error->error_msg + len, sizeof(error->error_msg) - len, - ", reason: %s, action: %s", - error_msg, - engine_mask ? "reset" : "continue"); + return error->error_msg; } static void capture_gen_state(struct i915_gpu_state *error) @@ -1755,21 +1754,15 @@ static void capture_gen_state(struct i915_gpu_state *error) memcpy(&error->device_info, INTEL_INFO(i915), sizeof(error->device_info)); + memcpy(&error->runtime_info, + RUNTIME_INFO(i915), + sizeof(error->runtime_info)); error->driver_caps = i915->caps; } -static __always_inline void dup_param(const char *type, void *x) -{ - if (!__builtin_strcmp(type, "char *")) - *(void **)x = kstrdup(*(void **)x, GFP_ATOMIC); -} - static void capture_params(struct i915_gpu_state *error) { - error->params = i915_modparams; -#define DUP(T, x, ...) dup_param(#T, &error->params.x); - I915_PARAMS_FOR_EACH(DUP); -#undef DUP + i915_params_copy(&error->params, &i915_modparams); } static unsigned long capture_find_epoch(const struct i915_gpu_state *error) @@ -1780,7 +1773,7 @@ static unsigned long capture_find_epoch(const struct i915_gpu_state *error) for (i = 0; i < ARRAY_SIZE(error->engine); i++) { const struct drm_i915_error_engine *ee = &error->engine[i]; - if (ee->hangcheck_stalled && + if (ee->hangcheck_timestamp && time_before(ee->hangcheck_timestamp, epoch)) epoch = ee->hangcheck_timestamp; } @@ -1788,6 +1781,14 @@ static unsigned long capture_find_epoch(const struct i915_gpu_state *error) return epoch; } +static void capture_finish(struct i915_gpu_state *error) +{ + struct i915_ggtt *ggtt = &error->i915->ggtt; + const u64 slot = ggtt->error_capture.start; + + ggtt->vm.clear_range(&ggtt->vm, slot, PAGE_SIZE); +} + static int capture(void *data) { struct i915_gpu_state *error = data; @@ -1812,6 +1813,7 @@ static int capture(void *data) error->epoch = capture_find_epoch(error); + capture_finish(error); return 0; } @@ -1822,9 +1824,16 @@ i915_capture_gpu_state(struct drm_i915_private *i915) { struct i915_gpu_state *error; + /* Check if GPU capture has been disabled */ + error = READ_ONCE(i915->gpu_error.first_error); + if (IS_ERR(error)) + return error; + error = kzalloc(sizeof(*error), GFP_ATOMIC); - if (!error) - return NULL; + if (!error) { + i915_disable_error_state(i915, -ENOMEM); + return ERR_PTR(-ENOMEM); + } kref_init(&error->ref); error->i915 = i915; @@ -1838,7 +1847,7 @@ i915_capture_gpu_state(struct drm_i915_private *i915) * i915_capture_error_state - capture an error record for later analysis * @i915: i915 device * @engine_mask: the mask of engines triggering the hang - * @error_msg: a message to insert into the error capture header + * @msg: a message to insert into the error capture header * * Should be called when an error is detected (either a hang or an error * interrupt) to capture error state from the time of the error. Fills @@ -1846,8 +1855,8 @@ i915_capture_gpu_state(struct drm_i915_private *i915) * to pick up. */ void i915_capture_error_state(struct drm_i915_private *i915, - u32 engine_mask, - const char *error_msg) + unsigned long engine_mask, + const char *msg) { static bool warned; struct i915_gpu_state *error; @@ -1860,14 +1869,10 @@ void i915_capture_error_state(struct drm_i915_private *i915, return; error = i915_capture_gpu_state(i915); - if (!error) { - DRM_DEBUG_DRIVER("out of memory, not capturing error state\n"); - i915_disable_error_state(i915, -ENOMEM); + if (IS_ERR(error)) return; - } - i915_error_capture_msg(i915, error, engine_mask, error_msg); - DRM_INFO("%s\n", error->error_msg); + dev_info(i915->drm.dev, "%s\n", error_msg(error, engine_mask, msg)); if (!error->simulated) { spin_lock_irqsave(&i915->gpu_error.lock, flags); @@ -1902,7 +1907,7 @@ i915_first_error_state(struct drm_i915_private *i915) spin_lock_irq(&i915->gpu_error.lock); error = i915->gpu_error.first_error; - if (error) + if (!IS_ERR_OR_NULL(error)) i915_gpu_state_get(error); spin_unlock_irq(&i915->gpu_error.lock); @@ -1915,10 +1920,11 @@ void i915_reset_error_state(struct drm_i915_private *i915) spin_lock_irq(&i915->gpu_error.lock); error = i915->gpu_error.first_error; - i915->gpu_error.first_error = NULL; + if (error != ERR_PTR(-ENODEV)) /* if disabled, always disabled */ + i915->gpu_error.first_error = NULL; spin_unlock_irq(&i915->gpu_error.lock); - if (!IS_ERR(error)) + if (!IS_ERR_OR_NULL(error)) i915_gpu_state_put(error); } diff --git a/drivers/gpu/drm/i915/i915_gpu_error.h b/drivers/gpu/drm/i915/i915_gpu_error.h index 3ec89a504de5..53b1f22dd365 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.h +++ b/drivers/gpu/drm/i915/i915_gpu_error.h @@ -45,6 +45,7 @@ struct i915_gpu_state { u32 reset_count; u32 suspend_count; struct intel_device_info device_info; + struct intel_runtime_info runtime_info; struct intel_driver_caps driver_caps; struct i915_params params; @@ -81,11 +82,7 @@ struct i915_gpu_state { int engine_id; /* Software tracked state */ bool idle; - bool waiting; - int num_waiters; unsigned long hangcheck_timestamp; - bool hangcheck_stalled; - enum intel_engine_hangcheck_action hangcheck_action; struct i915_address_space *vm; int num_requests; u32 reset_count; @@ -148,6 +145,7 @@ struct i915_gpu_state { struct drm_i915_error_object *default_state; struct drm_i915_error_request { + unsigned long flags; long jiffies; pid_t pid; u32 context; @@ -160,12 +158,6 @@ struct i915_gpu_state { } *requests, execlist[EXECLIST_MAX_PORTS]; unsigned int num_ports; - struct drm_i915_error_waiter { - char comm[TASK_COMM_LEN]; - pid_t pid; - u32 seqno; - } *waiters; - struct { u32 gfx_mode; union { @@ -192,8 +184,12 @@ struct i915_gpu_state { } *active_bo[I915_NUM_ENGINES], *pinned_bo; u32 active_bo_count[I915_NUM_ENGINES], pinned_bo_count; struct i915_address_space *active_vm[I915_NUM_ENGINES]; + + struct scatterlist *sgl, *fit; }; +struct i915_gpu_restart; + struct i915_gpu_error { /* For hangcheck timer */ #define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */ @@ -208,8 +204,6 @@ struct i915_gpu_error { atomic_t pending_fb_pin; - unsigned long missed_irq_rings; - /** * State variable controlling the reset flow and count * @@ -244,15 +238,6 @@ struct i915_gpu_error { * i915_mutex_lock_interruptible()?). I915_RESET_BACKOFF serves a * secondary role in preventing two concurrent global reset attempts. * - * #I915_RESET_HANDOFF - To perform the actual GPU reset, we need the - * struct_mutex. We try to acquire the struct_mutex in the reset worker, - * but it may be held by some long running waiter (that we cannot - * interrupt without causing trouble). Once we are ready to do the GPU - * reset, we set the I915_RESET_HANDOFF bit and wakeup any waiters. If - * they already hold the struct_mutex and want to participate they can - * inspect the bit and do the reset directly, otherwise the worker - * waits for the struct_mutex. - * * #I915_RESET_ENGINE[num_engines] - Since the driver doesn't need to * acquire the struct_mutex to reset an engine, we need an explicit * flag to prevent two concurrent reset attempts in the same engine. @@ -266,19 +251,14 @@ struct i915_gpu_error { */ unsigned long flags; #define I915_RESET_BACKOFF 0 -#define I915_RESET_HANDOFF 1 -#define I915_RESET_MODESET 2 +#define I915_RESET_MODESET 1 +#define I915_RESET_ENGINE 2 #define I915_WEDGED (BITS_PER_LONG - 1) -#define I915_RESET_ENGINE (I915_WEDGED - I915_NUM_ENGINES) /** Number of times an engine has been reset */ u32 reset_engine_count[I915_NUM_ENGINES]; - /** Set of stalled engines with guilty requests, in the current reset */ - u32 stalled_mask; - - /** Reason for the current *global* reset */ - const char *reason; + struct mutex wedge_mutex; /* serialises wedging/unwedging */ /** * Waitqueue to signal when a hang is detected. Used to for waiters @@ -292,39 +272,29 @@ struct i915_gpu_error { */ wait_queue_head_t reset_queue; - /* For missed irq/seqno simulation. */ - unsigned long test_irq_rings; + struct i915_gpu_restart *restart; }; struct drm_i915_error_state_buf { struct drm_i915_private *i915; - unsigned int bytes; - unsigned int size; + struct scatterlist *sgl, *cur, *end; + + char *buf; + size_t bytes; + size_t size; + loff_t iter; + int err; - u8 *buf; - loff_t start; - loff_t pos; }; #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) __printf(2, 3) void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...); -int i915_error_state_to_str(struct drm_i915_error_state_buf *estr, - const struct i915_gpu_state *gpu); -int i915_error_state_buf_init(struct drm_i915_error_state_buf *eb, - struct drm_i915_private *i915, - size_t count, loff_t pos); - -static inline void -i915_error_state_buf_release(struct drm_i915_error_state_buf *eb) -{ - kfree(eb->buf); -} struct i915_gpu_state *i915_capture_gpu_state(struct drm_i915_private *i915); void i915_capture_error_state(struct drm_i915_private *dev_priv, - u32 engine_mask, + unsigned long engine_mask, const char *error_msg); static inline struct i915_gpu_state * @@ -334,6 +304,9 @@ i915_gpu_state_get(struct i915_gpu_state *gpu) return gpu; } +ssize_t i915_gpu_state_copy_to_buffer(struct i915_gpu_state *error, + char *buf, loff_t offset, size_t count); + void __i915_gpu_state_free(struct kref *kref); static inline void i915_gpu_state_put(struct i915_gpu_state *gpu) { diff --git a/drivers/gpu/drm/i915/i915_ioc32.c b/drivers/gpu/drm/i915/i915_ioc32.c index 0e5c580d117c..c1007245f46d 100644 --- a/drivers/gpu/drm/i915/i915_ioc32.c +++ b/drivers/gpu/drm/i915/i915_ioc32.c @@ -28,8 +28,8 @@ */ #include <linux/compat.h> -#include <drm/drmP.h> #include <drm/i915_drm.h> +#include <drm/drm_ioctl.h> #include "i915_drv.h" struct drm_i915_getparam32 { @@ -52,7 +52,7 @@ static int compat_i915_getparam(struct file *file, unsigned int cmd, return -EFAULT; request = compat_alloc_user_space(sizeof(*request)); - if (!access_ok(VERIFY_WRITE, request, sizeof(*request)) || + if (!access_ok(request, sizeof(*request)) || __put_user(req32.param, &request->param) || __put_user((void __user *)(unsigned long)req32.value, &request->value)) diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index 2e242270e270..441d2674b272 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -31,7 +31,8 @@ #include <linux/sysrq.h> #include <linux/slab.h> #include <linux/circ_buf.h> -#include <drm/drmP.h> +#include <drm/drm_irq.h> +#include <drm/drm_drv.h> #include <drm/i915_drm.h> #include "i915_drv.h" #include "i915_trace.h" @@ -224,10 +225,10 @@ static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir); /* For display hotplug interrupt */ static inline void i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv, - uint32_t mask, - uint32_t bits) + u32 mask, + u32 bits) { - uint32_t val; + u32 val; lockdep_assert_held(&dev_priv->irq_lock); WARN_ON(bits & ~mask); @@ -251,8 +252,8 @@ i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv, * version is also available. */ void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv, - uint32_t mask, - uint32_t bits) + u32 mask, + u32 bits) { spin_lock_irq(&dev_priv->irq_lock); i915_hotplug_interrupt_update_locked(dev_priv, mask, bits); @@ -301,10 +302,10 @@ static bool gen11_reset_one_iir(struct drm_i915_private * const i915, * @enabled_irq_mask: mask of interrupt bits to enable */ void ilk_update_display_irq(struct drm_i915_private *dev_priv, - uint32_t interrupt_mask, - uint32_t enabled_irq_mask) + u32 interrupt_mask, + u32 enabled_irq_mask) { - uint32_t new_val; + u32 new_val; lockdep_assert_held(&dev_priv->irq_lock); @@ -331,8 +332,8 @@ void ilk_update_display_irq(struct drm_i915_private *dev_priv, * @enabled_irq_mask: mask of interrupt bits to enable */ static void ilk_update_gt_irq(struct drm_i915_private *dev_priv, - uint32_t interrupt_mask, - uint32_t enabled_irq_mask) + u32 interrupt_mask, + u32 enabled_irq_mask) { lockdep_assert_held(&dev_priv->irq_lock); @@ -346,13 +347,13 @@ static void ilk_update_gt_irq(struct drm_i915_private *dev_priv, I915_WRITE(GTIMR, dev_priv->gt_irq_mask); } -void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask) +void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, u32 mask) { ilk_update_gt_irq(dev_priv, mask, mask); POSTING_READ_FW(GTIMR); } -void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask) +void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, u32 mask) { ilk_update_gt_irq(dev_priv, mask, 0); } @@ -391,10 +392,10 @@ static i915_reg_t gen6_pm_ier(struct drm_i915_private *dev_priv) * @enabled_irq_mask: mask of interrupt bits to enable */ static void snb_update_pm_irq(struct drm_i915_private *dev_priv, - uint32_t interrupt_mask, - uint32_t enabled_irq_mask) + u32 interrupt_mask, + u32 enabled_irq_mask) { - uint32_t new_val; + u32 new_val; WARN_ON(enabled_irq_mask & ~interrupt_mask); @@ -578,11 +579,11 @@ void gen9_disable_guc_interrupts(struct drm_i915_private *dev_priv) * @enabled_irq_mask: mask of interrupt bits to enable */ static void bdw_update_port_irq(struct drm_i915_private *dev_priv, - uint32_t interrupt_mask, - uint32_t enabled_irq_mask) + u32 interrupt_mask, + u32 enabled_irq_mask) { - uint32_t new_val; - uint32_t old_val; + u32 new_val; + u32 old_val; lockdep_assert_held(&dev_priv->irq_lock); @@ -612,10 +613,10 @@ static void bdw_update_port_irq(struct drm_i915_private *dev_priv, */ void bdw_update_pipe_irq(struct drm_i915_private *dev_priv, enum pipe pipe, - uint32_t interrupt_mask, - uint32_t enabled_irq_mask) + u32 interrupt_mask, + u32 enabled_irq_mask) { - uint32_t new_val; + u32 new_val; lockdep_assert_held(&dev_priv->irq_lock); @@ -642,10 +643,10 @@ void bdw_update_pipe_irq(struct drm_i915_private *dev_priv, * @enabled_irq_mask: mask of interrupt bits to enable */ void ibx_display_interrupt_update(struct drm_i915_private *dev_priv, - uint32_t interrupt_mask, - uint32_t enabled_irq_mask) + u32 interrupt_mask, + u32 enabled_irq_mask) { - uint32_t sdeimr = I915_READ(SDEIMR); + u32 sdeimr = I915_READ(SDEIMR); sdeimr &= ~interrupt_mask; sdeimr |= (~enabled_irq_mask & interrupt_mask); @@ -822,11 +823,26 @@ static void i915_enable_asle_pipestat(struct drm_i915_private *dev_priv) static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe) { struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; + const struct drm_display_mode *mode = &vblank->hwmode; i915_reg_t high_frame, low_frame; u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal; - const struct drm_display_mode *mode = &dev->vblank[pipe].hwmode; unsigned long irqflags; + /* + * On i965gm TV output the frame counter only works up to + * the point when we enable the TV encoder. After that the + * frame counter ceases to work and reads zero. We need a + * vblank wait before enabling the TV encoder and so we + * have to enable vblank interrupts while the frame counter + * is still in a working state. However the core vblank code + * does not like us returning non-zero frame counter values + * when we've told it that we don't have a working frame + * counter. Thus we must stop non-zero values leaking out. + */ + if (!vblank->max_vblank_count) + return 0; + htotal = mode->crtc_htotal; hsync_start = mode->crtc_hsync_start; vbl_start = mode->crtc_vblank_start; @@ -950,7 +966,7 @@ static int __intel_get_crtc_scanline(struct intel_crtc *crtc) if (mode->flags & DRM_MODE_FLAG_INTERLACE) vtotal /= 2; - if (IS_GEN2(dev_priv)) + if (IS_GEN(dev_priv, 2)) position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN2; else position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3; @@ -998,6 +1014,9 @@ static bool i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe, int position; int vbl_start, vbl_end, hsync_start, htotal, vtotal; unsigned long irqflags; + bool use_scanline_counter = INTEL_GEN(dev_priv) >= 5 || + IS_G4X(dev_priv) || IS_GEN(dev_priv, 2) || + mode->private_flags & I915_MODE_FLAG_USE_SCANLINE_COUNTER; if (WARN_ON(!mode->crtc_clock)) { DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled " @@ -1030,7 +1049,7 @@ static bool i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe, if (stime) *stime = ktime_get(); - if (IS_GEN2(dev_priv) || IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) { + if (use_scanline_counter) { /* No obvious pixelcount register. Only query vertical * scanout position from Display scan line register. */ @@ -1090,7 +1109,7 @@ static bool i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe, else position += vtotal - vbl_end; - if (IS_GEN2(dev_priv) || IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) { + if (use_scanline_counter) { *vpos = position; *hpos = 0; } else { @@ -1152,76 +1171,6 @@ static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv) return; } -static void notify_ring(struct intel_engine_cs *engine) -{ - const u32 seqno = intel_engine_get_seqno(engine); - struct i915_request *rq = NULL; - struct task_struct *tsk = NULL; - struct intel_wait *wait; - - if (unlikely(!engine->breadcrumbs.irq_armed)) - return; - - rcu_read_lock(); - - spin_lock(&engine->breadcrumbs.irq_lock); - wait = engine->breadcrumbs.irq_wait; - if (wait) { - /* - * We use a callback from the dma-fence to submit - * requests after waiting on our own requests. To - * ensure minimum delay in queuing the next request to - * hardware, signal the fence now rather than wait for - * the signaler to be woken up. We still wake up the - * waiter in order to handle the irq-seqno coherency - * issues (we may receive the interrupt before the - * seqno is written, see __i915_request_irq_complete()) - * and to handle coalescing of multiple seqno updates - * and many waiters. - */ - if (i915_seqno_passed(seqno, wait->seqno)) { - struct i915_request *waiter = wait->request; - - if (waiter && - !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, - &waiter->fence.flags) && - intel_wait_check_request(wait, waiter)) - rq = i915_request_get(waiter); - - tsk = wait->tsk; - } else { - if (engine->irq_seqno_barrier && - i915_seqno_passed(seqno, wait->seqno - 1)) { - set_bit(ENGINE_IRQ_BREADCRUMB, - &engine->irq_posted); - tsk = wait->tsk; - } - } - - engine->breadcrumbs.irq_count++; - } else { - if (engine->breadcrumbs.irq_armed) - __intel_engine_disarm_breadcrumbs(engine); - } - spin_unlock(&engine->breadcrumbs.irq_lock); - - if (rq) { - spin_lock(&rq->lock); - dma_fence_signal_locked(&rq->fence); - GEM_BUG_ON(!i915_request_completed(rq)); - spin_unlock(&rq->lock); - - i915_request_put(rq); - } - - if (tsk && tsk->state & TASK_NORMAL) - wake_up_process(tsk); - - rcu_read_unlock(); - - trace_intel_engine_notify(engine, wait); -} - static void vlv_c0_read(struct drm_i915_private *dev_priv, struct intel_rps_ei *ei) { @@ -1376,8 +1325,8 @@ static void ivybridge_parity_work(struct work_struct *work) container_of(work, typeof(*dev_priv), l3_parity.error_work); u32 error_status, row, bank, subbank; char *parity_event[6]; - uint32_t misccpctl; - uint8_t slice = 0; + u32 misccpctl; + u8 slice = 0; /* We must turn off DOP level clock gating to access the L3 registers. * In order to prevent a get/put style interface, acquire struct mutex @@ -1466,20 +1415,20 @@ static void ilk_gt_irq_handler(struct drm_i915_private *dev_priv, u32 gt_iir) { if (gt_iir & GT_RENDER_USER_INTERRUPT) - notify_ring(dev_priv->engine[RCS]); + intel_engine_breadcrumbs_irq(dev_priv->engine[RCS]); if (gt_iir & ILK_BSD_USER_INTERRUPT) - notify_ring(dev_priv->engine[VCS]); + intel_engine_breadcrumbs_irq(dev_priv->engine[VCS]); } static void snb_gt_irq_handler(struct drm_i915_private *dev_priv, u32 gt_iir) { if (gt_iir & GT_RENDER_USER_INTERRUPT) - notify_ring(dev_priv->engine[RCS]); + intel_engine_breadcrumbs_irq(dev_priv->engine[RCS]); if (gt_iir & GT_BSD_USER_INTERRUPT) - notify_ring(dev_priv->engine[VCS]); + intel_engine_breadcrumbs_irq(dev_priv->engine[VCS]); if (gt_iir & GT_BLT_USER_INTERRUPT) - notify_ring(dev_priv->engine[BCS]); + intel_engine_breadcrumbs_irq(dev_priv->engine[BCS]); if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT | GT_BSD_CS_ERROR_INTERRUPT | @@ -1499,7 +1448,7 @@ gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir) tasklet = true; if (iir & GT_RENDER_USER_INTERRUPT) { - notify_ring(engine); + intel_engine_breadcrumbs_irq(engine); tasklet |= USES_GUC_SUBMISSION(engine->i915); } @@ -1738,13 +1687,13 @@ static void dp_aux_irq_handler(struct drm_i915_private *dev_priv) #if defined(CONFIG_DEBUG_FS) static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv, enum pipe pipe, - uint32_t crc0, uint32_t crc1, - uint32_t crc2, uint32_t crc3, - uint32_t crc4) + u32 crc0, u32 crc1, + u32 crc2, u32 crc3, + u32 crc4) { struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe]; struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); - uint32_t crcs[5]; + u32 crcs[5]; spin_lock(&pipe_crc->lock); /* @@ -1776,9 +1725,9 @@ static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv, static inline void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv, enum pipe pipe, - uint32_t crc0, uint32_t crc1, - uint32_t crc2, uint32_t crc3, - uint32_t crc4) {} + u32 crc0, u32 crc1, + u32 crc2, u32 crc3, + u32 crc4) {} #endif @@ -1804,7 +1753,7 @@ static void ivb_pipe_crc_irq_handler(struct drm_i915_private *dev_priv, static void i9xx_pipe_crc_irq_handler(struct drm_i915_private *dev_priv, enum pipe pipe) { - uint32_t res1, res2; + u32 res1, res2; if (INTEL_GEN(dev_priv) >= 3) res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe)); @@ -1845,7 +1794,7 @@ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir) if (HAS_VEBOX(dev_priv)) { if (pm_iir & PM_VEBOX_USER_INTERRUPT) - notify_ring(dev_priv->engine[VECS]); + intel_engine_breadcrumbs_irq(dev_priv->engine[VECS]); if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir); @@ -2547,7 +2496,7 @@ static void ilk_display_irq_handler(struct drm_i915_private *dev_priv, I915_WRITE(SDEIIR, pch_iir); } - if (IS_GEN5(dev_priv) && de_iir & DE_PCU_EVENT) + if (IS_GEN(dev_priv, 5) && de_iir & DE_PCU_EVENT) ironlake_rps_change_irq_handler(dev_priv); } @@ -2887,21 +2836,39 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl) return ret; } +static inline u32 gen8_master_intr_disable(void __iomem * const regs) +{ + raw_reg_write(regs, GEN8_MASTER_IRQ, 0); + + /* + * Now with master disabled, get a sample of level indications + * for this interrupt. Indications will be cleared on related acks. + * New indications can and will light up during processing, + * and will generate new interrupt after enabling master. + */ + return raw_reg_read(regs, GEN8_MASTER_IRQ); +} + +static inline void gen8_master_intr_enable(void __iomem * const regs) +{ + raw_reg_write(regs, GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL); +} + static irqreturn_t gen8_irq_handler(int irq, void *arg) { struct drm_i915_private *dev_priv = to_i915(arg); + void __iomem * const regs = dev_priv->regs; u32 master_ctl; u32 gt_iir[4]; if (!intel_irqs_enabled(dev_priv)) return IRQ_NONE; - master_ctl = I915_READ_FW(GEN8_MASTER_IRQ); - master_ctl &= ~GEN8_MASTER_IRQ_CONTROL; - if (!master_ctl) + master_ctl = gen8_master_intr_disable(regs); + if (!master_ctl) { + gen8_master_intr_enable(regs); return IRQ_NONE; - - I915_WRITE_FW(GEN8_MASTER_IRQ, 0); + } /* Find, clear, then process each source of interrupt */ gen8_gt_irq_ack(dev_priv, master_ctl, gt_iir); @@ -2913,53 +2880,13 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg) enable_rpm_wakeref_asserts(dev_priv); } - I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL); + gen8_master_intr_enable(regs); gen8_gt_irq_handler(dev_priv, master_ctl, gt_iir); return IRQ_HANDLED; } -struct wedge_me { - struct delayed_work work; - struct drm_i915_private *i915; - const char *name; -}; - -static void wedge_me(struct work_struct *work) -{ - struct wedge_me *w = container_of(work, typeof(*w), work.work); - - dev_err(w->i915->drm.dev, - "%s timed out, cancelling all in-flight rendering.\n", - w->name); - i915_gem_set_wedged(w->i915); -} - -static void __init_wedge(struct wedge_me *w, - struct drm_i915_private *i915, - long timeout, - const char *name) -{ - w->i915 = i915; - w->name = name; - - INIT_DELAYED_WORK_ONSTACK(&w->work, wedge_me); - schedule_delayed_work(&w->work, timeout); -} - -static void __fini_wedge(struct wedge_me *w) -{ - cancel_delayed_work_sync(&w->work); - destroy_delayed_work_on_stack(&w->work); - w->i915 = NULL; -} - -#define i915_wedge_on_timeout(W, DEV, TIMEOUT) \ - for (__init_wedge((W), (DEV), (TIMEOUT), __func__); \ - (W)->i915; \ - __fini_wedge((W))) - static u32 gen11_gt_engine_identity(struct drm_i915_private * const i915, const unsigned int bank, const unsigned int bit) @@ -3111,6 +3038,24 @@ gen11_gu_misc_irq_handler(struct drm_i915_private *dev_priv, const u32 iir) intel_opregion_asle_intr(dev_priv); } +static inline u32 gen11_master_intr_disable(void __iomem * const regs) +{ + raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, 0); + + /* + * Now with master disabled, get a sample of level indications + * for this interrupt. Indications will be cleared on related acks. + * New indications can and will light up during processing, + * and will generate new interrupt after enabling master. + */ + return raw_reg_read(regs, GEN11_GFX_MSTR_IRQ); +} + +static inline void gen11_master_intr_enable(void __iomem * const regs) +{ + raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, GEN11_MASTER_IRQ); +} + static irqreturn_t gen11_irq_handler(int irq, void *arg) { struct drm_i915_private * const i915 = to_i915(arg); @@ -3121,13 +3066,11 @@ static irqreturn_t gen11_irq_handler(int irq, void *arg) if (!intel_irqs_enabled(i915)) return IRQ_NONE; - master_ctl = raw_reg_read(regs, GEN11_GFX_MSTR_IRQ); - master_ctl &= ~GEN11_MASTER_IRQ; - if (!master_ctl) + master_ctl = gen11_master_intr_disable(regs); + if (!master_ctl) { + gen11_master_intr_enable(regs); return IRQ_NONE; - - /* Disable interrupts. */ - raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, 0); + } /* Find, clear, then process each source of interrupt. */ gen11_gt_irq_handler(i915, master_ctl); @@ -3147,211 +3090,13 @@ static irqreturn_t gen11_irq_handler(int irq, void *arg) gu_misc_iir = gen11_gu_misc_irq_ack(i915, master_ctl); - /* Acknowledge and enable interrupts. */ - raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, GEN11_MASTER_IRQ | master_ctl); + gen11_master_intr_enable(regs); gen11_gu_misc_irq_handler(i915, gu_misc_iir); return IRQ_HANDLED; } -static void i915_reset_device(struct drm_i915_private *dev_priv, - u32 engine_mask, - const char *reason) -{ - struct i915_gpu_error *error = &dev_priv->gpu_error; - struct kobject *kobj = &dev_priv->drm.primary->kdev->kobj; - char *error_event[] = { I915_ERROR_UEVENT "=1", NULL }; - char *reset_event[] = { I915_RESET_UEVENT "=1", NULL }; - char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL }; - struct wedge_me w; - - kobject_uevent_env(kobj, KOBJ_CHANGE, error_event); - - DRM_DEBUG_DRIVER("resetting chip\n"); - kobject_uevent_env(kobj, KOBJ_CHANGE, reset_event); - - /* Use a watchdog to ensure that our reset completes */ - i915_wedge_on_timeout(&w, dev_priv, 5*HZ) { - intel_prepare_reset(dev_priv); - - error->reason = reason; - error->stalled_mask = engine_mask; - - /* Signal that locked waiters should reset the GPU */ - smp_mb__before_atomic(); - set_bit(I915_RESET_HANDOFF, &error->flags); - wake_up_all(&error->wait_queue); - - /* Wait for anyone holding the lock to wakeup, without - * blocking indefinitely on struct_mutex. - */ - do { - if (mutex_trylock(&dev_priv->drm.struct_mutex)) { - i915_reset(dev_priv, engine_mask, reason); - mutex_unlock(&dev_priv->drm.struct_mutex); - } - } while (wait_on_bit_timeout(&error->flags, - I915_RESET_HANDOFF, - TASK_UNINTERRUPTIBLE, - 1)); - - error->stalled_mask = 0; - error->reason = NULL; - - intel_finish_reset(dev_priv); - } - - if (!test_bit(I915_WEDGED, &error->flags)) - kobject_uevent_env(kobj, KOBJ_CHANGE, reset_done_event); -} - -void i915_clear_error_registers(struct drm_i915_private *dev_priv) -{ - u32 eir; - - if (!IS_GEN2(dev_priv)) - I915_WRITE(PGTBL_ER, I915_READ(PGTBL_ER)); - - if (INTEL_GEN(dev_priv) < 4) - I915_WRITE(IPEIR, I915_READ(IPEIR)); - else - I915_WRITE(IPEIR_I965, I915_READ(IPEIR_I965)); - - I915_WRITE(EIR, I915_READ(EIR)); - eir = I915_READ(EIR); - if (eir) { - /* - * some errors might have become stuck, - * mask them. - */ - DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir); - I915_WRITE(EMR, I915_READ(EMR) | eir); - I915_WRITE(IIR, I915_MASTER_ERROR_INTERRUPT); - } - - if (INTEL_GEN(dev_priv) >= 8) { - I915_WRITE(GEN8_RING_FAULT_REG, - I915_READ(GEN8_RING_FAULT_REG) & ~RING_FAULT_VALID); - POSTING_READ(GEN8_RING_FAULT_REG); - } else if (INTEL_GEN(dev_priv) >= 6) { - struct intel_engine_cs *engine; - enum intel_engine_id id; - - for_each_engine(engine, dev_priv, id) { - I915_WRITE(RING_FAULT_REG(engine), - I915_READ(RING_FAULT_REG(engine)) & - ~RING_FAULT_VALID); - } - POSTING_READ(RING_FAULT_REG(dev_priv->engine[RCS])); - } -} - -/** - * i915_handle_error - handle a gpu error - * @dev_priv: i915 device private - * @engine_mask: mask representing engines that are hung - * @flags: control flags - * @fmt: Error message format string - * - * Do some basic checking of register state at error time and - * dump it to the syslog. Also call i915_capture_error_state() to make - * sure we get a record and make it available in debugfs. Fire a uevent - * so userspace knows something bad happened (should trigger collection - * of a ring dump etc.). - */ -void i915_handle_error(struct drm_i915_private *dev_priv, - u32 engine_mask, - unsigned long flags, - const char *fmt, ...) -{ - struct intel_engine_cs *engine; - unsigned int tmp; - char error_msg[80]; - char *msg = NULL; - - if (fmt) { - va_list args; - - va_start(args, fmt); - vscnprintf(error_msg, sizeof(error_msg), fmt, args); - va_end(args); - - msg = error_msg; - } - - /* - * In most cases it's guaranteed that we get here with an RPM - * reference held, for example because there is a pending GPU - * request that won't finish until the reset is done. This - * isn't the case at least when we get here by doing a - * simulated reset via debugfs, so get an RPM reference. - */ - intel_runtime_pm_get(dev_priv); - - engine_mask &= INTEL_INFO(dev_priv)->ring_mask; - - if (flags & I915_ERROR_CAPTURE) { - i915_capture_error_state(dev_priv, engine_mask, msg); - i915_clear_error_registers(dev_priv); - } - - /* - * Try engine reset when available. We fall back to full reset if - * single reset fails. - */ - if (intel_has_reset_engine(dev_priv) && - !i915_terminally_wedged(&dev_priv->gpu_error)) { - for_each_engine_masked(engine, dev_priv, engine_mask, tmp) { - BUILD_BUG_ON(I915_RESET_MODESET >= I915_RESET_ENGINE); - if (test_and_set_bit(I915_RESET_ENGINE + engine->id, - &dev_priv->gpu_error.flags)) - continue; - - if (i915_reset_engine(engine, msg) == 0) - engine_mask &= ~intel_engine_flag(engine); - - clear_bit(I915_RESET_ENGINE + engine->id, - &dev_priv->gpu_error.flags); - wake_up_bit(&dev_priv->gpu_error.flags, - I915_RESET_ENGINE + engine->id); - } - } - - if (!engine_mask) - goto out; - - /* Full reset needs the mutex, stop any other user trying to do so. */ - if (test_and_set_bit(I915_RESET_BACKOFF, &dev_priv->gpu_error.flags)) { - wait_event(dev_priv->gpu_error.reset_queue, - !test_bit(I915_RESET_BACKOFF, - &dev_priv->gpu_error.flags)); - goto out; - } - - /* Prevent any other reset-engine attempt. */ - for_each_engine(engine, dev_priv, tmp) { - while (test_and_set_bit(I915_RESET_ENGINE + engine->id, - &dev_priv->gpu_error.flags)) - wait_on_bit(&dev_priv->gpu_error.flags, - I915_RESET_ENGINE + engine->id, - TASK_UNINTERRUPTIBLE); - } - - i915_reset_device(dev_priv, engine_mask, msg); - - for_each_engine(engine, dev_priv, tmp) { - clear_bit(I915_RESET_ENGINE + engine->id, - &dev_priv->gpu_error.flags); - } - - clear_bit(I915_RESET_BACKOFF, &dev_priv->gpu_error.flags); - wake_up_all(&dev_priv->gpu_error.reset_queue); - -out: - intel_runtime_pm_put(dev_priv); -} - /* Called from drm generic code, passed 'crtc' which * we use as a pipe index */ @@ -3384,7 +3129,7 @@ static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe) { struct drm_i915_private *dev_priv = to_i915(dev); unsigned long irqflags; - uint32_t bit = INTEL_GEN(dev_priv) >= 7 ? + u32 bit = INTEL_GEN(dev_priv) >= 7 ? DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe); spin_lock_irqsave(&dev_priv->irq_lock, irqflags); @@ -3446,7 +3191,7 @@ static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe) { struct drm_i915_private *dev_priv = to_i915(dev); unsigned long irqflags; - uint32_t bit = INTEL_GEN(dev_priv) >= 7 ? + u32 bit = INTEL_GEN(dev_priv) >= 7 ? DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe); spin_lock_irqsave(&dev_priv->irq_lock, irqflags); @@ -3553,11 +3298,8 @@ static void ironlake_irq_reset(struct drm_device *dev) { struct drm_i915_private *dev_priv = to_i915(dev); - if (IS_GEN5(dev_priv)) - I915_WRITE(HWSTAM, 0xffffffff); - GEN3_IRQ_RESET(DE); - if (IS_GEN7(dev_priv)) + if (IS_GEN(dev_priv, 7)) I915_WRITE(GEN7_ERR_INT, 0xffffffff); if (IS_HASWELL(dev_priv)) { @@ -3598,8 +3340,7 @@ static void gen8_irq_reset(struct drm_device *dev) struct drm_i915_private *dev_priv = to_i915(dev); int pipe; - I915_WRITE(GEN8_MASTER_IRQ, 0); - POSTING_READ(GEN8_MASTER_IRQ); + gen8_master_intr_disable(dev_priv->regs); gen8_gt_irq_reset(dev_priv); @@ -3641,13 +3382,15 @@ static void gen11_irq_reset(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; int pipe; - I915_WRITE(GEN11_GFX_MSTR_IRQ, 0); - POSTING_READ(GEN11_GFX_MSTR_IRQ); + gen11_master_intr_disable(dev_priv->regs); gen11_gt_irq_reset(dev_priv); I915_WRITE(GEN11_DISPLAY_INT_CTL, 0); + I915_WRITE(EDP_PSR_IMR, 0xffffffff); + I915_WRITE(EDP_PSR_IIR, 0xffffffff); + for_each_pipe(dev_priv, pipe) if (intel_display_power_is_enabled(dev_priv, POWER_DOMAIN_PIPE(pipe))) @@ -3666,7 +3409,7 @@ static void gen11_irq_reset(struct drm_device *dev) void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv, u8 pipe_mask) { - uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN; + u32 extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN; enum pipe pipe; spin_lock_irq(&dev_priv->irq_lock); @@ -4011,7 +3754,7 @@ static void gen5_gt_irq_postinstall(struct drm_device *dev) } gt_irqs |= GT_RENDER_USER_INTERRUPT; - if (IS_GEN5(dev_priv)) { + if (IS_GEN(dev_priv, 5)) { gt_irqs |= ILK_BSD_USER_INTERRUPT; } else { gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT; @@ -4135,7 +3878,7 @@ static int valleyview_irq_postinstall(struct drm_device *dev) static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv) { /* These are interrupts we'll toggle with the ring mask register */ - uint32_t gt_interrupts[] = { + u32 gt_interrupts[] = { GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT | GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT | GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT | @@ -4149,9 +3892,6 @@ static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv) GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT }; - if (HAS_L3_DPF(dev_priv)) - gt_interrupts[0] |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT; - dev_priv->pm_ier = 0x0; dev_priv->pm_imr = ~dev_priv->pm_ier; GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]); @@ -4166,8 +3906,8 @@ static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv) static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv) { - uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE; - uint32_t de_pipe_enables; + u32 de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE; + u32 de_pipe_enables; u32 de_port_masked = GEN8_AUX_CHANNEL_A; u32 de_port_enables; u32 de_misc_masked = GEN8_DE_EDP_PSR; @@ -4244,8 +3984,7 @@ static int gen8_irq_postinstall(struct drm_device *dev) if (HAS_PCH_SPLIT(dev_priv)) ibx_irq_postinstall(dev); - I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL); - POSTING_READ(GEN8_MASTER_IRQ); + gen8_master_intr_enable(dev_priv->regs); return 0; } @@ -4307,7 +4046,7 @@ static int gen11_irq_postinstall(struct drm_device *dev) I915_WRITE(GEN11_DISPLAY_INT_CTL, GEN11_DISPLAY_IRQ_ENABLE); - I915_WRITE(GEN11_GFX_MSTR_IRQ, GEN11_MASTER_IRQ); + gen11_master_intr_enable(dev_priv->regs); POSTING_READ(GEN11_GFX_MSTR_IRQ); return 0; @@ -4336,8 +4075,6 @@ static void i8xx_irq_reset(struct drm_device *dev) i9xx_pipestat_irq_reset(dev_priv); - I915_WRITE16(HWSTAM, 0xffff); - GEN2_IRQ_RESET(); } @@ -4481,7 +4218,7 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg) I915_WRITE16(IIR, iir); if (iir & I915_USER_INTERRUPT) - notify_ring(dev_priv->engine[RCS]); + intel_engine_breadcrumbs_irq(dev_priv->engine[RCS]); if (iir & I915_MASTER_ERROR_INTERRUPT) i8xx_error_irq_handler(dev_priv, eir, eir_stuck); @@ -4505,8 +4242,6 @@ static void i915_irq_reset(struct drm_device *dev) i9xx_pipestat_irq_reset(dev_priv); - I915_WRITE(HWSTAM, 0xffffffff); - GEN3_IRQ_RESET(); } @@ -4591,7 +4326,7 @@ static irqreturn_t i915_irq_handler(int irq, void *arg) I915_WRITE(IIR, iir); if (iir & I915_USER_INTERRUPT) - notify_ring(dev_priv->engine[RCS]); + intel_engine_breadcrumbs_irq(dev_priv->engine[RCS]); if (iir & I915_MASTER_ERROR_INTERRUPT) i9xx_error_irq_handler(dev_priv, eir, eir_stuck); @@ -4616,8 +4351,6 @@ static void i965_irq_reset(struct drm_device *dev) i9xx_pipestat_irq_reset(dev_priv); - I915_WRITE(HWSTAM, 0xffffffff); - GEN3_IRQ_RESET(); } @@ -4738,10 +4471,10 @@ static irqreturn_t i965_irq_handler(int irq, void *arg) I915_WRITE(IIR, iir); if (iir & I915_USER_INTERRUPT) - notify_ring(dev_priv->engine[RCS]); + intel_engine_breadcrumbs_irq(dev_priv->engine[RCS]); if (iir & I915_BSD_USER_INTERRUPT) - notify_ring(dev_priv->engine[VCS]); + intel_engine_breadcrumbs_irq(dev_priv->engine[VCS]); if (iir & I915_MASTER_ERROR_INTERRUPT) i9xx_error_irq_handler(dev_priv, eir, eir_stuck); @@ -4804,23 +4537,17 @@ void intel_irq_init(struct drm_i915_private *dev_priv) if (INTEL_GEN(dev_priv) >= 8) rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC; - if (IS_GEN2(dev_priv)) { - /* Gen2 doesn't have a hardware frame counter */ - dev->max_vblank_count = 0; - } else if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) { - dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */ + if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv)) dev->driver->get_vblank_counter = g4x_get_vblank_counter; - } else { + else if (INTEL_GEN(dev_priv) >= 3) dev->driver->get_vblank_counter = i915_get_vblank_counter; - dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */ - } /* * Opt out of the vblank disable timer on everything except gen2. * Gen2 doesn't have a hardware frame counter and so depends on * vblank interrupts to produce sane vblank seuquence numbers. */ - if (!IS_GEN2(dev_priv)) + if (!IS_GEN(dev_priv, 2)) dev->vblank_disable_immediate = true; /* Most platforms treat the display irq block as an always-on @@ -4834,6 +4561,13 @@ void intel_irq_init(struct drm_i915_private *dev_priv) dev_priv->display_irqs_enabled = false; dev_priv->hotplug.hpd_storm_threshold = HPD_STORM_DEFAULT_THRESHOLD; + /* If we have MST support, we want to avoid doing short HPD IRQ storm + * detection, as short HPD storms will occur as a natural part of + * sideband messaging with MST. + * On older platforms however, IRQ storms can occur with both long and + * short pulses, as seen on some G4x systems. + */ + dev_priv->hotplug.hpd_short_storm_enabled = !HAS_DP_MST(dev_priv); dev->driver->get_vblank_timestamp = drm_calc_vbltimestamp_from_scanoutpos; dev->driver->get_scanout_position = i915_get_crtc_scanoutpos; @@ -4885,14 +4619,14 @@ void intel_irq_init(struct drm_i915_private *dev_priv) dev->driver->disable_vblank = ironlake_disable_vblank; dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup; } else { - if (IS_GEN2(dev_priv)) { + if (IS_GEN(dev_priv, 2)) { dev->driver->irq_preinstall = i8xx_irq_reset; dev->driver->irq_postinstall = i8xx_irq_postinstall; dev->driver->irq_handler = i8xx_irq_handler; dev->driver->irq_uninstall = i8xx_irq_reset; dev->driver->enable_vblank = i8xx_enable_vblank; dev->driver->disable_vblank = i8xx_disable_vblank; - } else if (IS_GEN3(dev_priv)) { + } else if (IS_GEN(dev_priv, 3)) { dev->driver->irq_preinstall = i915_irq_reset; dev->driver->irq_postinstall = i915_irq_postinstall; dev->driver->irq_uninstall = i915_irq_reset; diff --git a/drivers/gpu/drm/i915/i915_oa_bdw.c b/drivers/gpu/drm/i915/i915_oa_bdw.c index 4abd2e8b5083..4acdb94555b7 100644 --- a/drivers/gpu/drm/i915/i915_oa_bdw.c +++ b/drivers/gpu/drm/i915/i915_oa_bdw.c @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #include <linux/sysfs.h> diff --git a/drivers/gpu/drm/i915/i915_oa_bdw.h b/drivers/gpu/drm/i915/i915_oa_bdw.h index b812d16162ac..0e667f1a8aa1 100644 --- a/drivers/gpu/drm/i915/i915_oa_bdw.h +++ b/drivers/gpu/drm/i915/i915_oa_bdw.h @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #ifndef __I915_OA_BDW_H__ diff --git a/drivers/gpu/drm/i915/i915_oa_bxt.c b/drivers/gpu/drm/i915/i915_oa_bxt.c index cb6f304ec16a..a44195c39923 100644 --- a/drivers/gpu/drm/i915/i915_oa_bxt.c +++ b/drivers/gpu/drm/i915/i915_oa_bxt.c @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #include <linux/sysfs.h> diff --git a/drivers/gpu/drm/i915/i915_oa_bxt.h b/drivers/gpu/drm/i915/i915_oa_bxt.h index 690b963a2383..679e92cf4f1d 100644 --- a/drivers/gpu/drm/i915/i915_oa_bxt.h +++ b/drivers/gpu/drm/i915/i915_oa_bxt.h @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #ifndef __I915_OA_BXT_H__ diff --git a/drivers/gpu/drm/i915/i915_oa_cflgt2.c b/drivers/gpu/drm/i915/i915_oa_cflgt2.c index 8641ae30e343..7f60d51b8761 100644 --- a/drivers/gpu/drm/i915/i915_oa_cflgt2.c +++ b/drivers/gpu/drm/i915/i915_oa_cflgt2.c @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #include <linux/sysfs.h> diff --git a/drivers/gpu/drm/i915/i915_oa_cflgt2.h b/drivers/gpu/drm/i915/i915_oa_cflgt2.h index 1f3268ef2ea2..4d6025559bbe 100644 --- a/drivers/gpu/drm/i915/i915_oa_cflgt2.h +++ b/drivers/gpu/drm/i915/i915_oa_cflgt2.h @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #ifndef __I915_OA_CFLGT2_H__ diff --git a/drivers/gpu/drm/i915/i915_oa_cflgt3.c b/drivers/gpu/drm/i915/i915_oa_cflgt3.c index 792facdb6702..a92c38e3a0ce 100644 --- a/drivers/gpu/drm/i915/i915_oa_cflgt3.c +++ b/drivers/gpu/drm/i915/i915_oa_cflgt3.c @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #include <linux/sysfs.h> diff --git a/drivers/gpu/drm/i915/i915_oa_cflgt3.h b/drivers/gpu/drm/i915/i915_oa_cflgt3.h index c13b5aac01b9..0697f4077402 100644 --- a/drivers/gpu/drm/i915/i915_oa_cflgt3.h +++ b/drivers/gpu/drm/i915/i915_oa_cflgt3.h @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #ifndef __I915_OA_CFLGT3_H__ diff --git a/drivers/gpu/drm/i915/i915_oa_chv.c b/drivers/gpu/drm/i915/i915_oa_chv.c index 556febb2c3c8..71ec889a0114 100644 --- a/drivers/gpu/drm/i915/i915_oa_chv.c +++ b/drivers/gpu/drm/i915/i915_oa_chv.c @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #include <linux/sysfs.h> diff --git a/drivers/gpu/drm/i915/i915_oa_chv.h b/drivers/gpu/drm/i915/i915_oa_chv.h index b9622496979e..0986eae3135f 100644 --- a/drivers/gpu/drm/i915/i915_oa_chv.h +++ b/drivers/gpu/drm/i915/i915_oa_chv.h @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #ifndef __I915_OA_CHV_H__ diff --git a/drivers/gpu/drm/i915/i915_oa_cnl.c b/drivers/gpu/drm/i915/i915_oa_cnl.c index ba9140c87cc0..5c23d883d6c9 100644 --- a/drivers/gpu/drm/i915/i915_oa_cnl.c +++ b/drivers/gpu/drm/i915/i915_oa_cnl.c @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #include <linux/sysfs.h> diff --git a/drivers/gpu/drm/i915/i915_oa_cnl.h b/drivers/gpu/drm/i915/i915_oa_cnl.h index fb918b131105..e830a406aff2 100644 --- a/drivers/gpu/drm/i915/i915_oa_cnl.h +++ b/drivers/gpu/drm/i915/i915_oa_cnl.h @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #ifndef __I915_OA_CNL_H__ diff --git a/drivers/gpu/drm/i915/i915_oa_glk.c b/drivers/gpu/drm/i915/i915_oa_glk.c index 971db587957c..4bdda66df7d2 100644 --- a/drivers/gpu/drm/i915/i915_oa_glk.c +++ b/drivers/gpu/drm/i915/i915_oa_glk.c @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #include <linux/sysfs.h> diff --git a/drivers/gpu/drm/i915/i915_oa_glk.h b/drivers/gpu/drm/i915/i915_oa_glk.h index 63bd113f4bc9..06dedf991edb 100644 --- a/drivers/gpu/drm/i915/i915_oa_glk.h +++ b/drivers/gpu/drm/i915/i915_oa_glk.h @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #ifndef __I915_OA_GLK_H__ diff --git a/drivers/gpu/drm/i915/i915_oa_hsw.c b/drivers/gpu/drm/i915/i915_oa_hsw.c index 434a9b96d7ab..cc6526fdd2bd 100644 --- a/drivers/gpu/drm/i915/i915_oa_hsw.c +++ b/drivers/gpu/drm/i915/i915_oa_hsw.c @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #include <linux/sysfs.h> diff --git a/drivers/gpu/drm/i915/i915_oa_hsw.h b/drivers/gpu/drm/i915/i915_oa_hsw.h index 74d03439c157..3d0c870cd0bd 100644 --- a/drivers/gpu/drm/i915/i915_oa_hsw.h +++ b/drivers/gpu/drm/i915/i915_oa_hsw.h @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #ifndef __I915_OA_HSW_H__ diff --git a/drivers/gpu/drm/i915/i915_oa_icl.c b/drivers/gpu/drm/i915/i915_oa_icl.c index a5667926e3de..baa51427a543 100644 --- a/drivers/gpu/drm/i915/i915_oa_icl.c +++ b/drivers/gpu/drm/i915/i915_oa_icl.c @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #include <linux/sysfs.h> diff --git a/drivers/gpu/drm/i915/i915_oa_icl.h b/drivers/gpu/drm/i915/i915_oa_icl.h index ae1c24aafe4f..24eaa97d61ba 100644 --- a/drivers/gpu/drm/i915/i915_oa_icl.h +++ b/drivers/gpu/drm/i915/i915_oa_icl.h @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #ifndef __I915_OA_ICL_H__ diff --git a/drivers/gpu/drm/i915/i915_oa_kblgt2.c b/drivers/gpu/drm/i915/i915_oa_kblgt2.c index 2fa98a40bbc8..168e49ab0d4d 100644 --- a/drivers/gpu/drm/i915/i915_oa_kblgt2.c +++ b/drivers/gpu/drm/i915/i915_oa_kblgt2.c @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #include <linux/sysfs.h> diff --git a/drivers/gpu/drm/i915/i915_oa_kblgt2.h b/drivers/gpu/drm/i915/i915_oa_kblgt2.h index 25b803546dc1..a55398a904de 100644 --- a/drivers/gpu/drm/i915/i915_oa_kblgt2.h +++ b/drivers/gpu/drm/i915/i915_oa_kblgt2.h @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #ifndef __I915_OA_KBLGT2_H__ diff --git a/drivers/gpu/drm/i915/i915_oa_kblgt3.c b/drivers/gpu/drm/i915/i915_oa_kblgt3.c index f3cb6679a1bc..6ffa553c388e 100644 --- a/drivers/gpu/drm/i915/i915_oa_kblgt3.c +++ b/drivers/gpu/drm/i915/i915_oa_kblgt3.c @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #include <linux/sysfs.h> diff --git a/drivers/gpu/drm/i915/i915_oa_kblgt3.h b/drivers/gpu/drm/i915/i915_oa_kblgt3.h index d5b5b5c1923e..3ddd3483b7cc 100644 --- a/drivers/gpu/drm/i915/i915_oa_kblgt3.h +++ b/drivers/gpu/drm/i915/i915_oa_kblgt3.h @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #ifndef __I915_OA_KBLGT3_H__ diff --git a/drivers/gpu/drm/i915/i915_oa_sklgt2.c b/drivers/gpu/drm/i915/i915_oa_sklgt2.c index bf8b8cd8a50d..7ce6ee851d43 100644 --- a/drivers/gpu/drm/i915/i915_oa_sklgt2.c +++ b/drivers/gpu/drm/i915/i915_oa_sklgt2.c @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #include <linux/sysfs.h> diff --git a/drivers/gpu/drm/i915/i915_oa_sklgt2.h b/drivers/gpu/drm/i915/i915_oa_sklgt2.h index fe1aa2c03958..be6256037239 100644 --- a/drivers/gpu/drm/i915/i915_oa_sklgt2.h +++ b/drivers/gpu/drm/i915/i915_oa_sklgt2.h @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #ifndef __I915_OA_SKLGT2_H__ diff --git a/drivers/gpu/drm/i915/i915_oa_sklgt3.c b/drivers/gpu/drm/i915/i915_oa_sklgt3.c index ae534c7c8135..086ca2631e1c 100644 --- a/drivers/gpu/drm/i915/i915_oa_sklgt3.c +++ b/drivers/gpu/drm/i915/i915_oa_sklgt3.c @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #include <linux/sysfs.h> diff --git a/drivers/gpu/drm/i915/i915_oa_sklgt3.h b/drivers/gpu/drm/i915/i915_oa_sklgt3.h index 06746b2616c8..650beb068e56 100644 --- a/drivers/gpu/drm/i915/i915_oa_sklgt3.h +++ b/drivers/gpu/drm/i915/i915_oa_sklgt3.h @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #ifndef __I915_OA_SKLGT3_H__ diff --git a/drivers/gpu/drm/i915/i915_oa_sklgt4.c b/drivers/gpu/drm/i915/i915_oa_sklgt4.c index 817fba2d82df..b291a6eb8a87 100644 --- a/drivers/gpu/drm/i915/i915_oa_sklgt4.c +++ b/drivers/gpu/drm/i915/i915_oa_sklgt4.c @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #include <linux/sysfs.h> diff --git a/drivers/gpu/drm/i915/i915_oa_sklgt4.h b/drivers/gpu/drm/i915/i915_oa_sklgt4.h index 944fd525c8b1..8dcf849d131e 100644 --- a/drivers/gpu/drm/i915/i915_oa_sklgt4.h +++ b/drivers/gpu/drm/i915/i915_oa_sklgt4.h @@ -1,29 +1,10 @@ /* - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - * - * - * Copyright (c) 2015 Intel Corporation + * SPDX-License-Identifier: MIT * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * Copyright © 2018 Intel Corporation * + * Autogenerated file by GPU Top : https://github.com/rib/gputop + * DO NOT EDIT manually! */ #ifndef __I915_OA_SKLGT4_H__ diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c index 295e981e4a39..b5be0abbba35 100644 --- a/drivers/gpu/drm/i915/i915_params.c +++ b/drivers/gpu/drm/i915/i915_params.c @@ -77,15 +77,11 @@ i915_param_named(error_capture, bool, 0600, "triaging and debugging hangs."); #endif -i915_param_named_unsafe(enable_hangcheck, bool, 0644, +i915_param_named_unsafe(enable_hangcheck, bool, 0600, "Periodically check GPU activity for detecting hangs. " "WARNING: Disabling this can cause system wide hangs. " "(default: true)"); -i915_param_named_unsafe(enable_ppgtt, int, 0400, - "Override PPGTT usage. " - "(-1=auto [default], 0=disabled, 1=aliasing, 2=full, 3=full with extended address space)"); - i915_param_named_unsafe(enable_psr, int, 0600, "Enable PSR " "(0=disabled, 1=enabled) " @@ -101,8 +97,10 @@ i915_param_named_unsafe(disable_power_well, int, 0400, i915_param_named_unsafe(enable_ips, int, 0600, "Enable IPS (default: true)"); -i915_param_named(fastboot, bool, 0600, - "Try to skip unnecessary mode sets at boot time (default: false)"); +i915_param_named(fastboot, int, 0600, + "Try to skip unnecessary mode sets at boot time " + "(0=disabled, 1=enabled) " + "Default: -1 (use per-chip default)"); i915_param_named_unsafe(prefault_disable, bool, 0600, "Disable page prefaulting for pread/pwrite/reloc (default:false). " @@ -171,8 +169,10 @@ i915_param_named_unsafe(inject_load_failure, uint, 0400, i915_param_named(enable_dpcd_backlight, bool, 0600, "Enable support for DPCD backlight control (default:false)"); +#if IS_ENABLED(CONFIG_DRM_I915_GVT) i915_param_named(enable_gvt, bool, 0400, "Enable support for Intel GVT-g graphics virtualization host support(default:false)"); +#endif static __always_inline void _print_param(struct drm_printer *p, const char *name, @@ -188,7 +188,8 @@ static __always_inline void _print_param(struct drm_printer *p, else if (!__builtin_strcmp(type, "char *")) drm_printf(p, "i915.%s=%s\n", name, *(const char **)x); else - BUILD_BUG(); + WARN_ONCE(1, "no printer defined for param type %s (i915.%s)\n", + type, name); } /** @@ -204,3 +205,33 @@ void i915_params_dump(const struct i915_params *params, struct drm_printer *p) I915_PARAMS_FOR_EACH(PRINT); #undef PRINT } + +static __always_inline void dup_param(const char *type, void *x) +{ + if (!__builtin_strcmp(type, "char *")) + *(void **)x = kstrdup(*(void **)x, GFP_ATOMIC); +} + +void i915_params_copy(struct i915_params *dest, const struct i915_params *src) +{ + *dest = *src; +#define DUP(T, x, ...) dup_param(#T, &dest->x); + I915_PARAMS_FOR_EACH(DUP); +#undef DUP +} + +static __always_inline void free_param(const char *type, void *x) +{ + if (!__builtin_strcmp(type, "char *")) { + kfree(*(void **)x); + *(void **)x = NULL; + } +} + +/* free the allocated members, *not* the passed in params itself */ +void i915_params_free(struct i915_params *params) +{ +#define FREE(T, x, ...) free_param(#T, ¶ms->x); + I915_PARAMS_FOR_EACH(FREE); +#undef FREE +} diff --git a/drivers/gpu/drm/i915/i915_params.h b/drivers/gpu/drm/i915/i915_params.h index 6c4d4a21474b..3f14e9881a0d 100644 --- a/drivers/gpu/drm/i915/i915_params.h +++ b/drivers/gpu/drm/i915/i915_params.h @@ -33,6 +33,15 @@ struct drm_printer; #define ENABLE_GUC_SUBMISSION BIT(0) #define ENABLE_GUC_LOAD_HUC BIT(1) +/* + * Invoke param, a function-like macro, for each i915 param, with arguments: + * + * param(type, name, value) + * + * type: parameter type, one of {bool, int, unsigned int, char *} + * name: name of the parameter + * value: initial/default value of the parameter + */ #define I915_PARAMS_FOR_EACH(param) \ param(char *, vbt_firmware, NULL) \ param(int, modeset, -1) \ @@ -41,7 +50,6 @@ struct drm_printer; param(int, vbt_sdvo_panel_type, -1) \ param(int, enable_dc, -1) \ param(int, enable_fbc, -1) \ - param(int, enable_ppgtt, -1) \ param(int, enable_psr, -1) \ param(int, disable_power_well, -1) \ param(int, enable_ips, 1) \ @@ -55,10 +63,10 @@ struct drm_printer; param(int, edp_vswing, 0) \ param(int, reset, 2) \ param(unsigned int, inject_load_failure, 0) \ + param(int, fastboot, -1) \ /* leave bools at the end to not create holes */ \ param(bool, alpha_support, IS_ENABLED(CONFIG_DRM_I915_ALPHA_SUPPORT)) \ param(bool, enable_hangcheck, true) \ - param(bool, fastboot, false) \ param(bool, prefault_disable, false) \ param(bool, load_detect_test, false) \ param(bool, force_reset_modeset_test, false) \ @@ -79,6 +87,8 @@ struct i915_params { extern struct i915_params i915_modparams __read_mostly; void i915_params_dump(const struct i915_params *params, struct drm_printer *p); +void i915_params_copy(struct i915_params *dest, const struct i915_params *src); +void i915_params_free(struct i915_params *params); #endif diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c index d6f7b9fe1d26..66f82f3f050f 100644 --- a/drivers/gpu/drm/i915/i915_pci.c +++ b/drivers/gpu/drm/i915/i915_pci.c @@ -26,6 +26,9 @@ #include <linux/vgaarb.h> #include <linux/vga_switcheroo.h> +#include <drm/drm_drv.h> + +#include "i915_active.h" #include "i915_drv.h" #include "i915_selftest.h" @@ -33,19 +36,30 @@ #define GEN(x) .gen = (x), .gen_mask = BIT((x) - 1) #define GEN_DEFAULT_PIPEOFFSETS \ - .pipe_offsets = { PIPE_A_OFFSET, PIPE_B_OFFSET, \ - PIPE_C_OFFSET, PIPE_EDP_OFFSET }, \ - .trans_offsets = { TRANSCODER_A_OFFSET, TRANSCODER_B_OFFSET, \ - TRANSCODER_C_OFFSET, TRANSCODER_EDP_OFFSET }, \ - .palette_offsets = { PALETTE_A_OFFSET, PALETTE_B_OFFSET } + .pipe_offsets = { \ + [TRANSCODER_A] = PIPE_A_OFFSET, \ + [TRANSCODER_B] = PIPE_B_OFFSET, \ + [TRANSCODER_C] = PIPE_C_OFFSET, \ + [TRANSCODER_EDP] = PIPE_EDP_OFFSET, \ + }, \ + .trans_offsets = { \ + [TRANSCODER_A] = TRANSCODER_A_OFFSET, \ + [TRANSCODER_B] = TRANSCODER_B_OFFSET, \ + [TRANSCODER_C] = TRANSCODER_C_OFFSET, \ + [TRANSCODER_EDP] = TRANSCODER_EDP_OFFSET, \ + } #define GEN_CHV_PIPEOFFSETS \ - .pipe_offsets = { PIPE_A_OFFSET, PIPE_B_OFFSET, \ - CHV_PIPE_C_OFFSET }, \ - .trans_offsets = { TRANSCODER_A_OFFSET, TRANSCODER_B_OFFSET, \ - CHV_TRANSCODER_C_OFFSET, }, \ - .palette_offsets = { PALETTE_A_OFFSET, PALETTE_B_OFFSET, \ - CHV_PALETTE_C_OFFSET } + .pipe_offsets = { \ + [TRANSCODER_A] = PIPE_A_OFFSET, \ + [TRANSCODER_B] = PIPE_B_OFFSET, \ + [TRANSCODER_C] = CHV_PIPE_C_OFFSET, \ + }, \ + .trans_offsets = { \ + [TRANSCODER_A] = TRANSCODER_A_OFFSET, \ + [TRANSCODER_B] = TRANSCODER_B_OFFSET, \ + [TRANSCODER_C] = CHV_TRANSCODER_C_OFFSET, \ + } #define CURSOR_OFFSETS \ .cursor_offsets = { CURSOR_A_OFFSET, CURSOR_B_OFFSET, CHV_CURSOR_C_OFFSET } @@ -56,9 +70,15 @@ #define BDW_COLORS \ .color = { .degamma_lut_size = 512, .gamma_lut_size = 512 } #define CHV_COLORS \ - .color = { .degamma_lut_size = 65, .gamma_lut_size = 257 } + .color = { .degamma_lut_size = 65, .gamma_lut_size = 257, \ + .degamma_lut_tests = DRM_COLOR_LUT_NON_DECREASING, \ + .gamma_lut_tests = DRM_COLOR_LUT_NON_DECREASING, \ + } #define GLK_COLORS \ - .color = { .degamma_lut_size = 0, .gamma_lut_size = 1024 } + .color = { .degamma_lut_size = 0, .gamma_lut_size = 1024, \ + .degamma_lut_tests = DRM_COLOR_LUT_NON_DECREASING | \ + DRM_COLOR_LUT_EQUAL_CHANNELS, \ + } /* Keep in gen based order, and chronological order within a gen */ @@ -68,8 +88,10 @@ #define GEN2_FEATURES \ GEN(2), \ .num_pipes = 1, \ - .has_overlay = 1, .overlay_needs_physical = 1, \ - .has_gmch_display = 1, \ + .display.has_overlay = 1, \ + .display.overlay_needs_physical = 1, \ + .display.has_gmch = 1, \ + .gpu_reset_clobbers_display = true, \ .hws_needs_physical = 1, \ .unfenced_needs_alignment = 1, \ .ring_mask = RENDER_RING, \ @@ -82,7 +104,8 @@ static const struct intel_device_info intel_i830_info = { GEN2_FEATURES, PLATFORM(INTEL_I830), - .is_mobile = 1, .cursor_needs_physical = 1, + .is_mobile = 1, + .display.cursor_needs_physical = 1, .num_pipes = 2, /* legal, last one wins */ }; @@ -96,8 +119,8 @@ static const struct intel_device_info intel_i85x_info = { PLATFORM(INTEL_I85X), .is_mobile = 1, .num_pipes = 2, /* legal, last one wins */ - .cursor_needs_physical = 1, - .has_fbc = 1, + .display.cursor_needs_physical = 1, + .display.has_fbc = 1, }; static const struct intel_device_info intel_i865g_info = { @@ -108,7 +131,8 @@ static const struct intel_device_info intel_i865g_info = { #define GEN3_FEATURES \ GEN(3), \ .num_pipes = 2, \ - .has_gmch_display = 1, \ + .display.has_gmch = 1, \ + .gpu_reset_clobbers_display = true, \ .ring_mask = RENDER_RING, \ .has_snoop = true, \ .has_coherent_ggtt = true, \ @@ -120,8 +144,9 @@ static const struct intel_device_info intel_i915g_info = { GEN3_FEATURES, PLATFORM(INTEL_I915G), .has_coherent_ggtt = false, - .cursor_needs_physical = 1, - .has_overlay = 1, .overlay_needs_physical = 1, + .display.cursor_needs_physical = 1, + .display.has_overlay = 1, + .display.overlay_needs_physical = 1, .hws_needs_physical = 1, .unfenced_needs_alignment = 1, }; @@ -130,10 +155,11 @@ static const struct intel_device_info intel_i915gm_info = { GEN3_FEATURES, PLATFORM(INTEL_I915GM), .is_mobile = 1, - .cursor_needs_physical = 1, - .has_overlay = 1, .overlay_needs_physical = 1, - .supports_tv = 1, - .has_fbc = 1, + .display.cursor_needs_physical = 1, + .display.has_overlay = 1, + .display.overlay_needs_physical = 1, + .display.supports_tv = 1, + .display.has_fbc = 1, .hws_needs_physical = 1, .unfenced_needs_alignment = 1, }; @@ -141,8 +167,10 @@ static const struct intel_device_info intel_i915gm_info = { static const struct intel_device_info intel_i945g_info = { GEN3_FEATURES, PLATFORM(INTEL_I945G), - .has_hotplug = 1, .cursor_needs_physical = 1, - .has_overlay = 1, .overlay_needs_physical = 1, + .display.has_hotplug = 1, + .display.cursor_needs_physical = 1, + .display.has_overlay = 1, + .display.overlay_needs_physical = 1, .hws_needs_physical = 1, .unfenced_needs_alignment = 1, }; @@ -151,10 +179,12 @@ static const struct intel_device_info intel_i945gm_info = { GEN3_FEATURES, PLATFORM(INTEL_I945GM), .is_mobile = 1, - .has_hotplug = 1, .cursor_needs_physical = 1, - .has_overlay = 1, .overlay_needs_physical = 1, - .supports_tv = 1, - .has_fbc = 1, + .display.has_hotplug = 1, + .display.cursor_needs_physical = 1, + .display.has_overlay = 1, + .display.overlay_needs_physical = 1, + .display.supports_tv = 1, + .display.has_fbc = 1, .hws_needs_physical = 1, .unfenced_needs_alignment = 1, }; @@ -162,23 +192,24 @@ static const struct intel_device_info intel_i945gm_info = { static const struct intel_device_info intel_g33_info = { GEN3_FEATURES, PLATFORM(INTEL_G33), - .has_hotplug = 1, - .has_overlay = 1, + .display.has_hotplug = 1, + .display.has_overlay = 1, }; static const struct intel_device_info intel_pineview_info = { GEN3_FEATURES, PLATFORM(INTEL_PINEVIEW), .is_mobile = 1, - .has_hotplug = 1, - .has_overlay = 1, + .display.has_hotplug = 1, + .display.has_overlay = 1, }; #define GEN4_FEATURES \ GEN(4), \ .num_pipes = 2, \ - .has_hotplug = 1, \ - .has_gmch_display = 1, \ + .display.has_hotplug = 1, \ + .display.has_gmch = 1, \ + .gpu_reset_clobbers_display = true, \ .ring_mask = RENDER_RING, \ .has_snoop = true, \ .has_coherent_ggtt = true, \ @@ -189,7 +220,7 @@ static const struct intel_device_info intel_pineview_info = { static const struct intel_device_info intel_i965g_info = { GEN4_FEATURES, PLATFORM(INTEL_I965G), - .has_overlay = 1, + .display.has_overlay = 1, .hws_needs_physical = 1, .has_snoop = false, }; @@ -197,9 +228,10 @@ static const struct intel_device_info intel_i965g_info = { static const struct intel_device_info intel_i965gm_info = { GEN4_FEATURES, PLATFORM(INTEL_I965GM), - .is_mobile = 1, .has_fbc = 1, - .has_overlay = 1, - .supports_tv = 1, + .is_mobile = 1, + .display.has_fbc = 1, + .display.has_overlay = 1, + .display.supports_tv = 1, .hws_needs_physical = 1, .has_snoop = false, }; @@ -208,20 +240,23 @@ static const struct intel_device_info intel_g45_info = { GEN4_FEATURES, PLATFORM(INTEL_G45), .ring_mask = RENDER_RING | BSD_RING, + .gpu_reset_clobbers_display = false, }; static const struct intel_device_info intel_gm45_info = { GEN4_FEATURES, PLATFORM(INTEL_GM45), - .is_mobile = 1, .has_fbc = 1, - .supports_tv = 1, + .is_mobile = 1, + .display.has_fbc = 1, + .display.supports_tv = 1, .ring_mask = RENDER_RING | BSD_RING, + .gpu_reset_clobbers_display = false, }; #define GEN5_FEATURES \ GEN(5), \ .num_pipes = 2, \ - .has_hotplug = 1, \ + .display.has_hotplug = 1, \ .ring_mask = RENDER_RING | BSD_RING, \ .has_snoop = true, \ .has_coherent_ggtt = true, \ @@ -239,20 +274,21 @@ static const struct intel_device_info intel_ironlake_d_info = { static const struct intel_device_info intel_ironlake_m_info = { GEN5_FEATURES, PLATFORM(INTEL_IRONLAKE), - .is_mobile = 1, .has_fbc = 1, + .is_mobile = 1, + .display.has_fbc = 1, }; #define GEN6_FEATURES \ GEN(6), \ .num_pipes = 2, \ - .has_hotplug = 1, \ - .has_fbc = 1, \ + .display.has_hotplug = 1, \ + .display.has_fbc = 1, \ .ring_mask = RENDER_RING | BSD_RING | BLT_RING, \ .has_coherent_ggtt = true, \ .has_llc = 1, \ .has_rc6 = 1, \ .has_rc6p = 1, \ - .has_aliasing_ppgtt = 1, \ + .ppgtt = INTEL_PPGTT_ALIASING, \ GEN_DEFAULT_PIPEOFFSETS, \ GEN_DEFAULT_PAGE_SIZES, \ CURSOR_OFFSETS @@ -290,15 +326,14 @@ static const struct intel_device_info intel_sandybridge_m_gt2_info = { #define GEN7_FEATURES \ GEN(7), \ .num_pipes = 3, \ - .has_hotplug = 1, \ - .has_fbc = 1, \ + .display.has_hotplug = 1, \ + .display.has_fbc = 1, \ .ring_mask = RENDER_RING | BSD_RING | BLT_RING, \ .has_coherent_ggtt = true, \ .has_llc = 1, \ .has_rc6 = 1, \ .has_rc6p = 1, \ - .has_aliasing_ppgtt = 1, \ - .has_full_ppgtt = 1, \ + .ppgtt = INTEL_PPGTT_FULL, \ GEN_DEFAULT_PIPEOFFSETS, \ GEN_DEFAULT_PAGE_SIZES, \ IVB_CURSOR_OFFSETS @@ -349,10 +384,9 @@ static const struct intel_device_info intel_valleyview_info = { .num_pipes = 2, .has_runtime_pm = 1, .has_rc6 = 1, - .has_gmch_display = 1, - .has_hotplug = 1, - .has_aliasing_ppgtt = 1, - .has_full_ppgtt = 1, + .display.has_gmch = 1, + .display.has_hotplug = 1, + .ppgtt = INTEL_PPGTT_FULL, .has_snoop = true, .has_coherent_ggtt = false, .ring_mask = RENDER_RING | BSD_RING | BLT_RING, @@ -365,10 +399,10 @@ static const struct intel_device_info intel_valleyview_info = { #define G75_FEATURES \ GEN7_FEATURES, \ .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, \ - .has_ddi = 1, \ + .display.has_ddi = 1, \ .has_fpga_dbg = 1, \ - .has_psr = 1, \ - .has_dp_mst = 1, \ + .display.has_psr = 1, \ + .display.has_dp_mst = 1, \ .has_rc6p = 0 /* RC6p removed-by HSW */, \ .has_runtime_pm = 1 @@ -399,7 +433,7 @@ static const struct intel_device_info intel_haswell_gt3_info = { .page_sizes = I915_GTT_PAGE_SIZE_4K | \ I915_GTT_PAGE_SIZE_2M, \ .has_logical_ring_contexts = 1, \ - .has_full_48bit_ppgtt = 1, \ + .ppgtt = INTEL_PPGTT_FULL_4LVL, \ .has_64bit_reloc = 1, \ .has_reset_engine = 1 @@ -435,16 +469,15 @@ static const struct intel_device_info intel_cherryview_info = { PLATFORM(INTEL_CHERRYVIEW), GEN(8), .num_pipes = 3, - .has_hotplug = 1, + .display.has_hotplug = 1, .is_lp = 1, .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, .has_64bit_reloc = 1, .has_runtime_pm = 1, .has_rc6 = 1, .has_logical_ring_contexts = 1, - .has_gmch_display = 1, - .has_aliasing_ppgtt = 1, - .has_full_ppgtt = 1, + .display.has_gmch = 1, + .ppgtt = INTEL_PPGTT_FULL, .has_reset_engine = 1, .has_snoop = true, .has_coherent_ggtt = false, @@ -465,13 +498,15 @@ static const struct intel_device_info intel_cherryview_info = { GEN(9), \ GEN9_DEFAULT_PAGE_SIZES, \ .has_logical_ring_preemption = 1, \ - .has_csr = 1, \ + .display.has_csr = 1, \ .has_guc = 1, \ - .has_ipc = 1, \ + .display.has_ipc = 1, \ .ddb_size = 896 #define SKL_PLATFORM \ GEN9_FEATURES, \ + /* Display WA #0477 WaDisableIPC: skl */ \ + .display.has_ipc = 0, \ PLATFORM(INTEL_SKYLAKE) static const struct intel_device_info intel_skylake_gt1_info = { @@ -502,29 +537,26 @@ static const struct intel_device_info intel_skylake_gt4_info = { #define GEN9_LP_FEATURES \ GEN(9), \ .is_lp = 1, \ - .has_hotplug = 1, \ + .display.has_hotplug = 1, \ .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, \ .num_pipes = 3, \ .has_64bit_reloc = 1, \ - .has_ddi = 1, \ + .display.has_ddi = 1, \ .has_fpga_dbg = 1, \ - .has_fbc = 1, \ - .has_psr = 1, \ + .display.has_fbc = 1, \ + .display.has_psr = 1, \ .has_runtime_pm = 1, \ - .has_pooled_eu = 0, \ - .has_csr = 1, \ + .display.has_csr = 1, \ .has_rc6 = 1, \ - .has_dp_mst = 1, \ + .display.has_dp_mst = 1, \ .has_logical_ring_contexts = 1, \ .has_logical_ring_preemption = 1, \ .has_guc = 1, \ - .has_aliasing_ppgtt = 1, \ - .has_full_ppgtt = 1, \ - .has_full_48bit_ppgtt = 1, \ + .ppgtt = INTEL_PPGTT_FULL_4LVL, \ .has_reset_engine = 1, \ .has_snoop = true, \ .has_coherent_ggtt = false, \ - .has_ipc = 1, \ + .display.has_ipc = 1, \ GEN9_DEFAULT_PAGE_SIZES, \ GEN_DEFAULT_PIPEOFFSETS, \ IVB_CURSOR_OFFSETS, \ @@ -598,6 +630,22 @@ static const struct intel_device_info intel_cannonlake_info = { #define GEN11_FEATURES \ GEN10_FEATURES, \ + .pipe_offsets = { \ + [TRANSCODER_A] = PIPE_A_OFFSET, \ + [TRANSCODER_B] = PIPE_B_OFFSET, \ + [TRANSCODER_C] = PIPE_C_OFFSET, \ + [TRANSCODER_EDP] = PIPE_EDP_OFFSET, \ + [TRANSCODER_DSI_0] = PIPE_DSI0_OFFSET, \ + [TRANSCODER_DSI_1] = PIPE_DSI1_OFFSET, \ + }, \ + .trans_offsets = { \ + [TRANSCODER_A] = TRANSCODER_A_OFFSET, \ + [TRANSCODER_B] = TRANSCODER_B_OFFSET, \ + [TRANSCODER_C] = TRANSCODER_C_OFFSET, \ + [TRANSCODER_EDP] = TRANSCODER_EDP_OFFSET, \ + [TRANSCODER_DSI_0] = TRANSCODER_DSI0_OFFSET, \ + [TRANSCODER_DSI_1] = TRANSCODER_DSI1_OFFSET, \ + }, \ GEN(11), \ .ddb_size = 2048, \ .has_logical_ring_elsq = 1 @@ -663,14 +711,16 @@ static const struct pci_device_id pciidlist[] = { INTEL_KBL_GT2_IDS(&intel_kabylake_gt2_info), INTEL_KBL_GT3_IDS(&intel_kabylake_gt3_info), INTEL_KBL_GT4_IDS(&intel_kabylake_gt3_info), - INTEL_AML_GT2_IDS(&intel_kabylake_gt2_info), + INTEL_AML_KBL_GT2_IDS(&intel_kabylake_gt2_info), INTEL_CFL_S_GT1_IDS(&intel_coffeelake_gt1_info), INTEL_CFL_S_GT2_IDS(&intel_coffeelake_gt2_info), + INTEL_CFL_H_GT1_IDS(&intel_coffeelake_gt1_info), INTEL_CFL_H_GT2_IDS(&intel_coffeelake_gt2_info), INTEL_CFL_U_GT2_IDS(&intel_coffeelake_gt2_info), INTEL_CFL_U_GT3_IDS(&intel_coffeelake_gt3_info), INTEL_WHL_U_GT1_IDS(&intel_coffeelake_gt1_info), INTEL_WHL_U_GT2_IDS(&intel_coffeelake_gt2_info), + INTEL_AML_CFL_GT2_IDS(&intel_coffeelake_gt2_info), INTEL_WHL_U_GT3_IDS(&intel_coffeelake_gt3_info), INTEL_CNL_IDS(&intel_cannonlake_info), INTEL_ICL_11_IDS(&intel_icelake_11_info), @@ -751,6 +801,8 @@ static int __init i915_init(void) bool use_kms = true; int err; + i915_global_active_init(); + err = i915_mock_selftests(); if (err) return err > 0 ? 0 : err; @@ -782,6 +834,7 @@ static void __exit i915_exit(void) return; pci_unregister_driver(&i915_pci_driver); + i915_global_active_exit(); } module_init(i915_init); diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c index 664b96bb65a3..9ebf99f3d8d3 100644 --- a/drivers/gpu/drm/i915/i915_perf.c +++ b/drivers/gpu/drm/i915/i915_perf.c @@ -890,8 +890,8 @@ static int gen8_oa_read(struct i915_perf_stream *stream, DRM_DEBUG("OA buffer overflow (exponent = %d): force restart\n", dev_priv->perf.oa.period_exponent); - dev_priv->perf.oa.ops.oa_disable(dev_priv); - dev_priv->perf.oa.ops.oa_enable(dev_priv); + dev_priv->perf.oa.ops.oa_disable(stream); + dev_priv->perf.oa.ops.oa_enable(stream); /* * Note: .oa_enable() is expected to re-init the oabuffer and @@ -1114,8 +1114,8 @@ static int gen7_oa_read(struct i915_perf_stream *stream, DRM_DEBUG("OA buffer overflow (exponent = %d): force restart\n", dev_priv->perf.oa.period_exponent); - dev_priv->perf.oa.ops.oa_disable(dev_priv); - dev_priv->perf.oa.ops.oa_enable(dev_priv); + dev_priv->perf.oa.ops.oa_disable(stream); + dev_priv->perf.oa.ops.oa_enable(stream); oastatus1 = I915_READ(GEN7_OASTATUS1); } @@ -1365,7 +1365,7 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream) free_oa_buffer(dev_priv); intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, stream->wakeref); if (stream->ctx) oa_put_render_ctx_id(stream); @@ -1528,8 +1528,6 @@ static int alloc_oa_buffer(struct drm_i915_private *dev_priv) goto err_unpin; } - dev_priv->perf.oa.ops.init_oa_buffer(dev_priv); - DRM_DEBUG_DRIVER("OA Buffer initialized, gtt offset = 0x%x, vaddr = %p\n", i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma), dev_priv->perf.oa.oa_buffer.vaddr); @@ -1563,9 +1561,11 @@ static void config_oa_regs(struct drm_i915_private *dev_priv, } } -static int hsw_enable_metric_set(struct drm_i915_private *dev_priv, - const struct i915_oa_config *oa_config) +static int hsw_enable_metric_set(struct i915_perf_stream *stream) { + struct drm_i915_private *dev_priv = stream->dev_priv; + const struct i915_oa_config *oa_config = stream->oa_config; + /* PRM: * * OA unit is using “crclk” for its functionality. When trunk @@ -1677,6 +1677,11 @@ static void gen8_update_reg_state_unlocked(struct i915_gem_context *ctx, CTX_REG(reg_state, state_offset, flex_regs[i], value); } + + CTX_REG(reg_state, CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE, + gen8_make_rpcs(dev_priv, + &to_intel_context(ctx, + dev_priv->engine[RCS])->sseu)); } /* @@ -1767,9 +1772,10 @@ static int gen8_configure_all_contexts(struct drm_i915_private *dev_priv, return 0; } -static int gen8_enable_metric_set(struct drm_i915_private *dev_priv, - const struct i915_oa_config *oa_config) +static int gen8_enable_metric_set(struct i915_perf_stream *stream) { + struct drm_i915_private *dev_priv = stream->dev_priv; + const struct i915_oa_config *oa_config = stream->oa_config; int ret; /* @@ -1795,7 +1801,7 @@ static int gen8_enable_metric_set(struct drm_i915_private *dev_priv, * be read back from automatically triggered reports, as part of the * RPT_ID field. */ - if (IS_GEN(dev_priv, 9, 11)) { + if (IS_GEN_RANGE(dev_priv, 9, 11)) { I915_WRITE(GEN8_OA_DEBUG, _MASKED_BIT_ENABLE(GEN9_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS | GEN9_OA_DEBUG_INCLUDE_CLK_RATIO)); @@ -1837,10 +1843,10 @@ static void gen10_disable_metric_set(struct drm_i915_private *dev_priv) I915_READ(RPM_CONFIG1) & ~GEN10_GT_NOA_ENABLE); } -static void gen7_oa_enable(struct drm_i915_private *dev_priv) +static void gen7_oa_enable(struct i915_perf_stream *stream) { - struct i915_gem_context *ctx = - dev_priv->perf.oa.exclusive_stream->ctx; + struct drm_i915_private *dev_priv = stream->dev_priv; + struct i915_gem_context *ctx = stream->ctx; u32 ctx_id = dev_priv->perf.oa.specific_ctx_id; bool periodic = dev_priv->perf.oa.periodic; u32 period_exponent = dev_priv->perf.oa.period_exponent; @@ -1867,8 +1873,9 @@ static void gen7_oa_enable(struct drm_i915_private *dev_priv) GEN7_OACONTROL_ENABLE); } -static void gen8_oa_enable(struct drm_i915_private *dev_priv) +static void gen8_oa_enable(struct i915_perf_stream *stream) { + struct drm_i915_private *dev_priv = stream->dev_priv; u32 report_format = dev_priv->perf.oa.oa_buffer.format; /* @@ -1905,7 +1912,7 @@ static void i915_oa_stream_enable(struct i915_perf_stream *stream) { struct drm_i915_private *dev_priv = stream->dev_priv; - dev_priv->perf.oa.ops.oa_enable(dev_priv); + dev_priv->perf.oa.ops.oa_enable(stream); if (dev_priv->perf.oa.periodic) hrtimer_start(&dev_priv->perf.oa.poll_check_timer, @@ -1913,8 +1920,10 @@ static void i915_oa_stream_enable(struct i915_perf_stream *stream) HRTIMER_MODE_REL_PINNED); } -static void gen7_oa_disable(struct drm_i915_private *dev_priv) +static void gen7_oa_disable(struct i915_perf_stream *stream) { + struct drm_i915_private *dev_priv = stream->dev_priv; + I915_WRITE(GEN7_OACONTROL, 0); if (intel_wait_for_register(dev_priv, GEN7_OACONTROL, GEN7_OACONTROL_ENABLE, 0, @@ -1922,8 +1931,10 @@ static void gen7_oa_disable(struct drm_i915_private *dev_priv) DRM_ERROR("wait for OA to be disabled timed out\n"); } -static void gen8_oa_disable(struct drm_i915_private *dev_priv) +static void gen8_oa_disable(struct i915_perf_stream *stream) { + struct drm_i915_private *dev_priv = stream->dev_priv; + I915_WRITE(GEN8_OACONTROL, 0); if (intel_wait_for_register(dev_priv, GEN8_OACONTROL, GEN8_OA_COUNTER_ENABLE, 0, @@ -1943,7 +1954,7 @@ static void i915_oa_stream_disable(struct i915_perf_stream *stream) { struct drm_i915_private *dev_priv = stream->dev_priv; - dev_priv->perf.oa.ops.oa_disable(dev_priv); + dev_priv->perf.oa.ops.oa_disable(stream); if (dev_priv->perf.oa.periodic) hrtimer_cancel(&dev_priv->perf.oa.poll_check_timer); @@ -1998,7 +2009,7 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream, return -EINVAL; } - if (!dev_priv->perf.oa.ops.init_oa_buffer) { + if (!dev_priv->perf.oa.ops.enable_metric_set) { DRM_DEBUG("OA unit not supported\n"); return -ENODEV; } @@ -2081,7 +2092,7 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream, * In our case we are expecting that taking pm + FORCEWAKE * references will effectively disable RC6. */ - intel_runtime_pm_get(dev_priv); + stream->wakeref = intel_runtime_pm_get(dev_priv); intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); ret = alloc_oa_buffer(dev_priv); @@ -2092,22 +2103,21 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream, if (ret) goto err_lock; - ret = dev_priv->perf.oa.ops.enable_metric_set(dev_priv, - stream->oa_config); + stream->ops = &i915_oa_stream_ops; + dev_priv->perf.oa.exclusive_stream = stream; + + ret = dev_priv->perf.oa.ops.enable_metric_set(stream); if (ret) { DRM_DEBUG("Unable to enable metric set\n"); goto err_enable; } - stream->ops = &i915_oa_stream_ops; - - dev_priv->perf.oa.exclusive_stream = stream; - mutex_unlock(&dev_priv->drm.struct_mutex); return 0; err_enable: + dev_priv->perf.oa.exclusive_stream = NULL; dev_priv->perf.oa.ops.disable_metric_set(dev_priv); mutex_unlock(&dev_priv->drm.struct_mutex); @@ -2118,7 +2128,7 @@ err_oa_buf_alloc: put_oa_config(dev_priv, stream->oa_config); intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, stream->wakeref); err_config: if (stream->ctx) @@ -2641,7 +2651,7 @@ err: static u64 oa_exponent_to_ns(struct drm_i915_private *dev_priv, int exponent) { return div64_u64(1000000000ULL * (2ULL << exponent), - 1000ULL * INTEL_INFO(dev_priv)->cs_timestamp_frequency_khz); + 1000ULL * RUNTIME_INFO(dev_priv)->cs_timestamp_frequency_khz); } /** @@ -3016,7 +3026,7 @@ static bool chv_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr) (addr >= 0x182300 && addr <= 0x1823A4); } -static uint32_t mask_reg_value(u32 reg, u32 val) +static u32 mask_reg_value(u32 reg, u32 val) { /* HALF_SLICE_CHICKEN2 is programmed with a the * WaDisableSTUnitPowerOptimization workaround. Make sure the value @@ -3047,7 +3057,7 @@ static struct i915_oa_reg *alloc_oa_regs(struct drm_i915_private *dev_priv, if (!n_regs) return NULL; - if (!access_ok(VERIFY_READ, regs, n_regs * sizeof(u32) * 2)) + if (!access_ok(regs, n_regs * sizeof(u32) * 2)) return ERR_PTR(-EFAULT); /* No is_valid function means we're not allowing any register to be programmed. */ @@ -3387,7 +3397,6 @@ void i915_perf_init(struct drm_i915_private *dev_priv) dev_priv->perf.oa.ops.is_valid_mux_reg = hsw_is_valid_mux_addr; dev_priv->perf.oa.ops.is_valid_flex_reg = NULL; - dev_priv->perf.oa.ops.init_oa_buffer = gen7_init_oa_buffer; dev_priv->perf.oa.ops.enable_metric_set = hsw_enable_metric_set; dev_priv->perf.oa.ops.disable_metric_set = hsw_disable_metric_set; dev_priv->perf.oa.ops.oa_enable = gen7_oa_enable; @@ -3406,13 +3415,12 @@ void i915_perf_init(struct drm_i915_private *dev_priv) */ dev_priv->perf.oa.oa_formats = gen8_plus_oa_formats; - dev_priv->perf.oa.ops.init_oa_buffer = gen8_init_oa_buffer; dev_priv->perf.oa.ops.oa_enable = gen8_oa_enable; dev_priv->perf.oa.ops.oa_disable = gen8_oa_disable; dev_priv->perf.oa.ops.read = gen8_oa_read; dev_priv->perf.oa.ops.oa_hw_tail_read = gen8_oa_hw_tail_read; - if (IS_GEN8(dev_priv) || IS_GEN9(dev_priv)) { + if (IS_GEN_RANGE(dev_priv, 8, 9)) { dev_priv->perf.oa.ops.is_valid_b_counter_reg = gen7_is_valid_b_counter_addr; dev_priv->perf.oa.ops.is_valid_mux_reg = @@ -3428,7 +3436,7 @@ void i915_perf_init(struct drm_i915_private *dev_priv) dev_priv->perf.oa.ops.enable_metric_set = gen8_enable_metric_set; dev_priv->perf.oa.ops.disable_metric_set = gen8_disable_metric_set; - if (IS_GEN8(dev_priv)) { + if (IS_GEN(dev_priv, 8)) { dev_priv->perf.oa.ctx_oactxctrl_offset = 0x120; dev_priv->perf.oa.ctx_flexeu0_offset = 0x2ce; @@ -3439,7 +3447,7 @@ void i915_perf_init(struct drm_i915_private *dev_priv) dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<16); } - } else if (IS_GEN(dev_priv, 10, 11)) { + } else if (IS_GEN_RANGE(dev_priv, 10, 11)) { dev_priv->perf.oa.ops.is_valid_b_counter_reg = gen7_is_valid_b_counter_addr; dev_priv->perf.oa.ops.is_valid_mux_reg = @@ -3468,7 +3476,7 @@ void i915_perf_init(struct drm_i915_private *dev_priv) spin_lock_init(&dev_priv->perf.oa.oa_buffer.ptr_lock); oa_sample_rate_hard_limit = 1000 * - (INTEL_INFO(dev_priv)->cs_timestamp_frequency_khz / 2); + (RUNTIME_INFO(dev_priv)->cs_timestamp_frequency_khz / 2); dev_priv->perf.sysctl_header = register_sysctl_table(dev_root); mutex_init(&dev_priv->perf.metrics_lock); diff --git a/drivers/gpu/drm/i915/i915_pmu.c b/drivers/gpu/drm/i915/i915_pmu.c index d6c8f8fdfda5..b745c49a5af6 100644 --- a/drivers/gpu/drm/i915/i915_pmu.c +++ b/drivers/gpu/drm/i915/i915_pmu.c @@ -5,6 +5,7 @@ */ #include <linux/irq.h> +#include <linux/pm_runtime.h> #include "i915_pmu.h" #include "intel_ringbuffer.h" #include "i915_drv.h" @@ -167,6 +168,7 @@ engines_sample(struct drm_i915_private *dev_priv, unsigned int period_ns) { struct intel_engine_cs *engine; enum intel_engine_id id; + intel_wakeref_t wakeref; bool fw = false; if ((dev_priv->pmu.enable & ENGINE_SAMPLE_MASK) == 0) @@ -175,7 +177,8 @@ engines_sample(struct drm_i915_private *dev_priv, unsigned int period_ns) if (!dev_priv->gt.awake) return; - if (!intel_runtime_pm_get_if_in_use(dev_priv)) + wakeref = intel_runtime_pm_get_if_in_use(dev_priv); + if (!wakeref) return; for_each_engine(engine, dev_priv, id) { @@ -210,7 +213,7 @@ engines_sample(struct drm_i915_private *dev_priv, unsigned int period_ns) if (fw) intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, wakeref); } static void @@ -227,11 +230,12 @@ frequency_sample(struct drm_i915_private *dev_priv, unsigned int period_ns) u32 val; val = dev_priv->gt_pm.rps.cur_freq; - if (dev_priv->gt.awake && - intel_runtime_pm_get_if_in_use(dev_priv)) { - val = intel_get_cagf(dev_priv, - I915_READ_NOTRACE(GEN6_RPSTAT1)); - intel_runtime_pm_put(dev_priv); + if (dev_priv->gt.awake) { + intel_wakeref_t wakeref; + + with_intel_runtime_pm_if_in_use(dev_priv, wakeref) + val = intel_get_cagf(dev_priv, + I915_READ_NOTRACE(GEN6_RPSTAT1)); } add_sample_mult(&dev_priv->pmu.sample[__I915_SAMPLE_FREQ_ACT], @@ -443,12 +447,14 @@ static u64 __get_rc6(struct drm_i915_private *i915) static u64 get_rc6(struct drm_i915_private *i915) { #if IS_ENABLED(CONFIG_PM) + intel_wakeref_t wakeref; unsigned long flags; u64 val; - if (intel_runtime_pm_get_if_in_use(i915)) { + wakeref = intel_runtime_pm_get_if_in_use(i915); + if (wakeref) { val = __get_rc6(i915); - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); /* * If we are coming back from being runtime suspended we must @@ -478,7 +484,6 @@ static u64 get_rc6(struct drm_i915_private *i915) * counter value. */ spin_lock_irqsave(&i915->pmu.lock, flags); - spin_lock(&kdev->power.lock); /* * After the above branch intel_runtime_pm_get_if_in_use failed @@ -491,16 +496,13 @@ static u64 get_rc6(struct drm_i915_private *i915) * suspended and if not we cannot do better than report the last * known RC6 value. */ - if (kdev->power.runtime_status == RPM_SUSPENDED) { - if (!i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur) - i915->pmu.suspended_jiffies_last = - kdev->power.suspended_jiffies; + if (pm_runtime_status_suspended(kdev)) { + val = pm_runtime_suspended_time(kdev); - val = kdev->power.suspended_jiffies - - i915->pmu.suspended_jiffies_last; - val += jiffies - kdev->power.accounting_timestamp; + if (!i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur) + i915->pmu.suspended_time_last = val; - val = jiffies_to_nsecs(val); + val -= i915->pmu.suspended_time_last; val += i915->pmu.sample[__I915_SAMPLE_RC6].cur; i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur = val; @@ -510,7 +512,6 @@ static u64 get_rc6(struct drm_i915_private *i915) val = i915->pmu.sample[__I915_SAMPLE_RC6].cur; } - spin_unlock(&kdev->power.lock); spin_unlock_irqrestore(&i915->pmu.lock, flags); } @@ -594,7 +595,8 @@ static void i915_pmu_enable(struct perf_event *event) * Update the bitmask of enabled events and increment * the event reference counter. */ - GEM_BUG_ON(bit >= I915_PMU_MASK_BITS); + BUILD_BUG_ON(ARRAY_SIZE(i915->pmu.enable_count) != I915_PMU_MASK_BITS); + GEM_BUG_ON(bit >= ARRAY_SIZE(i915->pmu.enable_count)); GEM_BUG_ON(i915->pmu.enable_count[bit] == ~0); i915->pmu.enable |= BIT_ULL(bit); i915->pmu.enable_count[bit]++; @@ -615,11 +617,16 @@ static void i915_pmu_enable(struct perf_event *event) engine = intel_engine_lookup_user(i915, engine_event_class(event), engine_event_instance(event)); - GEM_BUG_ON(!engine); - engine->pmu.enable |= BIT(sample); - GEM_BUG_ON(sample >= I915_PMU_SAMPLE_BITS); + BUILD_BUG_ON(ARRAY_SIZE(engine->pmu.enable_count) != + I915_ENGINE_SAMPLE_COUNT); + BUILD_BUG_ON(ARRAY_SIZE(engine->pmu.sample) != + I915_ENGINE_SAMPLE_COUNT); + GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.enable_count)); + GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.sample)); GEM_BUG_ON(engine->pmu.enable_count[sample] == ~0); + + engine->pmu.enable |= BIT(sample); engine->pmu.enable_count[sample]++; } @@ -649,9 +656,11 @@ static void i915_pmu_disable(struct perf_event *event) engine = intel_engine_lookup_user(i915, engine_event_class(event), engine_event_instance(event)); - GEM_BUG_ON(!engine); - GEM_BUG_ON(sample >= I915_PMU_SAMPLE_BITS); + + GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.enable_count)); + GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.sample)); GEM_BUG_ON(engine->pmu.enable_count[sample] == 0); + /* * Decrement the reference count and clear the enabled * bitmask when the last listener on an event goes away. @@ -660,7 +669,7 @@ static void i915_pmu_disable(struct perf_event *event) engine->pmu.enable &= ~BIT(sample); } - GEM_BUG_ON(bit >= I915_PMU_MASK_BITS); + GEM_BUG_ON(bit >= ARRAY_SIZE(i915->pmu.enable_count)); GEM_BUG_ON(i915->pmu.enable_count[bit] == 0); /* * Decrement the reference count and clear the enabled diff --git a/drivers/gpu/drm/i915/i915_pmu.h b/drivers/gpu/drm/i915/i915_pmu.h index 7f164ca3db12..4fc4f2478301 100644 --- a/drivers/gpu/drm/i915/i915_pmu.h +++ b/drivers/gpu/drm/i915/i915_pmu.h @@ -31,6 +31,8 @@ enum { ((1 << I915_PMU_SAMPLE_BITS) + \ (I915_PMU_LAST + 1 - __I915_PMU_OTHER(0))) +#define I915_ENGINE_SAMPLE_COUNT (I915_SAMPLE_SEMA + 1) + struct i915_pmu_sample { u64 cur; }; @@ -95,9 +97,9 @@ struct i915_pmu { */ struct i915_pmu_sample sample[__I915_NUM_PMU_SAMPLERS]; /** - * @suspended_jiffies_last: Cached suspend time from PM core. + * @suspended_time_last: Cached suspend time from PM core. */ - unsigned long suspended_jiffies_last; + u64 suspended_time_last; /** * @i915_attr: Memory block holding device attributes. */ diff --git a/drivers/gpu/drm/i915/i915_query.c b/drivers/gpu/drm/i915/i915_query.c index 3f502eef2431..cbcb957b7141 100644 --- a/drivers/gpu/drm/i915/i915_query.c +++ b/drivers/gpu/drm/i915/i915_query.c @@ -13,7 +13,7 @@ static int query_topology_info(struct drm_i915_private *dev_priv, struct drm_i915_query_item *query_item) { - const struct sseu_dev_info *sseu = &INTEL_INFO(dev_priv)->sseu; + const struct sseu_dev_info *sseu = &RUNTIME_INFO(dev_priv)->sseu; struct drm_i915_query_topology_info topo; u32 slice_length, subslice_length, eu_length, total_length; @@ -27,8 +27,7 @@ static int query_topology_info(struct drm_i915_private *dev_priv, slice_length = sizeof(sseu->slice_mask); subslice_length = sseu->max_slices * - DIV_ROUND_UP(sseu->max_subslices, - sizeof(sseu->subslice_mask[0]) * BITS_PER_BYTE); + DIV_ROUND_UP(sseu->max_subslices, BITS_PER_BYTE); eu_length = sseu->max_slices * sseu->max_subslices * DIV_ROUND_UP(sseu->max_eus_per_subslice, BITS_PER_BYTE); @@ -47,7 +46,7 @@ static int query_topology_info(struct drm_i915_private *dev_priv, if (topo.flags != 0) return -EINVAL; - if (!access_ok(VERIFY_WRITE, u64_to_user_ptr(query_item->data_ptr), + if (!access_ok(u64_to_user_ptr(query_item->data_ptr), total_length)) return -EFAULT; diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index e31c27e45734..047855dd8c6b 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -117,14 +117,14 @@ */ typedef struct { - uint32_t reg; + u32 reg; } i915_reg_t; #define _MMIO(r) ((const i915_reg_t){ .reg = (r) }) #define INVALID_MMIO_REG _MMIO(0) -static inline uint32_t i915_mmio_reg_offset(i915_reg_t reg) +static inline u32 i915_mmio_reg_offset(i915_reg_t reg) { return reg.reg; } @@ -139,6 +139,12 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) return !i915_mmio_reg_equal(reg, INVALID_MMIO_REG); } +#define VLV_DISPLAY_BASE 0x180000 +#define VLV_MIPI_BASE VLV_DISPLAY_BASE +#define BXT_MIPI_BASE 0x60000 + +#define DISPLAY_MMIO_BASE(dev_priv) (INTEL_INFO(dev_priv)->display_mmio_offset) + /* * Given the first two numbers __a and __b of arbitrarily many evenly spaced * numbers, pick the 0-based __index'th value. @@ -157,20 +163,37 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) /* * Named helper wrappers around _PICK_EVEN() and _PICK(). */ -#define _PIPE(pipe, a, b) _PICK_EVEN(pipe, a, b) -#define _MMIO_PIPE(pipe, a, b) _MMIO(_PIPE(pipe, a, b)) -#define _PLANE(plane, a, b) _PICK_EVEN(plane, a, b) -#define _MMIO_PLANE(plane, a, b) _MMIO_PIPE(plane, a, b) -#define _TRANS(tran, a, b) _PICK_EVEN(tran, a, b) -#define _MMIO_TRANS(tran, a, b) _MMIO(_TRANS(tran, a, b)) -#define _PORT(port, a, b) _PICK_EVEN(port, a, b) -#define _MMIO_PORT(port, a, b) _MMIO(_PORT(port, a, b)) -#define _MMIO_PIPE3(pipe, a, b, c) _MMIO(_PICK(pipe, a, b, c)) -#define _MMIO_PORT3(pipe, a, b, c) _MMIO(_PICK(pipe, a, b, c)) -#define _PLL(pll, a, b) _PICK_EVEN(pll, a, b) -#define _MMIO_PLL(pll, a, b) _MMIO(_PLL(pll, a, b)) -#define _PHY3(phy, ...) _PICK(phy, __VA_ARGS__) -#define _MMIO_PHY3(phy, a, b, c) _MMIO(_PHY3(phy, a, b, c)) +#define _PIPE(pipe, a, b) _PICK_EVEN(pipe, a, b) +#define _PLANE(plane, a, b) _PICK_EVEN(plane, a, b) +#define _TRANS(tran, a, b) _PICK_EVEN(tran, a, b) +#define _PORT(port, a, b) _PICK_EVEN(port, a, b) +#define _PLL(pll, a, b) _PICK_EVEN(pll, a, b) + +#define _MMIO_PIPE(pipe, a, b) _MMIO(_PIPE(pipe, a, b)) +#define _MMIO_PLANE(plane, a, b) _MMIO(_PLANE(plane, a, b)) +#define _MMIO_TRANS(tran, a, b) _MMIO(_TRANS(tran, a, b)) +#define _MMIO_PORT(port, a, b) _MMIO(_PORT(port, a, b)) +#define _MMIO_PLL(pll, a, b) _MMIO(_PLL(pll, a, b)) + +#define _PHY3(phy, ...) _PICK(phy, __VA_ARGS__) + +#define _MMIO_PIPE3(pipe, a, b, c) _MMIO(_PICK(pipe, a, b, c)) +#define _MMIO_PORT3(pipe, a, b, c) _MMIO(_PICK(pipe, a, b, c)) +#define _MMIO_PHY3(phy, a, b, c) _MMIO(_PHY3(phy, a, b, c)) + +/* + * Device info offset array based helpers for groups of registers with unevenly + * spaced base offsets. + */ +#define _MMIO_PIPE2(pipe, reg) _MMIO(INTEL_INFO(dev_priv)->pipe_offsets[pipe] - \ + INTEL_INFO(dev_priv)->pipe_offsets[PIPE_A] + (reg) + \ + DISPLAY_MMIO_BASE(dev_priv)) +#define _MMIO_TRANS2(pipe, reg) _MMIO(INTEL_INFO(dev_priv)->trans_offsets[(pipe)] - \ + INTEL_INFO(dev_priv)->trans_offsets[TRANSCODER_A] + (reg) + \ + DISPLAY_MMIO_BASE(dev_priv)) +#define _CURSOR2(pipe, reg) _MMIO(INTEL_INFO(dev_priv)->cursor_offsets[(pipe)] - \ + INTEL_INFO(dev_priv)->cursor_offsets[PIPE_A] + (reg) + \ + DISPLAY_MMIO_BASE(dev_priv)) #define __MASKED_FIELD(mask, value) ((mask) << 16 | (value)) #define _MASKED_FIELD(mask, value) ({ \ @@ -330,6 +353,24 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define GEN11_GRDOM_MEDIA4 (1 << 8) #define GEN11_GRDOM_VECS (1 << 13) #define GEN11_GRDOM_VECS2 (1 << 14) +#define GEN11_GRDOM_SFC0 (1 << 17) +#define GEN11_GRDOM_SFC1 (1 << 18) + +#define GEN11_VCS_SFC_RESET_BIT(instance) (GEN11_GRDOM_SFC0 << ((instance) >> 1)) +#define GEN11_VECS_SFC_RESET_BIT(instance) (GEN11_GRDOM_SFC0 << (instance)) + +#define GEN11_VCS_SFC_FORCED_LOCK(engine) _MMIO((engine)->mmio_base + 0x88C) +#define GEN11_VCS_SFC_FORCED_LOCK_BIT (1 << 0) +#define GEN11_VCS_SFC_LOCK_STATUS(engine) _MMIO((engine)->mmio_base + 0x890) +#define GEN11_VCS_SFC_USAGE_BIT (1 << 0) +#define GEN11_VCS_SFC_LOCK_ACK_BIT (1 << 1) + +#define GEN11_VECS_SFC_FORCED_LOCK(engine) _MMIO((engine)->mmio_base + 0x201C) +#define GEN11_VECS_SFC_FORCED_LOCK_BIT (1 << 0) +#define GEN11_VECS_SFC_LOCK_ACK(engine) _MMIO((engine)->mmio_base + 0x2018) +#define GEN11_VECS_SFC_LOCK_ACK_BIT (1 << 0) +#define GEN11_VECS_SFC_USAGE(engine) _MMIO((engine)->mmio_base + 0x2014) +#define GEN11_VECS_SFC_USAGE_BIT (1 << 0) #define RING_PP_DIR_BASE(engine) _MMIO((engine)->mmio_base + 0x228) #define RING_PP_DIR_BASE_READ(engine) _MMIO((engine)->mmio_base + 0x518) @@ -1631,35 +1672,6 @@ enum i915_power_well_id { #define PHY_RESERVED (1 << 7) #define BXT_PORT_CL1CM_DW0(phy) _BXT_PHY((phy), _PORT_CL1CM_DW0_BC) -#define CNL_PORT_CL1CM_DW5 _MMIO(0x162014) -#define CL_POWER_DOWN_ENABLE (1 << 4) -#define SUS_CLOCK_CONFIG (3 << 0) - -#define _ICL_PORT_CL_DW5_A 0x162014 -#define _ICL_PORT_CL_DW5_B 0x6C014 -#define ICL_PORT_CL_DW5(port) _MMIO_PORT(port, _ICL_PORT_CL_DW5_A, \ - _ICL_PORT_CL_DW5_B) - -#define _CNL_PORT_CL_DW10_A 0x162028 -#define _ICL_PORT_CL_DW10_B 0x6c028 -#define ICL_PORT_CL_DW10(port) _MMIO_PORT(port, \ - _CNL_PORT_CL_DW10_A, \ - _ICL_PORT_CL_DW10_B) -#define PG_SEQ_DELAY_OVERRIDE_MASK (3 << 25) -#define PG_SEQ_DELAY_OVERRIDE_SHIFT 25 -#define PG_SEQ_DELAY_OVERRIDE_ENABLE (1 << 24) -#define PWR_UP_ALL_LANES (0x0 << 4) -#define PWR_DOWN_LN_3_2_1 (0xe << 4) -#define PWR_DOWN_LN_3_2 (0xc << 4) -#define PWR_DOWN_LN_3 (0x8 << 4) -#define PWR_DOWN_LN_2_1_0 (0x7 << 4) -#define PWR_DOWN_LN_1_0 (0x3 << 4) -#define PWR_DOWN_LN_1 (0x2 << 4) -#define PWR_DOWN_LN_3_1 (0xa << 4) -#define PWR_DOWN_LN_3_1_0 (0xb << 4) -#define PWR_DOWN_LN_MASK (0xf << 4) -#define PWR_DOWN_LN_SHIFT 4 - #define _PORT_CL1CM_DW9_A 0x162024 #define _PORT_CL1CM_DW9_BC 0x6C024 #define IREF0RC_OFFSET_SHIFT 8 @@ -1672,13 +1684,6 @@ enum i915_power_well_id { #define IREF1RC_OFFSET_MASK (0xFF << IREF1RC_OFFSET_SHIFT) #define BXT_PORT_CL1CM_DW10(phy) _BXT_PHY((phy), _PORT_CL1CM_DW10_BC) -#define _ICL_PORT_CL_DW12_A 0x162030 -#define _ICL_PORT_CL_DW12_B 0x6C030 -#define ICL_LANE_ENABLE_AUX (1 << 0) -#define ICL_PORT_CL_DW12(port) _MMIO_PORT((port), \ - _ICL_PORT_CL_DW12_A, \ - _ICL_PORT_CL_DW12_B) - #define _PORT_CL1CM_DW28_A 0x162070 #define _PORT_CL1CM_DW28_BC 0x6C070 #define OCL1_POWER_DOWN_EN (1 << 23) @@ -1691,6 +1696,74 @@ enum i915_power_well_id { #define OCL2_LDOFUSE_PWR_DIS (1 << 6) #define BXT_PORT_CL1CM_DW30(phy) _BXT_PHY((phy), _PORT_CL1CM_DW30_BC) +/* + * CNL/ICL Port/COMBO-PHY Registers + */ +#define _ICL_COMBOPHY_A 0x162000 +#define _ICL_COMBOPHY_B 0x6C000 +#define _ICL_COMBOPHY(port) _PICK(port, _ICL_COMBOPHY_A, \ + _ICL_COMBOPHY_B) + +/* CNL/ICL Port CL_DW registers */ +#define _ICL_PORT_CL_DW(dw, port) (_ICL_COMBOPHY(port) + \ + 4 * (dw)) + +#define CNL_PORT_CL1CM_DW5 _MMIO(0x162014) +#define ICL_PORT_CL_DW5(port) _MMIO(_ICL_PORT_CL_DW(5, port)) +#define CL_POWER_DOWN_ENABLE (1 << 4) +#define SUS_CLOCK_CONFIG (3 << 0) + +#define ICL_PORT_CL_DW10(port) _MMIO(_ICL_PORT_CL_DW(10, port)) +#define PG_SEQ_DELAY_OVERRIDE_MASK (3 << 25) +#define PG_SEQ_DELAY_OVERRIDE_SHIFT 25 +#define PG_SEQ_DELAY_OVERRIDE_ENABLE (1 << 24) +#define PWR_UP_ALL_LANES (0x0 << 4) +#define PWR_DOWN_LN_3_2_1 (0xe << 4) +#define PWR_DOWN_LN_3_2 (0xc << 4) +#define PWR_DOWN_LN_3 (0x8 << 4) +#define PWR_DOWN_LN_2_1_0 (0x7 << 4) +#define PWR_DOWN_LN_1_0 (0x3 << 4) +#define PWR_DOWN_LN_1 (0x2 << 4) +#define PWR_DOWN_LN_3_1 (0xa << 4) +#define PWR_DOWN_LN_3_1_0 (0xb << 4) +#define PWR_DOWN_LN_MASK (0xf << 4) +#define PWR_DOWN_LN_SHIFT 4 + +#define ICL_PORT_CL_DW12(port) _MMIO(_ICL_PORT_CL_DW(12, port)) +#define ICL_LANE_ENABLE_AUX (1 << 0) + +/* CNL/ICL Port COMP_DW registers */ +#define _ICL_PORT_COMP 0x100 +#define _ICL_PORT_COMP_DW(dw, port) (_ICL_COMBOPHY(port) + \ + _ICL_PORT_COMP + 4 * (dw)) + +#define CNL_PORT_COMP_DW0 _MMIO(0x162100) +#define ICL_PORT_COMP_DW0(port) _MMIO(_ICL_PORT_COMP_DW(0, port)) +#define COMP_INIT (1 << 31) + +#define CNL_PORT_COMP_DW1 _MMIO(0x162104) +#define ICL_PORT_COMP_DW1(port) _MMIO(_ICL_PORT_COMP_DW(1, port)) + +#define CNL_PORT_COMP_DW3 _MMIO(0x16210c) +#define ICL_PORT_COMP_DW3(port) _MMIO(_ICL_PORT_COMP_DW(3, port)) +#define PROCESS_INFO_DOT_0 (0 << 26) +#define PROCESS_INFO_DOT_1 (1 << 26) +#define PROCESS_INFO_DOT_4 (2 << 26) +#define PROCESS_INFO_MASK (7 << 26) +#define PROCESS_INFO_SHIFT 26 +#define VOLTAGE_INFO_0_85V (0 << 24) +#define VOLTAGE_INFO_0_95V (1 << 24) +#define VOLTAGE_INFO_1_05V (2 << 24) +#define VOLTAGE_INFO_MASK (3 << 24) +#define VOLTAGE_INFO_SHIFT 24 + +#define CNL_PORT_COMP_DW9 _MMIO(0x162124) +#define ICL_PORT_COMP_DW9(port) _MMIO(_ICL_PORT_COMP_DW(9, port)) + +#define CNL_PORT_COMP_DW10 _MMIO(0x162128) +#define ICL_PORT_COMP_DW10(port) _MMIO(_ICL_PORT_COMP_DW(10, port)) + +/* CNL/ICL Port PCS registers */ #define _CNL_PORT_PCS_DW1_GRP_AE 0x162304 #define _CNL_PORT_PCS_DW1_GRP_B 0x162384 #define _CNL_PORT_PCS_DW1_GRP_C 0x162B04 @@ -1708,7 +1781,6 @@ enum i915_power_well_id { _CNL_PORT_PCS_DW1_GRP_D, \ _CNL_PORT_PCS_DW1_GRP_AE, \ _CNL_PORT_PCS_DW1_GRP_F)) - #define CNL_PORT_PCS_DW1_LN0(port) _MMIO(_PICK(port, \ _CNL_PORT_PCS_DW1_LN0_AE, \ _CNL_PORT_PCS_DW1_LN0_B, \ @@ -1717,24 +1789,21 @@ enum i915_power_well_id { _CNL_PORT_PCS_DW1_LN0_AE, \ _CNL_PORT_PCS_DW1_LN0_F)) -#define _ICL_PORT_PCS_DW1_GRP_A 0x162604 -#define _ICL_PORT_PCS_DW1_GRP_B 0x6C604 -#define _ICL_PORT_PCS_DW1_LN0_A 0x162804 -#define _ICL_PORT_PCS_DW1_LN0_B 0x6C804 -#define _ICL_PORT_PCS_DW1_AUX_A 0x162304 -#define _ICL_PORT_PCS_DW1_AUX_B 0x6c304 -#define ICL_PORT_PCS_DW1_GRP(port) _MMIO_PORT(port,\ - _ICL_PORT_PCS_DW1_GRP_A, \ - _ICL_PORT_PCS_DW1_GRP_B) -#define ICL_PORT_PCS_DW1_LN0(port) _MMIO_PORT(port, \ - _ICL_PORT_PCS_DW1_LN0_A, \ - _ICL_PORT_PCS_DW1_LN0_B) -#define ICL_PORT_PCS_DW1_AUX(port) _MMIO_PORT(port, \ - _ICL_PORT_PCS_DW1_AUX_A, \ - _ICL_PORT_PCS_DW1_AUX_B) +#define _ICL_PORT_PCS_AUX 0x300 +#define _ICL_PORT_PCS_GRP 0x600 +#define _ICL_PORT_PCS_LN(ln) (0x800 + (ln) * 0x100) +#define _ICL_PORT_PCS_DW_AUX(dw, port) (_ICL_COMBOPHY(port) + \ + _ICL_PORT_PCS_AUX + 4 * (dw)) +#define _ICL_PORT_PCS_DW_GRP(dw, port) (_ICL_COMBOPHY(port) + \ + _ICL_PORT_PCS_GRP + 4 * (dw)) +#define _ICL_PORT_PCS_DW_LN(dw, ln, port) (_ICL_COMBOPHY(port) + \ + _ICL_PORT_PCS_LN(ln) + 4 * (dw)) +#define ICL_PORT_PCS_DW1_AUX(port) _MMIO(_ICL_PORT_PCS_DW_AUX(1, port)) +#define ICL_PORT_PCS_DW1_GRP(port) _MMIO(_ICL_PORT_PCS_DW_GRP(1, port)) +#define ICL_PORT_PCS_DW1_LN0(port) _MMIO(_ICL_PORT_PCS_DW_LN(1, 0, port)) #define COMMON_KEEPER_EN (1 << 26) -/* CNL Port TX registers */ +/* CNL/ICL Port TX registers */ #define _CNL_PORT_TX_AE_GRP_OFFSET 0x162340 #define _CNL_PORT_TX_B_GRP_OFFSET 0x1623C0 #define _CNL_PORT_TX_C_GRP_OFFSET 0x162B40 @@ -1745,7 +1814,7 @@ enum i915_power_well_id { #define _CNL_PORT_TX_C_LN0_OFFSET 0x162C40 #define _CNL_PORT_TX_D_LN0_OFFSET 0x162E40 #define _CNL_PORT_TX_F_LN0_OFFSET 0x162840 -#define _CNL_PORT_TX_DW_GRP(port, dw) (_PICK((port), \ +#define _CNL_PORT_TX_DW_GRP(dw, port) (_PICK((port), \ _CNL_PORT_TX_AE_GRP_OFFSET, \ _CNL_PORT_TX_B_GRP_OFFSET, \ _CNL_PORT_TX_B_GRP_OFFSET, \ @@ -1753,7 +1822,7 @@ enum i915_power_well_id { _CNL_PORT_TX_AE_GRP_OFFSET, \ _CNL_PORT_TX_F_GRP_OFFSET) + \ 4 * (dw)) -#define _CNL_PORT_TX_DW_LN0(port, dw) (_PICK((port), \ +#define _CNL_PORT_TX_DW_LN0(dw, port) (_PICK((port), \ _CNL_PORT_TX_AE_LN0_OFFSET, \ _CNL_PORT_TX_B_LN0_OFFSET, \ _CNL_PORT_TX_B_LN0_OFFSET, \ @@ -1762,23 +1831,22 @@ enum i915_power_well_id { _CNL_PORT_TX_F_LN0_OFFSET) + \ 4 * (dw)) -#define CNL_PORT_TX_DW2_GRP(port) _MMIO(_CNL_PORT_TX_DW_GRP((port), 2)) -#define CNL_PORT_TX_DW2_LN0(port) _MMIO(_CNL_PORT_TX_DW_LN0((port), 2)) -#define _ICL_PORT_TX_DW2_GRP_A 0x162688 -#define _ICL_PORT_TX_DW2_GRP_B 0x6C688 -#define _ICL_PORT_TX_DW2_LN0_A 0x162888 -#define _ICL_PORT_TX_DW2_LN0_B 0x6C888 -#define _ICL_PORT_TX_DW2_AUX_A 0x162388 -#define _ICL_PORT_TX_DW2_AUX_B 0x6c388 -#define ICL_PORT_TX_DW2_GRP(port) _MMIO_PORT(port, \ - _ICL_PORT_TX_DW2_GRP_A, \ - _ICL_PORT_TX_DW2_GRP_B) -#define ICL_PORT_TX_DW2_LN0(port) _MMIO_PORT(port, \ - _ICL_PORT_TX_DW2_LN0_A, \ - _ICL_PORT_TX_DW2_LN0_B) -#define ICL_PORT_TX_DW2_AUX(port) _MMIO_PORT(port, \ - _ICL_PORT_TX_DW2_AUX_A, \ - _ICL_PORT_TX_DW2_AUX_B) +#define _ICL_PORT_TX_AUX 0x380 +#define _ICL_PORT_TX_GRP 0x680 +#define _ICL_PORT_TX_LN(ln) (0x880 + (ln) * 0x100) + +#define _ICL_PORT_TX_DW_AUX(dw, port) (_ICL_COMBOPHY(port) + \ + _ICL_PORT_TX_AUX + 4 * (dw)) +#define _ICL_PORT_TX_DW_GRP(dw, port) (_ICL_COMBOPHY(port) + \ + _ICL_PORT_TX_GRP + 4 * (dw)) +#define _ICL_PORT_TX_DW_LN(dw, ln, port) (_ICL_COMBOPHY(port) + \ + _ICL_PORT_TX_LN(ln) + 4 * (dw)) + +#define CNL_PORT_TX_DW2_GRP(port) _MMIO(_CNL_PORT_TX_DW_GRP(2, port)) +#define CNL_PORT_TX_DW2_LN0(port) _MMIO(_CNL_PORT_TX_DW_LN0(2, port)) +#define ICL_PORT_TX_DW2_AUX(port) _MMIO(_ICL_PORT_TX_DW_AUX(2, port)) +#define ICL_PORT_TX_DW2_GRP(port) _MMIO(_ICL_PORT_TX_DW_GRP(2, port)) +#define ICL_PORT_TX_DW2_LN0(port) _MMIO(_ICL_PORT_TX_DW_LN(2, 0, port)) #define SWING_SEL_UPPER(x) (((x) >> 3) << 15) #define SWING_SEL_UPPER_MASK (1 << 15) #define SWING_SEL_LOWER(x) (((x) & 0x7) << 11) @@ -1790,29 +1858,15 @@ enum i915_power_well_id { #define _CNL_PORT_TX_DW4_LN0_AE 0x162450 #define _CNL_PORT_TX_DW4_LN1_AE 0x1624D0 -#define CNL_PORT_TX_DW4_GRP(port) _MMIO(_CNL_PORT_TX_DW_GRP((port), 4)) -#define CNL_PORT_TX_DW4_LN0(port) _MMIO(_CNL_PORT_TX_DW_LN0((port), 4)) -#define CNL_PORT_TX_DW4_LN(port, ln) _MMIO(_CNL_PORT_TX_DW_LN0((port), 4) + \ +#define CNL_PORT_TX_DW4_GRP(port) _MMIO(_CNL_PORT_TX_DW_GRP(4, (port))) +#define CNL_PORT_TX_DW4_LN0(port) _MMIO(_CNL_PORT_TX_DW_LN0(4, (port))) +#define CNL_PORT_TX_DW4_LN(port, ln) _MMIO(_CNL_PORT_TX_DW_LN0(4, (port)) + \ ((ln) * (_CNL_PORT_TX_DW4_LN1_AE - \ _CNL_PORT_TX_DW4_LN0_AE))) -#define _ICL_PORT_TX_DW4_GRP_A 0x162690 -#define _ICL_PORT_TX_DW4_GRP_B 0x6C690 -#define _ICL_PORT_TX_DW4_LN0_A 0x162890 -#define _ICL_PORT_TX_DW4_LN1_A 0x162990 -#define _ICL_PORT_TX_DW4_LN0_B 0x6C890 -#define _ICL_PORT_TX_DW4_AUX_A 0x162390 -#define _ICL_PORT_TX_DW4_AUX_B 0x6c390 -#define ICL_PORT_TX_DW4_GRP(port) _MMIO_PORT(port, \ - _ICL_PORT_TX_DW4_GRP_A, \ - _ICL_PORT_TX_DW4_GRP_B) -#define ICL_PORT_TX_DW4_LN(port, ln) _MMIO(_PORT(port, \ - _ICL_PORT_TX_DW4_LN0_A, \ - _ICL_PORT_TX_DW4_LN0_B) + \ - ((ln) * (_ICL_PORT_TX_DW4_LN1_A - \ - _ICL_PORT_TX_DW4_LN0_A))) -#define ICL_PORT_TX_DW4_AUX(port) _MMIO_PORT(port, \ - _ICL_PORT_TX_DW4_AUX_A, \ - _ICL_PORT_TX_DW4_AUX_B) +#define ICL_PORT_TX_DW4_AUX(port) _MMIO(_ICL_PORT_TX_DW_AUX(4, port)) +#define ICL_PORT_TX_DW4_GRP(port) _MMIO(_ICL_PORT_TX_DW_GRP(4, port)) +#define ICL_PORT_TX_DW4_LN0(port) _MMIO(_ICL_PORT_TX_DW_LN(4, 0, port)) +#define ICL_PORT_TX_DW4_LN(port, ln) _MMIO(_ICL_PORT_TX_DW_LN(4, ln, port)) #define LOADGEN_SELECT (1 << 31) #define POST_CURSOR_1(x) ((x) << 12) #define POST_CURSOR_1_MASK (0x3F << 12) @@ -1821,23 +1875,11 @@ enum i915_power_well_id { #define CURSOR_COEFF(x) ((x) << 0) #define CURSOR_COEFF_MASK (0x3F << 0) -#define CNL_PORT_TX_DW5_GRP(port) _MMIO(_CNL_PORT_TX_DW_GRP((port), 5)) -#define CNL_PORT_TX_DW5_LN0(port) _MMIO(_CNL_PORT_TX_DW_LN0((port), 5)) -#define _ICL_PORT_TX_DW5_GRP_A 0x162694 -#define _ICL_PORT_TX_DW5_GRP_B 0x6C694 -#define _ICL_PORT_TX_DW5_LN0_A 0x162894 -#define _ICL_PORT_TX_DW5_LN0_B 0x6C894 -#define _ICL_PORT_TX_DW5_AUX_A 0x162394 -#define _ICL_PORT_TX_DW5_AUX_B 0x6c394 -#define ICL_PORT_TX_DW5_GRP(port) _MMIO_PORT(port, \ - _ICL_PORT_TX_DW5_GRP_A, \ - _ICL_PORT_TX_DW5_GRP_B) -#define ICL_PORT_TX_DW5_LN0(port) _MMIO_PORT(port, \ - _ICL_PORT_TX_DW5_LN0_A, \ - _ICL_PORT_TX_DW5_LN0_B) -#define ICL_PORT_TX_DW5_AUX(port) _MMIO_PORT(port, \ - _ICL_PORT_TX_DW5_AUX_A, \ - _ICL_PORT_TX_DW5_AUX_B) +#define CNL_PORT_TX_DW5_GRP(port) _MMIO(_CNL_PORT_TX_DW_GRP(5, port)) +#define CNL_PORT_TX_DW5_LN0(port) _MMIO(_CNL_PORT_TX_DW_LN0(5, port)) +#define ICL_PORT_TX_DW5_AUX(port) _MMIO(_ICL_PORT_TX_DW_AUX(5, port)) +#define ICL_PORT_TX_DW5_GRP(port) _MMIO(_ICL_PORT_TX_DW_GRP(5, port)) +#define ICL_PORT_TX_DW5_LN0(port) _MMIO(_ICL_PORT_TX_DW_LN(5, 0, port)) #define TX_TRAINING_EN (1 << 31) #define TAP2_DISABLE (1 << 30) #define TAP3_DISABLE (1 << 29) @@ -1846,8 +1888,12 @@ enum i915_power_well_id { #define RTERM_SELECT(x) ((x) << 3) #define RTERM_SELECT_MASK (0x7 << 3) -#define CNL_PORT_TX_DW7_GRP(port) _MMIO(_CNL_PORT_TX_DW_GRP((port), 7)) -#define CNL_PORT_TX_DW7_LN0(port) _MMIO(_CNL_PORT_TX_DW_LN0((port), 7)) +#define CNL_PORT_TX_DW7_GRP(port) _MMIO(_CNL_PORT_TX_DW_GRP(7, (port))) +#define CNL_PORT_TX_DW7_LN0(port) _MMIO(_CNL_PORT_TX_DW_LN0(7, (port))) +#define ICL_PORT_TX_DW7_AUX(port) _MMIO(_ICL_PORT_TX_DW_AUX(7, port)) +#define ICL_PORT_TX_DW7_GRP(port) _MMIO(_ICL_PORT_TX_DW_GRP(7, port)) +#define ICL_PORT_TX_DW7_LN0(port) _MMIO(_ICL_PORT_TX_DW_LN(7, 0, port)) +#define ICL_PORT_TX_DW7_LN(port, ln) _MMIO(_ICL_PORT_TX_DW_LN(7, ln, port)) #define N_SCALAR(x) ((x) << 24) #define N_SCALAR_MASK (0x7F << 24) @@ -2054,47 +2100,10 @@ enum i915_power_well_id { #define BXT_PORT_CL2CM_DW6(phy) _BXT_PHY((phy), _PORT_CL2CM_DW6_BC) #define DW6_OLDO_DYN_PWR_DOWN_EN (1 << 28) -#define CNL_PORT_COMP_DW0 _MMIO(0x162100) -#define COMP_INIT (1 << 31) -#define CNL_PORT_COMP_DW1 _MMIO(0x162104) -#define CNL_PORT_COMP_DW3 _MMIO(0x16210c) -#define PROCESS_INFO_DOT_0 (0 << 26) -#define PROCESS_INFO_DOT_1 (1 << 26) -#define PROCESS_INFO_DOT_4 (2 << 26) -#define PROCESS_INFO_MASK (7 << 26) -#define PROCESS_INFO_SHIFT 26 -#define VOLTAGE_INFO_0_85V (0 << 24) -#define VOLTAGE_INFO_0_95V (1 << 24) -#define VOLTAGE_INFO_1_05V (2 << 24) -#define VOLTAGE_INFO_MASK (3 << 24) -#define VOLTAGE_INFO_SHIFT 24 -#define CNL_PORT_COMP_DW9 _MMIO(0x162124) -#define CNL_PORT_COMP_DW10 _MMIO(0x162128) - -#define _ICL_PORT_COMP_DW0_A 0x162100 -#define _ICL_PORT_COMP_DW0_B 0x6C100 -#define ICL_PORT_COMP_DW0(port) _MMIO_PORT(port, _ICL_PORT_COMP_DW0_A, \ - _ICL_PORT_COMP_DW0_B) -#define _ICL_PORT_COMP_DW1_A 0x162104 -#define _ICL_PORT_COMP_DW1_B 0x6C104 -#define ICL_PORT_COMP_DW1(port) _MMIO_PORT(port, _ICL_PORT_COMP_DW1_A, \ - _ICL_PORT_COMP_DW1_B) -#define _ICL_PORT_COMP_DW3_A 0x16210C -#define _ICL_PORT_COMP_DW3_B 0x6C10C -#define ICL_PORT_COMP_DW3(port) _MMIO_PORT(port, _ICL_PORT_COMP_DW3_A, \ - _ICL_PORT_COMP_DW3_B) -#define _ICL_PORT_COMP_DW9_A 0x162124 -#define _ICL_PORT_COMP_DW9_B 0x6C124 -#define ICL_PORT_COMP_DW9(port) _MMIO_PORT(port, _ICL_PORT_COMP_DW9_A, \ - _ICL_PORT_COMP_DW9_B) -#define _ICL_PORT_COMP_DW10_A 0x162128 -#define _ICL_PORT_COMP_DW10_B 0x6C128 -#define ICL_PORT_COMP_DW10(port) _MMIO_PORT(port, \ - _ICL_PORT_COMP_DW10_A, \ - _ICL_PORT_COMP_DW10_B) +#define FIA1_BASE 0x163000 /* ICL PHY DFLEX registers */ -#define PORT_TX_DFLEXDPMLE1 _MMIO(0x1638C0) +#define PORT_TX_DFLEXDPMLE1 _MMIO(FIA1_BASE + 0x008C0) #define DFLEXDPMLE1_DPMLETC_MASK(tc_port) (0xf << (4 * (tc_port))) #define DFLEXDPMLE1_DPMLETC_ML0(tc_port) (1 << (4 * (tc_port))) #define DFLEXDPMLE1_DPMLETC_ML1_0(tc_port) (3 << (4 * (tc_port))) @@ -2417,6 +2426,7 @@ enum i915_power_well_id { #define GEN8_GAMW_ECO_DEV_RW_IA _MMIO(0x4080) #define GAMW_ECO_ENABLE_64K_IPS_FIELD 0xF +#define GAMW_ECO_DEV_CTX_RELOAD_DISABLE (1 << 7) #define GAMT_CHKN_BIT_REG _MMIO(0x4ab8) #define GAMT_CHKN_DISABLE_L3_COH_PIPE (1 << 31) @@ -2577,6 +2587,7 @@ enum i915_power_well_id { /* chicken reg for WaConextSwitchWithConcurrentTLBInvalidate */ #define GEN9_CSFE_CHICKEN1_RCS _MMIO(0x20D4) #define GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE (1 << 2) +#define GEN11_ENABLE_32_PLANE_MODE (1 << 7) /* WaClearTdlStateAckDirtyBits */ #define GEN8_STATE_ACK _MMIO(0x20F0) @@ -2609,10 +2620,6 @@ enum i915_power_well_id { #define GEN11_GFX_DISABLE_LEGACY_MODE (1 << 3) -#define VLV_DISPLAY_BASE 0x180000 -#define VLV_MIPI_BASE VLV_DISPLAY_BASE -#define BXT_MIPI_BASE 0x60000 - #define VLV_GU_CTL0 _MMIO(VLV_DISPLAY_BASE + 0x2030) #define VLV_GU_CTL1 _MMIO(VLV_DISPLAY_BASE + 0x2034) #define SCPD0 _MMIO(0x209c) /* 915+ only */ @@ -2794,6 +2801,9 @@ enum i915_power_well_id { #define GEN6_RCS_PWR_FSM _MMIO(0x22ac) #define GEN9_RCS_FE_FSM2 _MMIO(0x22a4) +#define GEN10_CACHE_MODE_SS _MMIO(0xe420) +#define FLOAT_BLEND_OPTIMIZATION_ENABLE (1 << 4) + /* Fuse readout registers for GT */ #define HSW_PAVP_FUSE1 _MMIO(0x911C) #define HSW_F1_EU_DIS_SHIFT 16 @@ -2853,7 +2863,7 @@ enum i915_power_well_id { #define GEN11_GT_VEBOX_VDBOX_DISABLE _MMIO(0x9140) #define GEN11_GT_VDBOX_DISABLE_MASK 0xff #define GEN11_GT_VEBOX_DISABLE_SHIFT 16 -#define GEN11_GT_VEBOX_DISABLE_MASK (0xff << GEN11_GT_VEBOX_DISABLE_SHIFT) +#define GEN11_GT_VEBOX_DISABLE_MASK (0x0f << GEN11_GT_VEBOX_DISABLE_SHIFT) #define GEN11_EU_DISABLE _MMIO(0x9134) #define GEN11_EU_DIS_MASK 0xFF @@ -3169,9 +3179,9 @@ enum i915_power_well_id { /* * Clock control & power management */ -#define _DPLL_A (dev_priv->info.display_mmio_offset + 0x6014) -#define _DPLL_B (dev_priv->info.display_mmio_offset + 0x6018) -#define _CHV_DPLL_C (dev_priv->info.display_mmio_offset + 0x6030) +#define _DPLL_A (DISPLAY_MMIO_BASE(dev_priv) + 0x6014) +#define _DPLL_B (DISPLAY_MMIO_BASE(dev_priv) + 0x6018) +#define _CHV_DPLL_C (DISPLAY_MMIO_BASE(dev_priv) + 0x6030) #define DPLL(pipe) _MMIO_PIPE3((pipe), _DPLL_A, _DPLL_B, _CHV_DPLL_C) #define VGA0 _MMIO(0x6000) @@ -3268,9 +3278,9 @@ enum i915_power_well_id { #define SDVO_MULTIPLIER_SHIFT_HIRES 4 #define SDVO_MULTIPLIER_SHIFT_VGA 0 -#define _DPLL_A_MD (dev_priv->info.display_mmio_offset + 0x601c) -#define _DPLL_B_MD (dev_priv->info.display_mmio_offset + 0x6020) -#define _CHV_DPLL_C_MD (dev_priv->info.display_mmio_offset + 0x603c) +#define _DPLL_A_MD (DISPLAY_MMIO_BASE(dev_priv) + 0x601c) +#define _DPLL_B_MD (DISPLAY_MMIO_BASE(dev_priv) + 0x6020) +#define _CHV_DPLL_C_MD (DISPLAY_MMIO_BASE(dev_priv) + 0x603c) #define DPLL_MD(pipe) _MMIO_PIPE3((pipe), _DPLL_A_MD, _DPLL_B_MD, _CHV_DPLL_C_MD) /* @@ -3342,7 +3352,7 @@ enum i915_power_well_id { #define DSTATE_PLL_D3_OFF (1 << 3) #define DSTATE_GFX_CLOCK_GATING (1 << 1) #define DSTATE_DOT_CLOCK_GATING (1 << 0) -#define DSPCLK_GATE_D _MMIO(dev_priv->info.display_mmio_offset + 0x6200) +#define DSPCLK_GATE_D _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x6200) # define DPUNIT_B_CLOCK_GATE_DISABLE (1 << 30) /* 965 */ # define VSUNIT_CLOCK_GATE_DISABLE (1 << 29) /* 965 */ # define VRHUNIT_CLOCK_GATE_DISABLE (1 << 28) /* 965 */ @@ -3479,11 +3489,13 @@ enum i915_power_well_id { /* * Palette regs */ -#define PALETTE_A_OFFSET 0xa000 -#define PALETTE_B_OFFSET 0xa800 -#define CHV_PALETTE_C_OFFSET 0xc000 -#define PALETTE(pipe, i) _MMIO(dev_priv->info.palette_offsets[pipe] + \ - dev_priv->info.display_mmio_offset + (i) * 4) +#define _PALETTE_A 0xa000 +#define _PALETTE_B 0xa800 +#define _CHV_PALETTE_C 0xc000 +#define PALETTE(pipe, i) _MMIO(DISPLAY_MMIO_BASE(dev_priv) + \ + _PICK((pipe), _PALETTE_A, \ + _PALETTE_B, _CHV_PALETTE_C) + \ + (i) * 4) /* MCH MMIO space */ @@ -4065,15 +4077,27 @@ enum { #define _VSYNCSHIFT_B 0x61028 #define _PIPE_MULT_B 0x6102c +/* DSI 0 timing regs */ +#define _HTOTAL_DSI0 0x6b000 +#define _HSYNC_DSI0 0x6b008 +#define _VTOTAL_DSI0 0x6b00c +#define _VSYNC_DSI0 0x6b014 +#define _VSYNCSHIFT_DSI0 0x6b028 + +/* DSI 1 timing regs */ +#define _HTOTAL_DSI1 0x6b800 +#define _HSYNC_DSI1 0x6b808 +#define _VTOTAL_DSI1 0x6b80c +#define _VSYNC_DSI1 0x6b814 +#define _VSYNCSHIFT_DSI1 0x6b828 + #define TRANSCODER_A_OFFSET 0x60000 #define TRANSCODER_B_OFFSET 0x61000 #define TRANSCODER_C_OFFSET 0x62000 #define CHV_TRANSCODER_C_OFFSET 0x63000 #define TRANSCODER_EDP_OFFSET 0x6f000 - -#define _MMIO_TRANS2(pipe, reg) _MMIO(dev_priv->info.trans_offsets[(pipe)] - \ - dev_priv->info.trans_offsets[TRANSCODER_A] + (reg) + \ - dev_priv->info.display_mmio_offset) +#define TRANSCODER_DSI0_OFFSET 0x6b000 +#define TRANSCODER_DSI1_OFFSET 0x6b800 #define HTOTAL(trans) _MMIO_TRANS2(trans, _HTOTAL_A) #define HBLANK(trans) _MMIO_TRANS2(trans, _HBLANK_A) @@ -4153,9 +4177,13 @@ enum { /* Bspec claims those aren't shifted but stay at 0x64800 */ #define EDP_PSR_IMR _MMIO(0x64834) #define EDP_PSR_IIR _MMIO(0x64838) -#define EDP_PSR_ERROR(trans) (1 << (((trans) * 8 + 10) & 31)) -#define EDP_PSR_POST_EXIT(trans) (1 << (((trans) * 8 + 9) & 31)) -#define EDP_PSR_PRE_ENTRY(trans) (1 << (((trans) * 8 + 8) & 31)) +#define EDP_PSR_ERROR(shift) (1 << ((shift) + 2)) +#define EDP_PSR_POST_EXIT(shift) (1 << ((shift) + 1)) +#define EDP_PSR_PRE_ENTRY(shift) (1 << (shift)) +#define EDP_PSR_TRANSCODER_C_SHIFT 24 +#define EDP_PSR_TRANSCODER_B_SHIFT 16 +#define EDP_PSR_TRANSCODER_A_SHIFT 8 +#define EDP_PSR_TRANSCODER_EDP_SHIFT 0 #define EDP_PSR_AUX_CTL _MMIO(dev_priv->psr_mmio_base + 0x10) #define EDP_PSR_AUX_CTL_TIME_OUT_MASK (3 << 26) @@ -4199,7 +4227,7 @@ enum { #define EDP_PSR_DEBUG_MASK_LPSP (1 << 27) #define EDP_PSR_DEBUG_MASK_MEMUP (1 << 26) #define EDP_PSR_DEBUG_MASK_HPD (1 << 25) -#define EDP_PSR_DEBUG_MASK_DISP_REG_WRITE (1 << 16) +#define EDP_PSR_DEBUG_MASK_DISP_REG_WRITE (1 << 16) /* Reserved in ICL+ */ #define EDP_PSR_DEBUG_EXIT_ON_PIXEL_UNDERRUN (1 << 15) /* SKL+ */ #define EDP_PSR2_CTL _MMIO(0x6f900) @@ -4236,7 +4264,7 @@ enum { #define PSR_EVENT_FRONT_BUFFER_MODIFY (1 << 9) #define PSR_EVENT_WD_TIMER_EXPIRE (1 << 8) #define PSR_EVENT_PIPE_REGISTERS_UPDATE (1 << 6) -#define PSR_EVENT_REGISTER_UPDATE (1 << 5) +#define PSR_EVENT_REGISTER_UPDATE (1 << 5) /* Reserved in ICL+ */ #define PSR_EVENT_HDCP_ENABLE (1 << 4) #define PSR_EVENT_KVMR_SESSION_ENABLE (1 << 3) #define PSR_EVENT_VBI_ENABLE (1 << 2) @@ -4247,6 +4275,15 @@ enum { #define EDP_PSR2_STATUS_STATE_MASK (0xf << 28) #define EDP_PSR2_STATUS_STATE_SHIFT 28 +#define _PSR2_SU_STATUS_0 0x6F914 +#define _PSR2_SU_STATUS_1 0x6F918 +#define _PSR2_SU_STATUS_2 0x6F91C +#define _PSR2_SU_STATUS(index) _MMIO(_PICK_EVEN((index), _PSR2_SU_STATUS_0, _PSR2_SU_STATUS_1)) +#define PSR2_SU_STATUS(frame) (_PSR2_SU_STATUS((frame) / 3)) +#define PSR2_SU_STATUS_SHIFT(frame) (((frame) % 3) * 10) +#define PSR2_SU_STATUS_MASK(frame) (0x3ff << PSR2_SU_STATUS_SHIFT(frame)) +#define PSR2_SU_STATUS_FRAMES 8 + /* VGA port control */ #define ADPA _MMIO(0x61100) #define PCH_ADPA _MMIO(0xe1100) @@ -4297,7 +4334,7 @@ enum { /* Hotplug control (945+ only) */ -#define PORT_HOTPLUG_EN _MMIO(dev_priv->info.display_mmio_offset + 0x61110) +#define PORT_HOTPLUG_EN _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61110) #define PORTB_HOTPLUG_INT_EN (1 << 29) #define PORTC_HOTPLUG_INT_EN (1 << 28) #define PORTD_HOTPLUG_INT_EN (1 << 27) @@ -4327,7 +4364,7 @@ enum { #define CRT_HOTPLUG_DETECT_VOLTAGE_325MV (0 << 2) #define CRT_HOTPLUG_DETECT_VOLTAGE_475MV (1 << 2) -#define PORT_HOTPLUG_STAT _MMIO(dev_priv->info.display_mmio_offset + 0x61114) +#define PORT_HOTPLUG_STAT _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61114) /* * HDMI/DP bits are g4x+ * @@ -4409,7 +4446,7 @@ enum { #define PORT_DFT_I9XX _MMIO(0x61150) #define DC_BALANCE_RESET (1 << 25) -#define PORT_DFT2_G4X _MMIO(dev_priv->info.display_mmio_offset + 0x61154) +#define PORT_DFT2_G4X _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61154) #define DC_BALANCE_RESET_VLV (1 << 31) #define PIPE_SCRAMBLE_RESET_MASK ((1 << 14) | (0x3 << 0)) #define PIPE_C_SCRAMBLE_RESET (1 << 14) /* chv */ @@ -4569,6 +4606,7 @@ enum { * of the infoframe structure specified by CEA-861. */ #define VIDEO_DIP_DATA_SIZE 32 #define VIDEO_DIP_VSC_DATA_SIZE 36 +#define VIDEO_DIP_PPS_DATA_SIZE 132 #define VIDEO_DIP_CTL _MMIO(0x61170) /* Pre HSW: */ #define VIDEO_DIP_ENABLE (1 << 31) @@ -4588,6 +4626,15 @@ enum { #define VIDEO_DIP_FREQ_2VSYNC (2 << 16) #define VIDEO_DIP_FREQ_MASK (3 << 16) /* HSW and later: */ +#define DRM_DIP_ENABLE (1 << 28) +#define PSR_VSC_BIT_7_SET (1 << 27) +#define VSC_SELECT_MASK (0x3 << 25) +#define VSC_SELECT_SHIFT 25 +#define VSC_DIP_HW_HEA_DATA (0 << 25) +#define VSC_DIP_HW_HEA_SW_DATA (1 << 25) +#define VSC_DIP_HW_DATA_SW_HEA (2 << 25) +#define VSC_DIP_SW_HEA_DATA (3 << 25) +#define VDIP_ENABLE_PPS (1 << 24) #define VIDEO_DIP_ENABLE_VSC_HSW (1 << 20) #define VIDEO_DIP_ENABLE_GCP_HSW (1 << 16) #define VIDEO_DIP_ENABLE_AVI_HSW (1 << 12) @@ -4595,16 +4642,6 @@ enum { #define VIDEO_DIP_ENABLE_GMP_HSW (1 << 4) #define VIDEO_DIP_ENABLE_SPD_HSW (1 << 0) -#define DRM_DIP_ENABLE (1 << 28) -#define PSR_VSC_BIT_7_SET (1 << 27) -#define VSC_SELECT_MASK (0x3 << 25) -#define VSC_SELECT_SHIFT 25 -#define VSC_DIP_HW_HEA_DATA (0 << 25) -#define VSC_DIP_HW_HEA_SW_DATA (1 << 25) -#define VSC_DIP_HW_DATA_SW_HEA (2 << 25) -#define VSC_DIP_SW_HEA_DATA (3 << 25) -#define VDIP_ENABLE_PPS (1 << 24) - /* Panel power sequencing */ #define PPS_BASE 0x61200 #define VLV_PPS_BASE (VLV_DISPLAY_BASE + PPS_BASE) @@ -4617,6 +4654,17 @@ enum { #define _PP_STATUS 0x61200 #define PP_STATUS(pps_idx) _MMIO_PPS(pps_idx, _PP_STATUS) #define PP_ON (1 << 31) + +#define _PP_CONTROL_1 0xc7204 +#define _PP_CONTROL_2 0xc7304 +#define ICP_PP_CONTROL(x) _MMIO(((x) == 1) ? _PP_CONTROL_1 : \ + _PP_CONTROL_2) +#define POWER_CYCLE_DELAY_MASK (0x1f << 4) +#define POWER_CYCLE_DELAY_SHIFT 4 +#define VDD_OVERRIDE_FORCE (1 << 3) +#define BACKLIGHT_ENABLE (1 << 2) +#define PWR_DOWN_ON_RESET (1 << 1) +#define PWR_STATE_TARGET (1 << 0) /* * Indicates that all dependencies of the panel are on: * @@ -4651,7 +4699,6 @@ enum { #define EDP_FORCE_VDD (1 << 3) #define EDP_BLC_ENABLE (1 << 2) #define PANEL_POWER_RESET (1 << 1) -#define PANEL_POWER_OFF (0 << 0) #define PANEL_POWER_ON (1 << 0) #define _PP_ON_DELAYS 0x61208 @@ -4683,7 +4730,7 @@ enum { #define PANEL_POWER_CYCLE_DELAY_SHIFT 0 /* Panel fitting */ -#define PFIT_CONTROL _MMIO(dev_priv->info.display_mmio_offset + 0x61230) +#define PFIT_CONTROL _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61230) #define PFIT_ENABLE (1 << 31) #define PFIT_PIPE_MASK (3 << 29) #define PFIT_PIPE_SHIFT 29 @@ -4701,7 +4748,7 @@ enum { #define PFIT_SCALING_PROGRAMMED (1 << 26) #define PFIT_SCALING_PILLAR (2 << 26) #define PFIT_SCALING_LETTER (3 << 26) -#define PFIT_PGM_RATIOS _MMIO(dev_priv->info.display_mmio_offset + 0x61234) +#define PFIT_PGM_RATIOS _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61234) /* Pre-965 */ #define PFIT_VERT_SCALE_SHIFT 20 #define PFIT_VERT_SCALE_MASK 0xfff00000 @@ -4713,25 +4760,25 @@ enum { #define PFIT_HORIZ_SCALE_SHIFT_965 0 #define PFIT_HORIZ_SCALE_MASK_965 0x00001fff -#define PFIT_AUTO_RATIOS _MMIO(dev_priv->info.display_mmio_offset + 0x61238) +#define PFIT_AUTO_RATIOS _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61238) -#define _VLV_BLC_PWM_CTL2_A (dev_priv->info.display_mmio_offset + 0x61250) -#define _VLV_BLC_PWM_CTL2_B (dev_priv->info.display_mmio_offset + 0x61350) +#define _VLV_BLC_PWM_CTL2_A (DISPLAY_MMIO_BASE(dev_priv) + 0x61250) +#define _VLV_BLC_PWM_CTL2_B (DISPLAY_MMIO_BASE(dev_priv) + 0x61350) #define VLV_BLC_PWM_CTL2(pipe) _MMIO_PIPE(pipe, _VLV_BLC_PWM_CTL2_A, \ _VLV_BLC_PWM_CTL2_B) -#define _VLV_BLC_PWM_CTL_A (dev_priv->info.display_mmio_offset + 0x61254) -#define _VLV_BLC_PWM_CTL_B (dev_priv->info.display_mmio_offset + 0x61354) +#define _VLV_BLC_PWM_CTL_A (DISPLAY_MMIO_BASE(dev_priv) + 0x61254) +#define _VLV_BLC_PWM_CTL_B (DISPLAY_MMIO_BASE(dev_priv) + 0x61354) #define VLV_BLC_PWM_CTL(pipe) _MMIO_PIPE(pipe, _VLV_BLC_PWM_CTL_A, \ _VLV_BLC_PWM_CTL_B) -#define _VLV_BLC_HIST_CTL_A (dev_priv->info.display_mmio_offset + 0x61260) -#define _VLV_BLC_HIST_CTL_B (dev_priv->info.display_mmio_offset + 0x61360) +#define _VLV_BLC_HIST_CTL_A (DISPLAY_MMIO_BASE(dev_priv) + 0x61260) +#define _VLV_BLC_HIST_CTL_B (DISPLAY_MMIO_BASE(dev_priv) + 0x61360) #define VLV_BLC_HIST_CTL(pipe) _MMIO_PIPE(pipe, _VLV_BLC_HIST_CTL_A, \ _VLV_BLC_HIST_CTL_B) /* Backlight control */ -#define BLC_PWM_CTL2 _MMIO(dev_priv->info.display_mmio_offset + 0x61250) /* 965+ only */ +#define BLC_PWM_CTL2 _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61250) /* 965+ only */ #define BLM_PWM_ENABLE (1 << 31) #define BLM_COMBINATION_MODE (1 << 30) /* gen4 only */ #define BLM_PIPE_SELECT (1 << 29) @@ -4754,7 +4801,7 @@ enum { #define BLM_PHASE_IN_COUNT_MASK (0xff << 8) #define BLM_PHASE_IN_INCR_SHIFT (0) #define BLM_PHASE_IN_INCR_MASK (0xff << 0) -#define BLC_PWM_CTL _MMIO(dev_priv->info.display_mmio_offset + 0x61254) +#define BLC_PWM_CTL _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61254) /* * This is the most significant 15 bits of the number of backlight cycles in a * complete cycle of the modulated backlight control. @@ -4776,7 +4823,7 @@ enum { #define BACKLIGHT_DUTY_CYCLE_MASK_PNV (0xfffe) #define BLM_POLARITY_PNV (1 << 0) /* pnv only */ -#define BLC_HIST_CTL _MMIO(dev_priv->info.display_mmio_offset + 0x61260) +#define BLC_HIST_CTL _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61260) #define BLM_HISTOGRAM_ENABLE (1 << 31) /* New registers for PCH-split platforms. Safe where new bits show up, the @@ -4851,6 +4898,7 @@ enum { # define TV_OVERSAMPLE_NONE (2 << 18) /* Selects 8x oversampling */ # define TV_OVERSAMPLE_8X (3 << 18) +# define TV_OVERSAMPLE_MASK (3 << 18) /* Selects progressive mode rather than interlaced */ # define TV_PROGRESSIVE (1 << 17) /* Sets the colorburst to PAL mode. Required for non-M PAL modes. */ @@ -5400,47 +5448,47 @@ enum { * is 20 bytes in each direction, hence the 5 fixed * data registers */ -#define _DPA_AUX_CH_CTL (dev_priv->info.display_mmio_offset + 0x64010) -#define _DPA_AUX_CH_DATA1 (dev_priv->info.display_mmio_offset + 0x64014) -#define _DPA_AUX_CH_DATA2 (dev_priv->info.display_mmio_offset + 0x64018) -#define _DPA_AUX_CH_DATA3 (dev_priv->info.display_mmio_offset + 0x6401c) -#define _DPA_AUX_CH_DATA4 (dev_priv->info.display_mmio_offset + 0x64020) -#define _DPA_AUX_CH_DATA5 (dev_priv->info.display_mmio_offset + 0x64024) - -#define _DPB_AUX_CH_CTL (dev_priv->info.display_mmio_offset + 0x64110) -#define _DPB_AUX_CH_DATA1 (dev_priv->info.display_mmio_offset + 0x64114) -#define _DPB_AUX_CH_DATA2 (dev_priv->info.display_mmio_offset + 0x64118) -#define _DPB_AUX_CH_DATA3 (dev_priv->info.display_mmio_offset + 0x6411c) -#define _DPB_AUX_CH_DATA4 (dev_priv->info.display_mmio_offset + 0x64120) -#define _DPB_AUX_CH_DATA5 (dev_priv->info.display_mmio_offset + 0x64124) - -#define _DPC_AUX_CH_CTL (dev_priv->info.display_mmio_offset + 0x64210) -#define _DPC_AUX_CH_DATA1 (dev_priv->info.display_mmio_offset + 0x64214) -#define _DPC_AUX_CH_DATA2 (dev_priv->info.display_mmio_offset + 0x64218) -#define _DPC_AUX_CH_DATA3 (dev_priv->info.display_mmio_offset + 0x6421c) -#define _DPC_AUX_CH_DATA4 (dev_priv->info.display_mmio_offset + 0x64220) -#define _DPC_AUX_CH_DATA5 (dev_priv->info.display_mmio_offset + 0x64224) - -#define _DPD_AUX_CH_CTL (dev_priv->info.display_mmio_offset + 0x64310) -#define _DPD_AUX_CH_DATA1 (dev_priv->info.display_mmio_offset + 0x64314) -#define _DPD_AUX_CH_DATA2 (dev_priv->info.display_mmio_offset + 0x64318) -#define _DPD_AUX_CH_DATA3 (dev_priv->info.display_mmio_offset + 0x6431c) -#define _DPD_AUX_CH_DATA4 (dev_priv->info.display_mmio_offset + 0x64320) -#define _DPD_AUX_CH_DATA5 (dev_priv->info.display_mmio_offset + 0x64324) - -#define _DPE_AUX_CH_CTL (dev_priv->info.display_mmio_offset + 0x64410) -#define _DPE_AUX_CH_DATA1 (dev_priv->info.display_mmio_offset + 0x64414) -#define _DPE_AUX_CH_DATA2 (dev_priv->info.display_mmio_offset + 0x64418) -#define _DPE_AUX_CH_DATA3 (dev_priv->info.display_mmio_offset + 0x6441c) -#define _DPE_AUX_CH_DATA4 (dev_priv->info.display_mmio_offset + 0x64420) -#define _DPE_AUX_CH_DATA5 (dev_priv->info.display_mmio_offset + 0x64424) - -#define _DPF_AUX_CH_CTL (dev_priv->info.display_mmio_offset + 0x64510) -#define _DPF_AUX_CH_DATA1 (dev_priv->info.display_mmio_offset + 0x64514) -#define _DPF_AUX_CH_DATA2 (dev_priv->info.display_mmio_offset + 0x64518) -#define _DPF_AUX_CH_DATA3 (dev_priv->info.display_mmio_offset + 0x6451c) -#define _DPF_AUX_CH_DATA4 (dev_priv->info.display_mmio_offset + 0x64520) -#define _DPF_AUX_CH_DATA5 (dev_priv->info.display_mmio_offset + 0x64524) +#define _DPA_AUX_CH_CTL (DISPLAY_MMIO_BASE(dev_priv) + 0x64010) +#define _DPA_AUX_CH_DATA1 (DISPLAY_MMIO_BASE(dev_priv) + 0x64014) +#define _DPA_AUX_CH_DATA2 (DISPLAY_MMIO_BASE(dev_priv) + 0x64018) +#define _DPA_AUX_CH_DATA3 (DISPLAY_MMIO_BASE(dev_priv) + 0x6401c) +#define _DPA_AUX_CH_DATA4 (DISPLAY_MMIO_BASE(dev_priv) + 0x64020) +#define _DPA_AUX_CH_DATA5 (DISPLAY_MMIO_BASE(dev_priv) + 0x64024) + +#define _DPB_AUX_CH_CTL (DISPLAY_MMIO_BASE(dev_priv) + 0x64110) +#define _DPB_AUX_CH_DATA1 (DISPLAY_MMIO_BASE(dev_priv) + 0x64114) +#define _DPB_AUX_CH_DATA2 (DISPLAY_MMIO_BASE(dev_priv) + 0x64118) +#define _DPB_AUX_CH_DATA3 (DISPLAY_MMIO_BASE(dev_priv) + 0x6411c) +#define _DPB_AUX_CH_DATA4 (DISPLAY_MMIO_BASE(dev_priv) + 0x64120) +#define _DPB_AUX_CH_DATA5 (DISPLAY_MMIO_BASE(dev_priv) + 0x64124) + +#define _DPC_AUX_CH_CTL (DISPLAY_MMIO_BASE(dev_priv) + 0x64210) +#define _DPC_AUX_CH_DATA1 (DISPLAY_MMIO_BASE(dev_priv) + 0x64214) +#define _DPC_AUX_CH_DATA2 (DISPLAY_MMIO_BASE(dev_priv) + 0x64218) +#define _DPC_AUX_CH_DATA3 (DISPLAY_MMIO_BASE(dev_priv) + 0x6421c) +#define _DPC_AUX_CH_DATA4 (DISPLAY_MMIO_BASE(dev_priv) + 0x64220) +#define _DPC_AUX_CH_DATA5 (DISPLAY_MMIO_BASE(dev_priv) + 0x64224) + +#define _DPD_AUX_CH_CTL (DISPLAY_MMIO_BASE(dev_priv) + 0x64310) +#define _DPD_AUX_CH_DATA1 (DISPLAY_MMIO_BASE(dev_priv) + 0x64314) +#define _DPD_AUX_CH_DATA2 (DISPLAY_MMIO_BASE(dev_priv) + 0x64318) +#define _DPD_AUX_CH_DATA3 (DISPLAY_MMIO_BASE(dev_priv) + 0x6431c) +#define _DPD_AUX_CH_DATA4 (DISPLAY_MMIO_BASE(dev_priv) + 0x64320) +#define _DPD_AUX_CH_DATA5 (DISPLAY_MMIO_BASE(dev_priv) + 0x64324) + +#define _DPE_AUX_CH_CTL (DISPLAY_MMIO_BASE(dev_priv) + 0x64410) +#define _DPE_AUX_CH_DATA1 (DISPLAY_MMIO_BASE(dev_priv) + 0x64414) +#define _DPE_AUX_CH_DATA2 (DISPLAY_MMIO_BASE(dev_priv) + 0x64418) +#define _DPE_AUX_CH_DATA3 (DISPLAY_MMIO_BASE(dev_priv) + 0x6441c) +#define _DPE_AUX_CH_DATA4 (DISPLAY_MMIO_BASE(dev_priv) + 0x64420) +#define _DPE_AUX_CH_DATA5 (DISPLAY_MMIO_BASE(dev_priv) + 0x64424) + +#define _DPF_AUX_CH_CTL (DISPLAY_MMIO_BASE(dev_priv) + 0x64510) +#define _DPF_AUX_CH_DATA1 (DISPLAY_MMIO_BASE(dev_priv) + 0x64514) +#define _DPF_AUX_CH_DATA2 (DISPLAY_MMIO_BASE(dev_priv) + 0x64518) +#define _DPF_AUX_CH_DATA3 (DISPLAY_MMIO_BASE(dev_priv) + 0x6451c) +#define _DPF_AUX_CH_DATA4 (DISPLAY_MMIO_BASE(dev_priv) + 0x64520) +#define _DPF_AUX_CH_DATA5 (DISPLAY_MMIO_BASE(dev_priv) + 0x64524) #define DP_AUX_CH_CTL(aux_ch) _MMIO_PORT(aux_ch, _DPA_AUX_CH_CTL, _DPB_AUX_CH_CTL) #define DP_AUX_CH_DATA(aux_ch, i) _MMIO(_PORT(aux_ch, _DPA_AUX_CH_DATA1, _DPB_AUX_CH_DATA1) + (i) * 4) /* 5 registers */ @@ -5640,9 +5688,9 @@ enum { */ #define PIPE_EDP_OFFSET 0x7f000 -#define _MMIO_PIPE2(pipe, reg) _MMIO(dev_priv->info.pipe_offsets[pipe] - \ - dev_priv->info.pipe_offsets[PIPE_A] + (reg) + \ - dev_priv->info.display_mmio_offset) +/* ICL DSI 0 and 1 */ +#define PIPE_DSI0_OFFSET 0x7b000 +#define PIPE_DSI1_OFFSET 0x7b800 #define PIPECONF(pipe) _MMIO_PIPE2(pipe, _PIPEACONF) #define PIPEDSL(pipe) _MMIO_PIPE2(pipe, _PIPEADSL) @@ -5665,6 +5713,12 @@ enum { #define PIPEMISC_DITHER_TYPE_SP (0 << 2) #define PIPEMISC(pipe) _MMIO_PIPE2(pipe, _PIPE_MISC_A) +/* Skylake+ pipe bottom (background) color */ +#define _SKL_BOTTOM_COLOR_A 0x70034 +#define SKL_BOTTOM_COLOR_GAMMA_ENABLE (1 << 31) +#define SKL_BOTTOM_COLOR_CSC_ENABLE (1 << 30) +#define SKL_BOTTOM_COLOR(pipe) _MMIO_PIPE2(pipe, _SKL_BOTTOM_COLOR_A) + #define VLV_DPFLIPSTAT _MMIO(VLV_DISPLAY_BASE + 0x70028) #define PIPEB_LINE_COMPARE_INT_EN (1 << 29) #define PIPEB_HLINE_INT_EN (1 << 28) @@ -5716,7 +5770,7 @@ enum { #define DPINVGTT_STATUS_MASK 0xff #define DPINVGTT_STATUS_MASK_CHV 0xfff -#define DSPARB _MMIO(dev_priv->info.display_mmio_offset + 0x70030) +#define DSPARB _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x70030) #define DSPARB_CSTART_MASK (0x7f << 7) #define DSPARB_CSTART_SHIFT 7 #define DSPARB_BSTART_MASK (0x7f) @@ -5751,7 +5805,7 @@ enum { #define DSPARB_SPRITEF_MASK_VLV (0xff << 8) /* pnv/gen4/g4x/vlv/chv */ -#define DSPFW1 _MMIO(dev_priv->info.display_mmio_offset + 0x70034) +#define DSPFW1 _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x70034) #define DSPFW_SR_SHIFT 23 #define DSPFW_SR_MASK (0x1ff << 23) #define DSPFW_CURSORB_SHIFT 16 @@ -5762,7 +5816,7 @@ enum { #define DSPFW_PLANEA_SHIFT 0 #define DSPFW_PLANEA_MASK (0x7f << 0) #define DSPFW_PLANEA_MASK_VLV (0xff << 0) /* vlv/chv */ -#define DSPFW2 _MMIO(dev_priv->info.display_mmio_offset + 0x70038) +#define DSPFW2 _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x70038) #define DSPFW_FBC_SR_EN (1 << 31) /* g4x */ #define DSPFW_FBC_SR_SHIFT 28 #define DSPFW_FBC_SR_MASK (0x7 << 28) /* g4x */ @@ -5778,7 +5832,7 @@ enum { #define DSPFW_SPRITEA_SHIFT 0 #define DSPFW_SPRITEA_MASK (0x7f << 0) /* g4x */ #define DSPFW_SPRITEA_MASK_VLV (0xff << 0) /* vlv/chv */ -#define DSPFW3 _MMIO(dev_priv->info.display_mmio_offset + 0x7003c) +#define DSPFW3 _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x7003c) #define DSPFW_HPLL_SR_EN (1 << 31) #define PINEVIEW_SELF_REFRESH_EN (1 << 30) #define DSPFW_CURSOR_SR_SHIFT 24 @@ -5946,7 +6000,7 @@ enum { #define PLANE_WM_EN (1 << 31) #define PLANE_WM_LINES_SHIFT 14 #define PLANE_WM_LINES_MASK 0x1f -#define PLANE_WM_BLOCKS_MASK 0x3ff +#define PLANE_WM_BLOCKS_MASK 0x7ff /* skl+: 10 bits, icl+ 11 bits */ #define _CUR_WM_0(pipe) _PIPE(pipe, _CUR_WM_A_0, _CUR_WM_B_0) #define CUR_WM(pipe, level) _MMIO(_CUR_WM_0(pipe) + ((4) * (level))) @@ -6091,10 +6145,6 @@ enum { #define _CURBBASE_IVB 0x71084 #define _CURBPOS_IVB 0x71088 -#define _CURSOR2(pipe, reg) _MMIO(dev_priv->info.cursor_offsets[(pipe)] - \ - dev_priv->info.cursor_offsets[PIPE_A] + (reg) + \ - dev_priv->info.display_mmio_offset) - #define CURCNTR(pipe) _CURSOR2(pipe, _CURACNTR) #define CURBASE(pipe) _CURSOR2(pipe, _CURABASE) #define CURPOS(pipe) _CURSOR2(pipe, _CURAPOS) @@ -6198,35 +6248,39 @@ enum { * [10:1f] all * [30:32] all */ -#define SWF0(i) _MMIO(dev_priv->info.display_mmio_offset + 0x70410 + (i) * 4) -#define SWF1(i) _MMIO(dev_priv->info.display_mmio_offset + 0x71410 + (i) * 4) -#define SWF3(i) _MMIO(dev_priv->info.display_mmio_offset + 0x72414 + (i) * 4) +#define SWF0(i) _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x70410 + (i) * 4) +#define SWF1(i) _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x71410 + (i) * 4) +#define SWF3(i) _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x72414 + (i) * 4) #define SWF_ILK(i) _MMIO(0x4F000 + (i) * 4) /* Pipe B */ -#define _PIPEBDSL (dev_priv->info.display_mmio_offset + 0x71000) -#define _PIPEBCONF (dev_priv->info.display_mmio_offset + 0x71008) -#define _PIPEBSTAT (dev_priv->info.display_mmio_offset + 0x71024) +#define _PIPEBDSL (DISPLAY_MMIO_BASE(dev_priv) + 0x71000) +#define _PIPEBCONF (DISPLAY_MMIO_BASE(dev_priv) + 0x71008) +#define _PIPEBSTAT (DISPLAY_MMIO_BASE(dev_priv) + 0x71024) #define _PIPEBFRAMEHIGH 0x71040 #define _PIPEBFRAMEPIXEL 0x71044 -#define _PIPEB_FRMCOUNT_G4X (dev_priv->info.display_mmio_offset + 0x71040) -#define _PIPEB_FLIPCOUNT_G4X (dev_priv->info.display_mmio_offset + 0x71044) +#define _PIPEB_FRMCOUNT_G4X (DISPLAY_MMIO_BASE(dev_priv) + 0x71040) +#define _PIPEB_FLIPCOUNT_G4X (DISPLAY_MMIO_BASE(dev_priv) + 0x71044) /* Display B control */ -#define _DSPBCNTR (dev_priv->info.display_mmio_offset + 0x71180) +#define _DSPBCNTR (DISPLAY_MMIO_BASE(dev_priv) + 0x71180) #define DISPPLANE_ALPHA_TRANS_ENABLE (1 << 15) #define DISPPLANE_ALPHA_TRANS_DISABLE 0 #define DISPPLANE_SPRITE_ABOVE_DISPLAY 0 #define DISPPLANE_SPRITE_ABOVE_OVERLAY (1) -#define _DSPBADDR (dev_priv->info.display_mmio_offset + 0x71184) -#define _DSPBSTRIDE (dev_priv->info.display_mmio_offset + 0x71188) -#define _DSPBPOS (dev_priv->info.display_mmio_offset + 0x7118C) -#define _DSPBSIZE (dev_priv->info.display_mmio_offset + 0x71190) -#define _DSPBSURF (dev_priv->info.display_mmio_offset + 0x7119C) -#define _DSPBTILEOFF (dev_priv->info.display_mmio_offset + 0x711A4) -#define _DSPBOFFSET (dev_priv->info.display_mmio_offset + 0x711A4) -#define _DSPBSURFLIVE (dev_priv->info.display_mmio_offset + 0x711AC) +#define _DSPBADDR (DISPLAY_MMIO_BASE(dev_priv) + 0x71184) +#define _DSPBSTRIDE (DISPLAY_MMIO_BASE(dev_priv) + 0x71188) +#define _DSPBPOS (DISPLAY_MMIO_BASE(dev_priv) + 0x7118C) +#define _DSPBSIZE (DISPLAY_MMIO_BASE(dev_priv) + 0x71190) +#define _DSPBSURF (DISPLAY_MMIO_BASE(dev_priv) + 0x7119C) +#define _DSPBTILEOFF (DISPLAY_MMIO_BASE(dev_priv) + 0x711A4) +#define _DSPBOFFSET (DISPLAY_MMIO_BASE(dev_priv) + 0x711A4) +#define _DSPBSURFLIVE (DISPLAY_MMIO_BASE(dev_priv) + 0x711AC) + +/* ICL DSI 0 and 1 */ +#define _PIPEDSI0CONF 0x7b008 +#define _PIPEDSI1CONF 0x7b808 /* Sprite A control */ #define _DVSACNTR 0x72180 @@ -6515,6 +6569,7 @@ enum { #define PLANE_CTL_KEY_ENABLE_DESTINATION (2 << 21) #define PLANE_CTL_ORDER_BGRX (0 << 20) #define PLANE_CTL_ORDER_RGBX (1 << 20) +#define PLANE_CTL_YUV420_Y_PLANE (1 << 19) #define PLANE_CTL_YUV_TO_RGB_CSC_FORMAT_BT709 (1 << 18) #define PLANE_CTL_YUV422_ORDER_MASK (0x3 << 16) #define PLANE_CTL_YUV422_YUYV (0 << 16) @@ -6558,17 +6613,33 @@ enum { #define _PLANE_KEYVAL_2_A 0x70294 #define _PLANE_KEYMSK_1_A 0x70198 #define _PLANE_KEYMSK_2_A 0x70298 +#define PLANE_KEYMSK_ALPHA_ENABLE (1 << 31) #define _PLANE_KEYMAX_1_A 0x701a0 #define _PLANE_KEYMAX_2_A 0x702a0 +#define PLANE_KEYMAX_ALPHA(a) ((a) << 24) #define _PLANE_AUX_DIST_1_A 0x701c0 #define _PLANE_AUX_DIST_2_A 0x702c0 #define _PLANE_AUX_OFFSET_1_A 0x701c4 #define _PLANE_AUX_OFFSET_2_A 0x702c4 +#define _PLANE_CUS_CTL_1_A 0x701c8 +#define _PLANE_CUS_CTL_2_A 0x702c8 +#define PLANE_CUS_ENABLE (1 << 31) +#define PLANE_CUS_PLANE_6 (0 << 30) +#define PLANE_CUS_PLANE_7 (1 << 30) +#define PLANE_CUS_HPHASE_SIGN_NEGATIVE (1 << 19) +#define PLANE_CUS_HPHASE_0 (0 << 16) +#define PLANE_CUS_HPHASE_0_25 (1 << 16) +#define PLANE_CUS_HPHASE_0_5 (2 << 16) +#define PLANE_CUS_VPHASE_SIGN_NEGATIVE (1 << 15) +#define PLANE_CUS_VPHASE_0 (0 << 12) +#define PLANE_CUS_VPHASE_0_25 (1 << 12) +#define PLANE_CUS_VPHASE_0_5 (2 << 12) #define _PLANE_COLOR_CTL_1_A 0x701CC /* GLK+ */ #define _PLANE_COLOR_CTL_2_A 0x702CC /* GLK+ */ #define _PLANE_COLOR_CTL_3_A 0x703CC /* GLK+ */ #define PLANE_COLOR_PIPE_GAMMA_ENABLE (1 << 30) /* Pre-ICL */ #define PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE (1 << 28) +#define PLANE_COLOR_INPUT_CSC_ENABLE (1 << 20) /* ICL+ */ #define PLANE_COLOR_PIPE_CSC_ENABLE (1 << 23) /* Pre-ICL */ #define PLANE_COLOR_CSC_MODE_BYPASS (0 << 17) #define PLANE_COLOR_CSC_MODE_YUV601_TO_RGB709 (1 << 17) @@ -6585,6 +6656,55 @@ enum { #define _PLANE_NV12_BUF_CFG_1_A 0x70278 #define _PLANE_NV12_BUF_CFG_2_A 0x70378 +/* Input CSC Register Definitions */ +#define _PLANE_INPUT_CSC_RY_GY_1_A 0x701E0 +#define _PLANE_INPUT_CSC_RY_GY_2_A 0x702E0 + +#define _PLANE_INPUT_CSC_RY_GY_1_B 0x711E0 +#define _PLANE_INPUT_CSC_RY_GY_2_B 0x712E0 + +#define _PLANE_INPUT_CSC_RY_GY_1(pipe) \ + _PIPE(pipe, _PLANE_INPUT_CSC_RY_GY_1_A, \ + _PLANE_INPUT_CSC_RY_GY_1_B) +#define _PLANE_INPUT_CSC_RY_GY_2(pipe) \ + _PIPE(pipe, _PLANE_INPUT_CSC_RY_GY_2_A, \ + _PLANE_INPUT_CSC_RY_GY_2_B) + +#define PLANE_INPUT_CSC_COEFF(pipe, plane, index) \ + _MMIO_PLANE(plane, _PLANE_INPUT_CSC_RY_GY_1(pipe) + (index) * 4, \ + _PLANE_INPUT_CSC_RY_GY_2(pipe) + (index) * 4) + +#define _PLANE_INPUT_CSC_PREOFF_HI_1_A 0x701F8 +#define _PLANE_INPUT_CSC_PREOFF_HI_2_A 0x702F8 + +#define _PLANE_INPUT_CSC_PREOFF_HI_1_B 0x711F8 +#define _PLANE_INPUT_CSC_PREOFF_HI_2_B 0x712F8 + +#define _PLANE_INPUT_CSC_PREOFF_HI_1(pipe) \ + _PIPE(pipe, _PLANE_INPUT_CSC_PREOFF_HI_1_A, \ + _PLANE_INPUT_CSC_PREOFF_HI_1_B) +#define _PLANE_INPUT_CSC_PREOFF_HI_2(pipe) \ + _PIPE(pipe, _PLANE_INPUT_CSC_PREOFF_HI_2_A, \ + _PLANE_INPUT_CSC_PREOFF_HI_2_B) +#define PLANE_INPUT_CSC_PREOFF(pipe, plane, index) \ + _MMIO_PLANE(plane, _PLANE_INPUT_CSC_PREOFF_HI_1(pipe) + (index) * 4, \ + _PLANE_INPUT_CSC_PREOFF_HI_2(pipe) + (index) * 4) + +#define _PLANE_INPUT_CSC_POSTOFF_HI_1_A 0x70204 +#define _PLANE_INPUT_CSC_POSTOFF_HI_2_A 0x70304 + +#define _PLANE_INPUT_CSC_POSTOFF_HI_1_B 0x71204 +#define _PLANE_INPUT_CSC_POSTOFF_HI_2_B 0x71304 + +#define _PLANE_INPUT_CSC_POSTOFF_HI_1(pipe) \ + _PIPE(pipe, _PLANE_INPUT_CSC_POSTOFF_HI_1_A, \ + _PLANE_INPUT_CSC_POSTOFF_HI_1_B) +#define _PLANE_INPUT_CSC_POSTOFF_HI_2(pipe) \ + _PIPE(pipe, _PLANE_INPUT_CSC_POSTOFF_HI_2_A, \ + _PLANE_INPUT_CSC_POSTOFF_HI_2_B) +#define PLANE_INPUT_CSC_POSTOFF(pipe, plane, index) \ + _MMIO_PLANE(plane, _PLANE_INPUT_CSC_POSTOFF_HI_1(pipe) + (index) * 4, \ + _PLANE_INPUT_CSC_POSTOFF_HI_2(pipe) + (index) * 4) #define _PLANE_CTL_1_B 0x71180 #define _PLANE_CTL_2_B 0x71280 @@ -6664,8 +6784,7 @@ enum { #define _PLANE_BUF_CFG_1_B 0x7127c #define _PLANE_BUF_CFG_2_B 0x7137c -#define SKL_DDB_ENTRY_MASK 0x3FF -#define ICL_DDB_ENTRY_MASK 0x7FF +#define DDB_ENTRY_MASK 0x7FF /* skl+: 10 bits, icl+ 11 bits */ #define DDB_ENTRY_END_SHIFT 16 #define _PLANE_BUF_CFG_1(pipe) \ _PIPE(pipe, _PLANE_BUF_CFG_1_A, _PLANE_BUF_CFG_1_B) @@ -6701,6 +6820,15 @@ enum { #define PLANE_AUX_OFFSET(pipe, plane) \ _MMIO_PLANE(plane, _PLANE_AUX_OFFSET_1(pipe), _PLANE_AUX_OFFSET_2(pipe)) +#define _PLANE_CUS_CTL_1_B 0x711c8 +#define _PLANE_CUS_CTL_2_B 0x712c8 +#define _PLANE_CUS_CTL_1(pipe) \ + _PIPE(pipe, _PLANE_CUS_CTL_1_A, _PLANE_CUS_CTL_1_B) +#define _PLANE_CUS_CTL_2(pipe) \ + _PIPE(pipe, _PLANE_CUS_CTL_2_A, _PLANE_CUS_CTL_2_B) +#define PLANE_CUS_CTL(pipe, plane) \ + _MMIO_PLANE(plane, _PLANE_CUS_CTL_1(pipe), _PLANE_CUS_CTL_2(pipe)) + #define _PLANE_COLOR_CTL_1_B 0x711CC #define _PLANE_COLOR_CTL_2_B 0x712CC #define _PLANE_COLOR_CTL_3_B 0x713CC @@ -6854,11 +6982,12 @@ enum { #define _PS_2B_CTRL 0x68A80 #define _PS_1C_CTRL 0x69180 #define PS_SCALER_EN (1 << 31) -#define PS_SCALER_MODE_MASK (3 << 28) -#define PS_SCALER_MODE_DYN (0 << 28) -#define PS_SCALER_MODE_HQ (1 << 28) +#define SKL_PS_SCALER_MODE_MASK (3 << 28) +#define SKL_PS_SCALER_MODE_DYN (0 << 28) +#define SKL_PS_SCALER_MODE_HQ (1 << 28) #define SKL_PS_SCALER_MODE_NV12 (2 << 28) #define PS_SCALER_MODE_PLANAR (1 << 29) +#define PS_SCALER_MODE_NORMAL (0 << 29) #define PS_PLANE_SEL_MASK (7 << 25) #define PS_PLANE_SEL(plane) (((plane) + 1) << 25) #define PS_FILTER_MASK (3 << 23) @@ -6875,6 +7004,8 @@ enum { #define PS_VADAPT_MODE_LEAST_ADAPT (0 << 5) #define PS_VADAPT_MODE_MOD_ADAPT (1 << 5) #define PS_VADAPT_MODE_MOST_ADAPT (3 << 5) +#define PS_PLANE_Y_SEL_MASK (7 << 5) +#define PS_PLANE_Y_SEL(plane) (((plane) + 1) << 5) #define _PS_PWR_GATE_1A 0x68160 #define _PS_PWR_GATE_2A 0x68260 @@ -7321,9 +7452,10 @@ enum { #define BDW_DPRS_MASK_VBLANK_SRD (1 << 0) #define CHICKEN_PIPESL_1(pipe) _MMIO_PIPE(pipe, _CHICKEN_PIPESL_1_A, _CHICKEN_PIPESL_1_B) -#define CHICKEN_TRANS_A 0x420c0 -#define CHICKEN_TRANS_B 0x420c4 -#define CHICKEN_TRANS(trans) _MMIO_TRANS(trans, CHICKEN_TRANS_A, CHICKEN_TRANS_B) +#define CHICKEN_TRANS_A _MMIO(0x420c0) +#define CHICKEN_TRANS_B _MMIO(0x420c4) +#define CHICKEN_TRANS_C _MMIO(0x420c8) +#define CHICKEN_TRANS_EDP _MMIO(0x420cc) #define VSC_DATA_SEL_SOFTWARE_CONTROL (1 << 25) /* GLK and CNL+ */ #define DDI_TRAINING_OVERRIDE_ENABLE (1 << 19) #define DDI_TRAINING_OVERRIDE_VALUE (1 << 18) @@ -7413,6 +7545,10 @@ enum { #define GEN9_SLICE_COMMON_ECO_CHICKEN1 _MMIO(0x731c) #define GEN11_STATE_CACHE_REDIRECT_TO_CS (1 << 11) +#define GEN7_SARCHKMD _MMIO(0xB000) +#define GEN7_DISABLE_DEMAND_PREFETCH (1 << 31) +#define GEN7_DISABLE_SAMPLER_PREFETCH (1 << 30) + #define GEN7_L3SQCREG1 _MMIO(0xB010) #define VLV_B0_WA_L3SQCREG1_VALUE 0x00D30000 @@ -7481,6 +7617,7 @@ enum { #define _PIPEB_CHICKEN 0x71038 #define _PIPEC_CHICKEN 0x72038 #define PER_PIXEL_ALPHA_BYPASS_EN (1 << 7) +#define PM_FILL_MAINTAIN_DBUF_FULLNESS (1 << 0) #define PIPE_CHICKEN(pipe) _MMIO_PIPE(pipe, _PIPEA_CHICKEN,\ _PIPEB_CHICKEN) @@ -7667,6 +7804,7 @@ enum { #define ICP_DDIB_HPD_LONG_DETECT (2 << 4) #define ICP_DDIB_HPD_SHORT_LONG_DETECT (3 << 4) #define ICP_DDIA_HPD_ENABLE (1 << 3) +#define ICP_DDIA_HPD_OP_DRIVE_1 (1 << 2) #define ICP_DDIA_HPD_STATUS_MASK (3 << 0) #define ICP_DDIA_HPD_NO_DETECT (0 << 0) #define ICP_DDIA_HPD_SHORT_DETECT (1 << 0) @@ -7828,8 +7966,7 @@ enum { #define CNP_RAWCLK_DIV_MASK (0x3ff << 16) #define CNP_RAWCLK_DIV(div) ((div) << 16) #define CNP_RAWCLK_FRAC_MASK (0xf << 26) -#define CNP_RAWCLK_FRAC(frac) ((frac) << 26) -#define ICP_RAWCLK_DEN(den) ((den) << 26) +#define CNP_RAWCLK_DEN(den) ((den) << 26) #define ICP_RAWCLK_NUM(num) ((num) << 11) #define PCH_DPLL_TMR_CFG _MMIO(0xc6208) @@ -8629,8 +8766,7 @@ enum { #define GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC (1 << 9) #define GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC (1 << 7) -#define GAMW_ECO_DEV_RW_IA_REG _MMIO(0x4080) -#define GAMW_ECO_DEV_CTX_RELOAD_DISABLE (1 << 7) +#define GEN10_SAMPLER_MODE _MMIO(0xE18C) /* IVYBRIDGE DPF */ #define GEN7_L3CDERRST1(slice) _MMIO(0xB008 + (slice) * 0x200) /* L3CD Error Status 1 */ @@ -8692,7 +8828,7 @@ enum { #define GEN9_ENABLE_GPGPU_PREEMPTION (1 << 2) /* Audio */ -#define G4X_AUD_VID_DID _MMIO(dev_priv->info.display_mmio_offset + 0x62020) +#define G4X_AUD_VID_DID _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x62020) #define INTEL_AUDIO_DEVCL 0x808629FB #define INTEL_AUDIO_DEVBLC 0x80862801 #define INTEL_AUDIO_DEVCTG 0x80862802 @@ -8931,6 +9067,15 @@ enum skl_power_gate { #define CNL_AUX_ANAOVRD1_ENABLE (1 << 16) #define CNL_AUX_ANAOVRD1_LDO_BYPASS (1 << 23) +#define _ICL_AUX_REG_IDX(pw_idx) ((pw_idx) - ICL_PW_CTL_IDX_AUX_A) +#define _ICL_AUX_ANAOVRD1_A 0x162398 +#define _ICL_AUX_ANAOVRD1_B 0x6C398 +#define ICL_AUX_ANAOVRD1(pw_idx) _MMIO(_PICK(_ICL_AUX_REG_IDX(pw_idx), \ + _ICL_AUX_ANAOVRD1_A, \ + _ICL_AUX_ANAOVRD1_B)) +#define ICL_AUX_ANAOVRD1_LDO_BYPASS (1 << 7) +#define ICL_AUX_ANAOVRD1_ENABLE (1 << 0) + /* HDCP Key Registers */ #define HDCP_KEY_CONF _MMIO(0x66c00) #define HDCP_AKSV_SEND_TRIGGER BIT(31) @@ -9013,11 +9158,45 @@ enum skl_power_gate { #define HDCP_STATUS_CIPHER BIT(16) #define HDCP_STATUS_FRAME_CNT(x) (((x) >> 8) & 0xff) +/* HDCP2.2 Registers */ +#define _PORTA_HDCP2_BASE 0x66800 +#define _PORTB_HDCP2_BASE 0x66500 +#define _PORTC_HDCP2_BASE 0x66600 +#define _PORTD_HDCP2_BASE 0x66700 +#define _PORTE_HDCP2_BASE 0x66A00 +#define _PORTF_HDCP2_BASE 0x66900 +#define _PORT_HDCP2_BASE(port, x) _MMIO(_PICK((port), \ + _PORTA_HDCP2_BASE, \ + _PORTB_HDCP2_BASE, \ + _PORTC_HDCP2_BASE, \ + _PORTD_HDCP2_BASE, \ + _PORTE_HDCP2_BASE, \ + _PORTF_HDCP2_BASE) + (x)) + +#define HDCP2_AUTH_DDI(port) _PORT_HDCP2_BASE(port, 0x98) +#define AUTH_LINK_AUTHENTICATED BIT(31) +#define AUTH_LINK_TYPE BIT(30) +#define AUTH_FORCE_CLR_INPUTCTR BIT(19) +#define AUTH_CLR_KEYS BIT(18) + +#define HDCP2_CTL_DDI(port) _PORT_HDCP2_BASE(port, 0xB0) +#define CTL_LINK_ENCRYPTION_REQ BIT(31) + +#define HDCP2_STATUS_DDI(port) _PORT_HDCP2_BASE(port, 0xB4) +#define STREAM_ENCRYPTION_STATUS_A BIT(31) +#define STREAM_ENCRYPTION_STATUS_B BIT(30) +#define STREAM_ENCRYPTION_STATUS_C BIT(29) +#define LINK_TYPE_STATUS BIT(22) +#define LINK_AUTH_STATUS BIT(21) +#define LINK_ENCRYPTION_STATUS BIT(20) + /* Per-pipe DDI Function Control */ #define _TRANS_DDI_FUNC_CTL_A 0x60400 #define _TRANS_DDI_FUNC_CTL_B 0x61400 #define _TRANS_DDI_FUNC_CTL_C 0x62400 #define _TRANS_DDI_FUNC_CTL_EDP 0x6F400 +#define _TRANS_DDI_FUNC_CTL_DSI0 0x6b400 +#define _TRANS_DDI_FUNC_CTL_DSI1 0x6bc00 #define TRANS_DDI_FUNC_CTL(tran) _MMIO_TRANS2(tran, _TRANS_DDI_FUNC_CTL_A) #define TRANS_DDI_FUNC_ENABLE (1 << 31) @@ -9055,11 +9234,25 @@ enum skl_power_gate { | TRANS_DDI_HDMI_SCRAMBLER_RESET_FREQ \ | TRANS_DDI_HDMI_SCRAMBLING) +#define _TRANS_DDI_FUNC_CTL2_A 0x60404 +#define _TRANS_DDI_FUNC_CTL2_B 0x61404 +#define _TRANS_DDI_FUNC_CTL2_C 0x62404 +#define _TRANS_DDI_FUNC_CTL2_EDP 0x6f404 +#define _TRANS_DDI_FUNC_CTL2_DSI0 0x6b404 +#define _TRANS_DDI_FUNC_CTL2_DSI1 0x6bc04 +#define TRANS_DDI_FUNC_CTL2(tran) _MMIO_TRANS2(tran, \ + _TRANS_DDI_FUNC_CTL2_A) +#define PORT_SYNC_MODE_ENABLE (1 << 4) +#define PORT_SYNC_MODE_MASTER_SELECT(x) ((x) << 0) +#define PORT_SYNC_MODE_MASTER_SELECT_MASK (0x7 << 0) +#define PORT_SYNC_MODE_MASTER_SELECT_SHIFT 0 + /* DisplayPort Transport Control */ #define _DP_TP_CTL_A 0x64040 #define _DP_TP_CTL_B 0x64140 #define DP_TP_CTL(port) _MMIO_PORT(port, _DP_TP_CTL_A, _DP_TP_CTL_B) #define DP_TP_CTL_ENABLE (1 << 31) +#define DP_TP_CTL_FEC_ENABLE (1 << 30) #define DP_TP_CTL_MODE_SST (0 << 27) #define DP_TP_CTL_MODE_MST (1 << 27) #define DP_TP_CTL_FORCE_ACT (1 << 25) @@ -9078,6 +9271,7 @@ enum skl_power_gate { #define _DP_TP_STATUS_A 0x64044 #define _DP_TP_STATUS_B 0x64144 #define DP_TP_STATUS(port) _MMIO_PORT(port, _DP_TP_STATUS_A, _DP_TP_STATUS_B) +#define DP_TP_STATUS_FEC_ENABLE_LIVE (1 << 28) #define DP_TP_STATUS_IDLE_DONE (1 << 25) #define DP_TP_STATUS_ACT_SENT (1 << 24) #define DP_TP_STATUS_MODE_STATUS_MST (1 << 23) @@ -9226,6 +9420,8 @@ enum skl_power_gate { #define TRANS_MSA_MISC(tran) _MMIO_TRANS2(tran, _TRANSA_MSA_MISC) #define TRANS_MSA_SYNC_CLK (1 << 0) +#define TRANS_MSA_SAMPLING_444 (2 << 1) +#define TRANS_MSA_CLRSP_YCBCR (2 << 3) #define TRANS_MSA_6_BPC (0 << 5) #define TRANS_MSA_8_BPC (1 << 5) #define TRANS_MSA_10_BPC (2 << 5) @@ -9367,7 +9563,7 @@ enum skl_power_gate { #define _MG_PLL3_ENABLE 0x46038 #define _MG_PLL4_ENABLE 0x4603C /* Bits are the same as DPLL0_ENABLE */ -#define MG_PLL_ENABLE(port) _MMIO_PORT((port) - PORT_C, _MG_PLL1_ENABLE, \ +#define MG_PLL_ENABLE(tc_port) _MMIO_PORT((tc_port), _MG_PLL1_ENABLE, \ _MG_PLL2_ENABLE) #define _MG_REFCLKIN_CTL_PORT1 0x16892C @@ -9376,9 +9572,9 @@ enum skl_power_gate { #define _MG_REFCLKIN_CTL_PORT4 0x16B92C #define MG_REFCLKIN_CTL_OD_2_MUX(x) ((x) << 8) #define MG_REFCLKIN_CTL_OD_2_MUX_MASK (0x7 << 8) -#define MG_REFCLKIN_CTL(port) _MMIO_PORT((port) - PORT_C, \ - _MG_REFCLKIN_CTL_PORT1, \ - _MG_REFCLKIN_CTL_PORT2) +#define MG_REFCLKIN_CTL(tc_port) _MMIO_PORT((tc_port), \ + _MG_REFCLKIN_CTL_PORT1, \ + _MG_REFCLKIN_CTL_PORT2) #define _MG_CLKTOP2_CORECLKCTL1_PORT1 0x1688D8 #define _MG_CLKTOP2_CORECLKCTL1_PORT2 0x1698D8 @@ -9388,9 +9584,9 @@ enum skl_power_gate { #define MG_CLKTOP2_CORECLKCTL1_B_DIVRATIO_MASK (0xff << 16) #define MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO(x) ((x) << 8) #define MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO_MASK (0xff << 8) -#define MG_CLKTOP2_CORECLKCTL1(port) _MMIO_PORT((port) - PORT_C, \ - _MG_CLKTOP2_CORECLKCTL1_PORT1, \ - _MG_CLKTOP2_CORECLKCTL1_PORT2) +#define MG_CLKTOP2_CORECLKCTL1(tc_port) _MMIO_PORT((tc_port), \ + _MG_CLKTOP2_CORECLKCTL1_PORT1, \ + _MG_CLKTOP2_CORECLKCTL1_PORT2) #define _MG_CLKTOP2_HSCLKCTL_PORT1 0x1688D4 #define _MG_CLKTOP2_HSCLKCTL_PORT2 0x1698D4 @@ -9408,9 +9604,9 @@ enum skl_power_gate { #define MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO(x) ((x) << 8) #define MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_SHIFT 8 #define MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK (0xf << 8) -#define MG_CLKTOP2_HSCLKCTL(port) _MMIO_PORT((port) - PORT_C, \ - _MG_CLKTOP2_HSCLKCTL_PORT1, \ - _MG_CLKTOP2_HSCLKCTL_PORT2) +#define MG_CLKTOP2_HSCLKCTL(tc_port) _MMIO_PORT((tc_port), \ + _MG_CLKTOP2_HSCLKCTL_PORT1, \ + _MG_CLKTOP2_HSCLKCTL_PORT2) #define _MG_PLL_DIV0_PORT1 0x168A00 #define _MG_PLL_DIV0_PORT2 0x169A00 @@ -9422,8 +9618,8 @@ enum skl_power_gate { #define MG_PLL_DIV0_FBDIV_FRAC(x) ((x) << 8) #define MG_PLL_DIV0_FBDIV_INT_MASK (0xff << 0) #define MG_PLL_DIV0_FBDIV_INT(x) ((x) << 0) -#define MG_PLL_DIV0(port) _MMIO_PORT((port) - PORT_C, _MG_PLL_DIV0_PORT1, \ - _MG_PLL_DIV0_PORT2) +#define MG_PLL_DIV0(tc_port) _MMIO_PORT((tc_port), _MG_PLL_DIV0_PORT1, \ + _MG_PLL_DIV0_PORT2) #define _MG_PLL_DIV1_PORT1 0x168A04 #define _MG_PLL_DIV1_PORT2 0x169A04 @@ -9437,8 +9633,8 @@ enum skl_power_gate { #define MG_PLL_DIV1_NDIVRATIO(x) ((x) << 4) #define MG_PLL_DIV1_FBPREDIV_MASK (0xf << 0) #define MG_PLL_DIV1_FBPREDIV(x) ((x) << 0) -#define MG_PLL_DIV1(port) _MMIO_PORT((port) - PORT_C, _MG_PLL_DIV1_PORT1, \ - _MG_PLL_DIV1_PORT2) +#define MG_PLL_DIV1(tc_port) _MMIO_PORT((tc_port), _MG_PLL_DIV1_PORT1, \ + _MG_PLL_DIV1_PORT2) #define _MG_PLL_LF_PORT1 0x168A08 #define _MG_PLL_LF_PORT2 0x169A08 @@ -9450,8 +9646,8 @@ enum skl_power_gate { #define MG_PLL_LF_GAINCTRL(x) ((x) << 16) #define MG_PLL_LF_INT_COEFF(x) ((x) << 8) #define MG_PLL_LF_PROP_COEFF(x) ((x) << 0) -#define MG_PLL_LF(port) _MMIO_PORT((port) - PORT_C, _MG_PLL_LF_PORT1, \ - _MG_PLL_LF_PORT2) +#define MG_PLL_LF(tc_port) _MMIO_PORT((tc_port), _MG_PLL_LF_PORT1, \ + _MG_PLL_LF_PORT2) #define _MG_PLL_FRAC_LOCK_PORT1 0x168A0C #define _MG_PLL_FRAC_LOCK_PORT2 0x169A0C @@ -9463,9 +9659,9 @@ enum skl_power_gate { #define MG_PLL_FRAC_LOCK_DCODITHEREN (1 << 10) #define MG_PLL_FRAC_LOCK_FEEDFWRDCAL_EN (1 << 8) #define MG_PLL_FRAC_LOCK_FEEDFWRDGAIN(x) ((x) << 0) -#define MG_PLL_FRAC_LOCK(port) _MMIO_PORT((port) - PORT_C, \ - _MG_PLL_FRAC_LOCK_PORT1, \ - _MG_PLL_FRAC_LOCK_PORT2) +#define MG_PLL_FRAC_LOCK(tc_port) _MMIO_PORT((tc_port), \ + _MG_PLL_FRAC_LOCK_PORT1, \ + _MG_PLL_FRAC_LOCK_PORT2) #define _MG_PLL_SSC_PORT1 0x168A10 #define _MG_PLL_SSC_PORT2 0x169A10 @@ -9477,8 +9673,8 @@ enum skl_power_gate { #define MG_PLL_SSC_STEPNUM(x) ((x) << 10) #define MG_PLL_SSC_FLLEN (1 << 9) #define MG_PLL_SSC_STEPSIZE(x) ((x) << 0) -#define MG_PLL_SSC(port) _MMIO_PORT((port) - PORT_C, _MG_PLL_SSC_PORT1, \ - _MG_PLL_SSC_PORT2) +#define MG_PLL_SSC(tc_port) _MMIO_PORT((tc_port), _MG_PLL_SSC_PORT1, \ + _MG_PLL_SSC_PORT2) #define _MG_PLL_BIAS_PORT1 0x168A14 #define _MG_PLL_BIAS_PORT2 0x169A14 @@ -9497,8 +9693,8 @@ enum skl_power_gate { #define MG_PLL_BIAS_VREF_RDAC_MASK (0x7 << 5) #define MG_PLL_BIAS_IREFTRIM(x) ((x) << 0) #define MG_PLL_BIAS_IREFTRIM_MASK (0x1f << 0) -#define MG_PLL_BIAS(port) _MMIO_PORT((port) - PORT_C, _MG_PLL_BIAS_PORT1, \ - _MG_PLL_BIAS_PORT2) +#define MG_PLL_BIAS(tc_port) _MMIO_PORT((tc_port), _MG_PLL_BIAS_PORT1, \ + _MG_PLL_BIAS_PORT2) #define _MG_PLL_TDC_COLDST_BIAS_PORT1 0x168A18 #define _MG_PLL_TDC_COLDST_BIAS_PORT2 0x169A18 @@ -9509,9 +9705,9 @@ enum skl_power_gate { #define MG_PLL_TDC_COLDST_COLDSTART (1 << 16) #define MG_PLL_TDC_TDCOVCCORR_EN (1 << 2) #define MG_PLL_TDC_TDCSEL(x) ((x) << 0) -#define MG_PLL_TDC_COLDST_BIAS(port) _MMIO_PORT((port) - PORT_C, \ - _MG_PLL_TDC_COLDST_BIAS_PORT1, \ - _MG_PLL_TDC_COLDST_BIAS_PORT2) +#define MG_PLL_TDC_COLDST_BIAS(tc_port) _MMIO_PORT((tc_port), \ + _MG_PLL_TDC_COLDST_BIAS_PORT1, \ + _MG_PLL_TDC_COLDST_BIAS_PORT2) #define _CNL_DPLL0_CFGCR0 0x6C000 #define _CNL_DPLL1_CFGCR0 0x6C080 @@ -9793,6 +9989,10 @@ enum skl_power_gate { #define _MIPI_PORT(port, a, c) (((port) == PORT_A) ? a : c) /* ports A and C only */ #define _MMIO_MIPI(port, a, c) _MMIO(_MIPI_PORT(port, a, c)) +/* Gen11 DSI */ +#define _MMIO_DSI(tc, dsi0, dsi1) _MMIO_TRANS((tc) - TRANSCODER_DSI_0, \ + dsi0, dsi1) + #define MIPIO_TXESC_CLK_DIV1 _MMIO(0x160004) #define GLK_TX_ESC_CLK_DIV1_MASK 0x3FF #define MIPIO_TXESC_CLK_DIV2 _MMIO(0x160008) @@ -9956,6 +10156,39 @@ enum skl_power_gate { _ICL_DSI_IO_MODECTL_1) #define COMBO_PHY_MODE_DSI (1 << 0) +/* Display Stream Splitter Control */ +#define DSS_CTL1 _MMIO(0x67400) +#define SPLITTER_ENABLE (1 << 31) +#define JOINER_ENABLE (1 << 30) +#define DUAL_LINK_MODE_INTERLEAVE (1 << 24) +#define DUAL_LINK_MODE_FRONTBACK (0 << 24) +#define OVERLAP_PIXELS_MASK (0xf << 16) +#define OVERLAP_PIXELS(pixels) ((pixels) << 16) +#define LEFT_DL_BUF_TARGET_DEPTH_MASK (0xfff << 0) +#define LEFT_DL_BUF_TARGET_DEPTH(pixels) ((pixels) << 0) +#define MAX_DL_BUFFER_TARGET_DEPTH 0x5a0 + +#define DSS_CTL2 _MMIO(0x67404) +#define LEFT_BRANCH_VDSC_ENABLE (1 << 31) +#define RIGHT_BRANCH_VDSC_ENABLE (1 << 15) +#define RIGHT_DL_BUF_TARGET_DEPTH_MASK (0xfff << 0) +#define RIGHT_DL_BUF_TARGET_DEPTH(pixels) ((pixels) << 0) + +#define _ICL_PIPE_DSS_CTL1_PB 0x78200 +#define _ICL_PIPE_DSS_CTL1_PC 0x78400 +#define ICL_PIPE_DSS_CTL1(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ + _ICL_PIPE_DSS_CTL1_PB, \ + _ICL_PIPE_DSS_CTL1_PC) +#define BIG_JOINER_ENABLE (1 << 29) +#define MASTER_BIG_JOINER_ENABLE (1 << 28) +#define VGA_CENTERING_ENABLE (1 << 27) + +#define _ICL_PIPE_DSS_CTL2_PB 0x78204 +#define _ICL_PIPE_DSS_CTL2_PC 0x78404 +#define ICL_PIPE_DSS_CTL2(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ + _ICL_PIPE_DSS_CTL2_PB, \ + _ICL_PIPE_DSS_CTL2_PC) + #define BXT_P_DSI_REGULATOR_CFG _MMIO(0x160020) #define STAP_SELECT (1 << 0) @@ -10292,6 +10525,235 @@ enum skl_power_gate { _ICL_DSI_T_INIT_MASTER_0,\ _ICL_DSI_T_INIT_MASTER_1) +#define _DPHY_CLK_TIMING_PARAM_0 0x162180 +#define _DPHY_CLK_TIMING_PARAM_1 0x6c180 +#define DPHY_CLK_TIMING_PARAM(port) _MMIO_PORT(port, \ + _DPHY_CLK_TIMING_PARAM_0,\ + _DPHY_CLK_TIMING_PARAM_1) +#define _DSI_CLK_TIMING_PARAM_0 0x6b080 +#define _DSI_CLK_TIMING_PARAM_1 0x6b880 +#define DSI_CLK_TIMING_PARAM(port) _MMIO_PORT(port, \ + _DSI_CLK_TIMING_PARAM_0,\ + _DSI_CLK_TIMING_PARAM_1) +#define CLK_PREPARE_OVERRIDE (1 << 31) +#define CLK_PREPARE(x) ((x) << 28) +#define CLK_PREPARE_MASK (0x7 << 28) +#define CLK_PREPARE_SHIFT 28 +#define CLK_ZERO_OVERRIDE (1 << 27) +#define CLK_ZERO(x) ((x) << 20) +#define CLK_ZERO_MASK (0xf << 20) +#define CLK_ZERO_SHIFT 20 +#define CLK_PRE_OVERRIDE (1 << 19) +#define CLK_PRE(x) ((x) << 16) +#define CLK_PRE_MASK (0x3 << 16) +#define CLK_PRE_SHIFT 16 +#define CLK_POST_OVERRIDE (1 << 15) +#define CLK_POST(x) ((x) << 8) +#define CLK_POST_MASK (0x7 << 8) +#define CLK_POST_SHIFT 8 +#define CLK_TRAIL_OVERRIDE (1 << 7) +#define CLK_TRAIL(x) ((x) << 0) +#define CLK_TRAIL_MASK (0xf << 0) +#define CLK_TRAIL_SHIFT 0 + +#define _DPHY_DATA_TIMING_PARAM_0 0x162184 +#define _DPHY_DATA_TIMING_PARAM_1 0x6c184 +#define DPHY_DATA_TIMING_PARAM(port) _MMIO_PORT(port, \ + _DPHY_DATA_TIMING_PARAM_0,\ + _DPHY_DATA_TIMING_PARAM_1) +#define _DSI_DATA_TIMING_PARAM_0 0x6B084 +#define _DSI_DATA_TIMING_PARAM_1 0x6B884 +#define DSI_DATA_TIMING_PARAM(port) _MMIO_PORT(port, \ + _DSI_DATA_TIMING_PARAM_0,\ + _DSI_DATA_TIMING_PARAM_1) +#define HS_PREPARE_OVERRIDE (1 << 31) +#define HS_PREPARE(x) ((x) << 24) +#define HS_PREPARE_MASK (0x7 << 24) +#define HS_PREPARE_SHIFT 24 +#define HS_ZERO_OVERRIDE (1 << 23) +#define HS_ZERO(x) ((x) << 16) +#define HS_ZERO_MASK (0xf << 16) +#define HS_ZERO_SHIFT 16 +#define HS_TRAIL_OVERRIDE (1 << 15) +#define HS_TRAIL(x) ((x) << 8) +#define HS_TRAIL_MASK (0x7 << 8) +#define HS_TRAIL_SHIFT 8 +#define HS_EXIT_OVERRIDE (1 << 7) +#define HS_EXIT(x) ((x) << 0) +#define HS_EXIT_MASK (0x7 << 0) +#define HS_EXIT_SHIFT 0 + +#define _DPHY_TA_TIMING_PARAM_0 0x162188 +#define _DPHY_TA_TIMING_PARAM_1 0x6c188 +#define DPHY_TA_TIMING_PARAM(port) _MMIO_PORT(port, \ + _DPHY_TA_TIMING_PARAM_0,\ + _DPHY_TA_TIMING_PARAM_1) +#define _DSI_TA_TIMING_PARAM_0 0x6b098 +#define _DSI_TA_TIMING_PARAM_1 0x6b898 +#define DSI_TA_TIMING_PARAM(port) _MMIO_PORT(port, \ + _DSI_TA_TIMING_PARAM_0,\ + _DSI_TA_TIMING_PARAM_1) +#define TA_SURE_OVERRIDE (1 << 31) +#define TA_SURE(x) ((x) << 16) +#define TA_SURE_MASK (0x1f << 16) +#define TA_SURE_SHIFT 16 +#define TA_GO_OVERRIDE (1 << 15) +#define TA_GO(x) ((x) << 8) +#define TA_GO_MASK (0xf << 8) +#define TA_GO_SHIFT 8 +#define TA_GET_OVERRIDE (1 << 7) +#define TA_GET(x) ((x) << 0) +#define TA_GET_MASK (0xf << 0) +#define TA_GET_SHIFT 0 + +/* DSI transcoder configuration */ +#define _DSI_TRANS_FUNC_CONF_0 0x6b030 +#define _DSI_TRANS_FUNC_CONF_1 0x6b830 +#define DSI_TRANS_FUNC_CONF(tc) _MMIO_DSI(tc, \ + _DSI_TRANS_FUNC_CONF_0,\ + _DSI_TRANS_FUNC_CONF_1) +#define OP_MODE_MASK (0x3 << 28) +#define OP_MODE_SHIFT 28 +#define CMD_MODE_NO_GATE (0x0 << 28) +#define CMD_MODE_TE_GATE (0x1 << 28) +#define VIDEO_MODE_SYNC_EVENT (0x2 << 28) +#define VIDEO_MODE_SYNC_PULSE (0x3 << 28) +#define LINK_READY (1 << 20) +#define PIX_FMT_MASK (0x3 << 16) +#define PIX_FMT_SHIFT 16 +#define PIX_FMT_RGB565 (0x0 << 16) +#define PIX_FMT_RGB666_PACKED (0x1 << 16) +#define PIX_FMT_RGB666_LOOSE (0x2 << 16) +#define PIX_FMT_RGB888 (0x3 << 16) +#define PIX_FMT_RGB101010 (0x4 << 16) +#define PIX_FMT_RGB121212 (0x5 << 16) +#define PIX_FMT_COMPRESSED (0x6 << 16) +#define BGR_TRANSMISSION (1 << 15) +#define PIX_VIRT_CHAN(x) ((x) << 12) +#define PIX_VIRT_CHAN_MASK (0x3 << 12) +#define PIX_VIRT_CHAN_SHIFT 12 +#define PIX_BUF_THRESHOLD_MASK (0x3 << 10) +#define PIX_BUF_THRESHOLD_SHIFT 10 +#define PIX_BUF_THRESHOLD_1_4 (0x0 << 10) +#define PIX_BUF_THRESHOLD_1_2 (0x1 << 10) +#define PIX_BUF_THRESHOLD_3_4 (0x2 << 10) +#define PIX_BUF_THRESHOLD_FULL (0x3 << 10) +#define CONTINUOUS_CLK_MASK (0x3 << 8) +#define CONTINUOUS_CLK_SHIFT 8 +#define CLK_ENTER_LP_AFTER_DATA (0x0 << 8) +#define CLK_HS_OR_LP (0x2 << 8) +#define CLK_HS_CONTINUOUS (0x3 << 8) +#define LINK_CALIBRATION_MASK (0x3 << 4) +#define LINK_CALIBRATION_SHIFT 4 +#define CALIBRATION_DISABLED (0x0 << 4) +#define CALIBRATION_ENABLED_INITIAL_ONLY (0x2 << 4) +#define CALIBRATION_ENABLED_INITIAL_PERIODIC (0x3 << 4) +#define S3D_ORIENTATION_LANDSCAPE (1 << 1) +#define EOTP_DISABLED (1 << 0) + +#define _DSI_CMD_RXCTL_0 0x6b0d4 +#define _DSI_CMD_RXCTL_1 0x6b8d4 +#define DSI_CMD_RXCTL(tc) _MMIO_DSI(tc, \ + _DSI_CMD_RXCTL_0,\ + _DSI_CMD_RXCTL_1) +#define READ_UNLOADS_DW (1 << 16) +#define RECEIVED_UNASSIGNED_TRIGGER (1 << 15) +#define RECEIVED_ACKNOWLEDGE_TRIGGER (1 << 14) +#define RECEIVED_TEAR_EFFECT_TRIGGER (1 << 13) +#define RECEIVED_RESET_TRIGGER (1 << 12) +#define RECEIVED_PAYLOAD_WAS_LOST (1 << 11) +#define RECEIVED_CRC_WAS_LOST (1 << 10) +#define NUMBER_RX_PLOAD_DW_MASK (0xff << 0) +#define NUMBER_RX_PLOAD_DW_SHIFT 0 + +#define _DSI_CMD_TXCTL_0 0x6b0d0 +#define _DSI_CMD_TXCTL_1 0x6b8d0 +#define DSI_CMD_TXCTL(tc) _MMIO_DSI(tc, \ + _DSI_CMD_TXCTL_0,\ + _DSI_CMD_TXCTL_1) +#define KEEP_LINK_IN_HS (1 << 24) +#define FREE_HEADER_CREDIT_MASK (0x1f << 8) +#define FREE_HEADER_CREDIT_SHIFT 0x8 +#define FREE_PLOAD_CREDIT_MASK (0xff << 0) +#define FREE_PLOAD_CREDIT_SHIFT 0 +#define MAX_HEADER_CREDIT 0x10 +#define MAX_PLOAD_CREDIT 0x40 + +#define _DSI_CMD_TXHDR_0 0x6b100 +#define _DSI_CMD_TXHDR_1 0x6b900 +#define DSI_CMD_TXHDR(tc) _MMIO_DSI(tc, \ + _DSI_CMD_TXHDR_0,\ + _DSI_CMD_TXHDR_1) +#define PAYLOAD_PRESENT (1 << 31) +#define LP_DATA_TRANSFER (1 << 30) +#define VBLANK_FENCE (1 << 29) +#define PARAM_WC_MASK (0xffff << 8) +#define PARAM_WC_LOWER_SHIFT 8 +#define PARAM_WC_UPPER_SHIFT 16 +#define VC_MASK (0x3 << 6) +#define VC_SHIFT 6 +#define DT_MASK (0x3f << 0) +#define DT_SHIFT 0 + +#define _DSI_CMD_TXPYLD_0 0x6b104 +#define _DSI_CMD_TXPYLD_1 0x6b904 +#define DSI_CMD_TXPYLD(tc) _MMIO_DSI(tc, \ + _DSI_CMD_TXPYLD_0,\ + _DSI_CMD_TXPYLD_1) + +#define _DSI_LP_MSG_0 0x6b0d8 +#define _DSI_LP_MSG_1 0x6b8d8 +#define DSI_LP_MSG(tc) _MMIO_DSI(tc, \ + _DSI_LP_MSG_0,\ + _DSI_LP_MSG_1) +#define LPTX_IN_PROGRESS (1 << 17) +#define LINK_IN_ULPS (1 << 16) +#define LINK_ULPS_TYPE_LP11 (1 << 8) +#define LINK_ENTER_ULPS (1 << 0) + +/* DSI timeout registers */ +#define _DSI_HSTX_TO_0 0x6b044 +#define _DSI_HSTX_TO_1 0x6b844 +#define DSI_HSTX_TO(tc) _MMIO_DSI(tc, \ + _DSI_HSTX_TO_0,\ + _DSI_HSTX_TO_1) +#define HSTX_TIMEOUT_VALUE_MASK (0xffff << 16) +#define HSTX_TIMEOUT_VALUE_SHIFT 16 +#define HSTX_TIMEOUT_VALUE(x) ((x) << 16) +#define HSTX_TIMED_OUT (1 << 0) + +#define _DSI_LPRX_HOST_TO_0 0x6b048 +#define _DSI_LPRX_HOST_TO_1 0x6b848 +#define DSI_LPRX_HOST_TO(tc) _MMIO_DSI(tc, \ + _DSI_LPRX_HOST_TO_0,\ + _DSI_LPRX_HOST_TO_1) +#define LPRX_TIMED_OUT (1 << 16) +#define LPRX_TIMEOUT_VALUE_MASK (0xffff << 0) +#define LPRX_TIMEOUT_VALUE_SHIFT 0 +#define LPRX_TIMEOUT_VALUE(x) ((x) << 0) + +#define _DSI_PWAIT_TO_0 0x6b040 +#define _DSI_PWAIT_TO_1 0x6b840 +#define DSI_PWAIT_TO(tc) _MMIO_DSI(tc, \ + _DSI_PWAIT_TO_0,\ + _DSI_PWAIT_TO_1) +#define PRESET_TIMEOUT_VALUE_MASK (0xffff << 16) +#define PRESET_TIMEOUT_VALUE_SHIFT 16 +#define PRESET_TIMEOUT_VALUE(x) ((x) << 16) +#define PRESPONSE_TIMEOUT_VALUE_MASK (0xffff << 0) +#define PRESPONSE_TIMEOUT_VALUE_SHIFT 0 +#define PRESPONSE_TIMEOUT_VALUE(x) ((x) << 0) + +#define _DSI_TA_TO_0 0x6b04c +#define _DSI_TA_TO_1 0x6b84c +#define DSI_TA_TO(tc) _MMIO_DSI(tc, \ + _DSI_TA_TO_0,\ + _DSI_TA_TO_1) +#define TA_TIMED_OUT (1 << 16) +#define TA_TIMEOUT_VALUE_MASK (0xffff << 0) +#define TA_TIMEOUT_VALUE_SHIFT 0 +#define TA_TIMEOUT_VALUE(x) ((x) << 0) + /* bits 31:0 */ #define _MIPIA_DBI_BW_CTRL (dev_priv->mipi_mmio_base + 0xb084) #define _MIPIC_DBI_BW_CTRL (dev_priv->mipi_mmio_base + 0xb884) @@ -10404,10 +10866,6 @@ enum skl_power_gate { #define MIPI_READ_DATA_VALID(port) _MMIO_MIPI(port, _MIPIA_READ_DATA_VALID, _MIPIC_READ_DATA_VALID) #define READ_DATA_VALID(n) (1 << (n)) -/* For UMS only (deprecated): */ -#define _PALETTE_A (dev_priv->info.display_mmio_offset + 0xa000) -#define _PALETTE_B (dev_priv->info.display_mmio_offset + 0xa800) - /* MOCS (Memory Object Control State) registers */ #define GEN9_LNCFCMOCS(i) _MMIO(0xb020 + (i) * 4) /* L3 Cache Control */ @@ -10693,6 +11151,7 @@ enum skl_power_gate { #define ICL_DSC1_PICTURE_PARAMETER_SET_16(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ _ICL_DSC1_PICTURE_PARAMETER_SET_16_PB, \ _ICL_DSC1_PICTURE_PARAMETER_SET_16_PC) +#define DSC_SLICE_ROW_PER_FRAME(slice_row_per_frame) ((slice_row_per_frame) << 20) #define DSC_SLICE_PER_LINE(slice_per_line) ((slice_per_line) << 16) #define DSC_SLICE_CHUNK_SIZE(slice_chunk_size) ((slice_chunk_size) << 0) @@ -10747,17 +11206,17 @@ enum skl_power_gate { _ICL_DSC1_RC_BUF_THRESH_1_UDW_PB, \ _ICL_DSC1_RC_BUF_THRESH_1_UDW_PC) -#define PORT_TX_DFLEXDPSP _MMIO(0x1638A0) +#define PORT_TX_DFLEXDPSP _MMIO(FIA1_BASE + 0x008A0) #define TC_LIVE_STATE_TBT(tc_port) (1 << ((tc_port) * 8 + 6)) #define TC_LIVE_STATE_TC(tc_port) (1 << ((tc_port) * 8 + 5)) #define DP_LANE_ASSIGNMENT_SHIFT(tc_port) ((tc_port) * 8) #define DP_LANE_ASSIGNMENT_MASK(tc_port) (0xf << ((tc_port) * 8)) #define DP_LANE_ASSIGNMENT(tc_port, x) ((x) << ((tc_port) * 8)) -#define PORT_TX_DFLEXDPPMS _MMIO(0x163890) +#define PORT_TX_DFLEXDPPMS _MMIO(FIA1_BASE + 0x00890) #define DP_PHY_MODE_STATUS_COMPLETED(tc_port) (1 << (tc_port)) -#define PORT_TX_DFLEXDPCSSS _MMIO(0x163894) +#define PORT_TX_DFLEXDPCSSS _MMIO(FIA1_BASE + 0x00894) #define DP_PHY_MODE_STATUS_NOT_SAFE(tc_port) (1 << (tc_port)) #endif /* _I915_REG_H_ */ diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c index a492385b2089..c2a5c48c7541 100644 --- a/drivers/gpu/drm/i915/i915_request.c +++ b/drivers/gpu/drm/i915/i915_request.c @@ -29,6 +29,8 @@ #include <linux/sched/signal.h> #include "i915_drv.h" +#include "i915_active.h" +#include "i915_reset.h" static const char *i915_fence_get_driver_name(struct dma_fence *fence) { @@ -59,7 +61,7 @@ static bool i915_fence_signaled(struct dma_fence *fence) static bool i915_fence_enable_signaling(struct dma_fence *fence) { - return intel_engine_enable_signaling(to_request(fence), true); + return i915_request_enable_breadcrumb(to_request(fence)); } static signed long i915_fence_wait(struct dma_fence *fence, @@ -111,177 +113,10 @@ i915_request_remove_from_client(struct i915_request *request) spin_unlock(&file_priv->mm.lock); } -static struct i915_dependency * -i915_dependency_alloc(struct drm_i915_private *i915) +static void reserve_gt(struct drm_i915_private *i915) { - return kmem_cache_alloc(i915->dependencies, GFP_KERNEL); -} - -static void -i915_dependency_free(struct drm_i915_private *i915, - struct i915_dependency *dep) -{ - kmem_cache_free(i915->dependencies, dep); -} - -static void -__i915_sched_node_add_dependency(struct i915_sched_node *node, - struct i915_sched_node *signal, - struct i915_dependency *dep, - unsigned long flags) -{ - INIT_LIST_HEAD(&dep->dfs_link); - list_add(&dep->wait_link, &signal->waiters_list); - list_add(&dep->signal_link, &node->signalers_list); - dep->signaler = signal; - dep->flags = flags; -} - -static int -i915_sched_node_add_dependency(struct drm_i915_private *i915, - struct i915_sched_node *node, - struct i915_sched_node *signal) -{ - struct i915_dependency *dep; - - dep = i915_dependency_alloc(i915); - if (!dep) - return -ENOMEM; - - __i915_sched_node_add_dependency(node, signal, dep, - I915_DEPENDENCY_ALLOC); - return 0; -} - -static void -i915_sched_node_fini(struct drm_i915_private *i915, - struct i915_sched_node *node) -{ - struct i915_dependency *dep, *tmp; - - GEM_BUG_ON(!list_empty(&node->link)); - - /* - * Everyone we depended upon (the fences we wait to be signaled) - * should retire before us and remove themselves from our list. - * However, retirement is run independently on each timeline and - * so we may be called out-of-order. - */ - list_for_each_entry_safe(dep, tmp, &node->signalers_list, signal_link) { - GEM_BUG_ON(!i915_sched_node_signaled(dep->signaler)); - GEM_BUG_ON(!list_empty(&dep->dfs_link)); - - list_del(&dep->wait_link); - if (dep->flags & I915_DEPENDENCY_ALLOC) - i915_dependency_free(i915, dep); - } - - /* Remove ourselves from everyone who depends upon us */ - list_for_each_entry_safe(dep, tmp, &node->waiters_list, wait_link) { - GEM_BUG_ON(dep->signaler != node); - GEM_BUG_ON(!list_empty(&dep->dfs_link)); - - list_del(&dep->signal_link); - if (dep->flags & I915_DEPENDENCY_ALLOC) - i915_dependency_free(i915, dep); - } -} - -static void -i915_sched_node_init(struct i915_sched_node *node) -{ - INIT_LIST_HEAD(&node->signalers_list); - INIT_LIST_HEAD(&node->waiters_list); - INIT_LIST_HEAD(&node->link); - node->attr.priority = I915_PRIORITY_INVALID; -} - -static int reset_all_global_seqno(struct drm_i915_private *i915, u32 seqno) -{ - struct intel_engine_cs *engine; - struct i915_timeline *timeline; - enum intel_engine_id id; - int ret; - - /* Carefully retire all requests without writing to the rings */ - ret = i915_gem_wait_for_idle(i915, - I915_WAIT_INTERRUPTIBLE | - I915_WAIT_LOCKED, - MAX_SCHEDULE_TIMEOUT); - if (ret) - return ret; - - GEM_BUG_ON(i915->gt.active_requests); - - /* If the seqno wraps around, we need to clear the breadcrumb rbtree */ - for_each_engine(engine, i915, id) { - GEM_TRACE("%s seqno %d (current %d) -> %d\n", - engine->name, - engine->timeline.seqno, - intel_engine_get_seqno(engine), - seqno); - - if (!i915_seqno_passed(seqno, engine->timeline.seqno)) { - /* Flush any waiters before we reuse the seqno */ - intel_engine_disarm_breadcrumbs(engine); - intel_engine_init_hangcheck(engine); - GEM_BUG_ON(!list_empty(&engine->breadcrumbs.signals)); - } - - /* Check we are idle before we fiddle with hw state! */ - GEM_BUG_ON(!intel_engine_is_idle(engine)); - GEM_BUG_ON(i915_gem_active_isset(&engine->timeline.last_request)); - - /* Finally reset hw state */ - intel_engine_init_global_seqno(engine, seqno); - engine->timeline.seqno = seqno; - } - - list_for_each_entry(timeline, &i915->gt.timelines, link) - memset(timeline->global_sync, 0, sizeof(timeline->global_sync)); - - i915->gt.request_serial = seqno; - - return 0; -} - -int i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno) -{ - struct drm_i915_private *i915 = to_i915(dev); - - lockdep_assert_held(&i915->drm.struct_mutex); - - if (seqno == 0) - return -EINVAL; - - /* HWS page needs to be set less than what we will inject to ring */ - return reset_all_global_seqno(i915, seqno - 1); -} - -static int reserve_gt(struct drm_i915_private *i915) -{ - int ret; - - /* - * Reservation is fine until we may need to wrap around - * - * By incrementing the serial for every request, we know that no - * individual engine may exceed that serial (as each is reset to 0 - * on any wrap). This protects even the most pessimistic of migrations - * of every request from all engines onto just one. - */ - while (unlikely(++i915->gt.request_serial == 0)) { - ret = reset_all_global_seqno(i915, 0); - if (ret) { - i915->gt.request_serial--; - return ret; - } - } - if (!i915->gt.active_requests++) i915_gem_unpark(i915); - - return 0; } static void unreserve_gt(struct drm_i915_private *i915) @@ -291,12 +126,6 @@ static void unreserve_gt(struct drm_i915_private *i915) i915_gem_park(i915); } -void i915_gem_retire_noop(struct i915_gem_active *active, - struct i915_request *request) -{ - /* Space left intentionally blank */ -} - static void advance_ring(struct i915_request *request) { struct intel_ring *ring = request->ring; @@ -348,10 +177,11 @@ static void free_capture_list(struct i915_request *request) static void __retire_engine_request(struct intel_engine_cs *engine, struct i915_request *rq) { - GEM_TRACE("%s(%s) fence %llx:%d, global=%d, current %d\n", + GEM_TRACE("%s(%s) fence %llx:%lld, global=%d, current %d:%d\n", __func__, engine->name, rq->fence.context, rq->fence.seqno, rq->global_seqno, + hwsp_seqno(rq), intel_engine_get_seqno(engine)); GEM_BUG_ON(!i915_request_completed(rq)); @@ -364,10 +194,11 @@ static void __retire_engine_request(struct intel_engine_cs *engine, spin_unlock(&engine->timeline.lock); spin_lock(&rq->lock); - if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags)) + i915_request_mark_complete(rq); + if (!i915_request_signaled(rq)) dma_fence_signal_locked(&rq->fence); if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &rq->fence.flags)) - intel_engine_cancel_signaling(rq); + i915_request_cancel_breadcrumb(rq); if (rq->waitboost) { GEM_BUG_ON(!atomic_read(&rq->i915->gt_pm.rps.num_waiters)); atomic_dec(&rq->i915->gt_pm.rps.num_waiters); @@ -408,12 +239,13 @@ static void __retire_engine_upto(struct intel_engine_cs *engine, static void i915_request_retire(struct i915_request *request) { - struct i915_gem_active *active, *next; + struct i915_active_request *active, *next; - GEM_TRACE("%s fence %llx:%d, global=%d, current %d\n", + GEM_TRACE("%s fence %llx:%lld, global=%d, current %d:%d\n", request->engine->name, request->fence.context, request->fence.seqno, request->global_seqno, + hwsp_seqno(request), intel_engine_get_seqno(request->engine)); lockdep_assert_held(&request->i915->drm.struct_mutex); @@ -441,10 +273,10 @@ static void i915_request_retire(struct i915_request *request) * we may spend an inordinate amount of time simply handling * the retirement of requests and processing their callbacks. * Of which, this loop itself is particularly hot due to the - * cache misses when jumping around the list of i915_gem_active. - * So we try to keep this loop as streamlined as possible and - * also prefetch the next i915_gem_active to try and hide - * the likely cache miss. + * cache misses when jumping around the list of + * i915_active_request. So we try to keep this loop as + * streamlined as possible and also prefetch the next + * i915_active_request to try and hide the likely cache miss. */ prefetchw(next); @@ -473,10 +305,11 @@ void i915_request_retire_upto(struct i915_request *rq) struct intel_ring *ring = rq->ring; struct i915_request *tmp; - GEM_TRACE("%s fence %llx:%d, global=%d, current %d\n", + GEM_TRACE("%s fence %llx:%lld, global=%d, current %d:%d\n", rq->engine->name, rq->fence.context, rq->fence.seqno, rq->global_seqno, + hwsp_seqno(rq), intel_engine_get_seqno(rq->engine)); lockdep_assert_held(&rq->i915->drm.struct_mutex); @@ -495,7 +328,7 @@ void i915_request_retire_upto(struct i915_request *rq) static u32 timeline_get_seqno(struct i915_timeline *tl) { - return ++tl->seqno; + return tl->seqno += 1 + tl->has_initial_breadcrumb; } static void move_to_timeline(struct i915_request *request, @@ -509,15 +342,23 @@ static void move_to_timeline(struct i915_request *request, spin_unlock(&request->timeline->lock); } +static u32 next_global_seqno(struct i915_timeline *tl) +{ + if (!++tl->seqno) + ++tl->seqno; + return tl->seqno; +} + void __i915_request_submit(struct i915_request *request) { struct intel_engine_cs *engine = request->engine; u32 seqno; - GEM_TRACE("%s fence %llx:%d -> global=%d, current %d\n", + GEM_TRACE("%s fence %llx:%lld -> global=%d, current %d:%d\n", engine->name, request->fence.context, request->fence.seqno, engine->timeline.seqno + 1, + hwsp_seqno(request), intel_engine_get_seqno(engine)); GEM_BUG_ON(!irqs_disabled()); @@ -525,26 +366,27 @@ void __i915_request_submit(struct i915_request *request) GEM_BUG_ON(request->global_seqno); - seqno = timeline_get_seqno(&engine->timeline); + seqno = next_global_seqno(&engine->timeline); GEM_BUG_ON(!seqno); GEM_BUG_ON(intel_engine_signaled(engine, seqno)); /* We may be recursing from the signal callback of another i915 fence */ spin_lock_nested(&request->lock, SINGLE_DEPTH_NESTING); + GEM_BUG_ON(test_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags)); + set_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags); request->global_seqno = seqno; - if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags)) - intel_engine_enable_signaling(request, false); + if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags) && + !i915_request_enable_breadcrumb(request)) + intel_engine_queue_breadcrumbs(engine); spin_unlock(&request->lock); - engine->emit_breadcrumb(request, - request->ring->vaddr + request->postfix); + engine->emit_fini_breadcrumb(request, + request->ring->vaddr + request->postfix); /* Transfer from per-context onto the global per-engine timeline */ move_to_timeline(request, &engine->timeline); trace_i915_request_execute(request); - - wake_up_all(&request->execute); } void i915_request_submit(struct i915_request *request) @@ -564,10 +406,11 @@ void __i915_request_unsubmit(struct i915_request *request) { struct intel_engine_cs *engine = request->engine; - GEM_TRACE("%s fence %llx:%d <- global=%d, current %d\n", + GEM_TRACE("%s fence %llx:%lld <- global=%d, current %d:%d\n", engine->name, request->fence.context, request->fence.seqno, request->global_seqno, + hwsp_seqno(request), intel_engine_get_seqno(engine)); GEM_BUG_ON(!irqs_disabled()); @@ -586,7 +429,9 @@ void __i915_request_unsubmit(struct i915_request *request) spin_lock_nested(&request->lock, SINGLE_DEPTH_NESTING); request->global_seqno = 0; if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags)) - intel_engine_cancel_signaling(request); + i915_request_cancel_breadcrumb(request); + GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags)); + clear_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags); spin_unlock(&request->lock); /* Transfer back from the global per-engine timeline to per-context */ @@ -644,6 +489,43 @@ submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) return NOTIFY_DONE; } +static void ring_retire_requests(struct intel_ring *ring) +{ + struct i915_request *rq, *rn; + + list_for_each_entry_safe(rq, rn, &ring->request_list, ring_link) { + if (!i915_request_completed(rq)) + break; + + i915_request_retire(rq); + } +} + +static noinline struct i915_request * +i915_request_alloc_slow(struct intel_context *ce) +{ + struct intel_ring *ring = ce->ring; + struct i915_request *rq; + + if (list_empty(&ring->request_list)) + goto out; + + /* Ratelimit ourselves to prevent oom from malicious clients */ + rq = list_last_entry(&ring->request_list, typeof(*rq), ring_link); + cond_synchronize_rcu(rq->rcustate); + + /* Retire our old requests in the hope that we free some */ + ring_retire_requests(ring); + +out: + return kmem_cache_alloc(ce->gem_context->i915->requests, GFP_KERNEL); +} + +static int add_timeline_barrier(struct i915_request *rq) +{ + return i915_request_await_active_request(rq, &rq->timeline->barrier); +} + /** * i915_request_alloc - allocate a request structure * @@ -686,13 +568,7 @@ i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx) if (IS_ERR(ce)) return ERR_CAST(ce); - ret = reserve_gt(i915); - if (ret) - goto err_unpin; - - ret = intel_ring_wait_for_space(ce->ring, MIN_SPACE_FOR_ADD_REQUEST); - if (ret) - goto err_unreserve; + reserve_gt(i915); /* Move our oldest request to the slab-cache (if not in use!) */ rq = list_first_entry(&ce->ring->request_list, typeof(*rq), ring_link); @@ -706,7 +582,7 @@ i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx) * We use RCU to look up requests in flight. The lookups may * race with the request being allocated from the slab freelist. * That is the request we are writing to here, may be in the process - * of being read by __i915_gem_active_get_rcu(). As such, + * of being read by __i915_active_request_get_rcu(). As such, * we have to be very careful when overwriting the contents. During * the RCU lookup, we change chase the request->engine pointer, * read the request->global_seqno and increment the reference count. @@ -732,26 +608,7 @@ i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx) rq = kmem_cache_alloc(i915->requests, GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); if (unlikely(!rq)) { - i915_retire_requests(i915); - - /* Ratelimit ourselves to prevent oom from malicious clients */ - rq = i915_gem_active_raw(&ce->ring->timeline->last_request, - &i915->drm.struct_mutex); - if (rq) - cond_synchronize_rcu(rq->rcustate); - - /* - * We've forced the client to stall and catch up with whatever - * backlog there might have been. As we are assuming that we - * caused the mempressure, now is an opportune time to - * recover as much memory from the request pool as is possible. - * Having already penalized the client to stall, we spend - * a little extra time to re-optimise page allocation. - */ - kmem_cache_shrink(i915->requests); - rcu_barrier(); /* Recover the TYPESAFE_BY_RCU pages */ - - rq = kmem_cache_alloc(i915->requests, GFP_KERNEL); + rq = i915_request_alloc_slow(ce); if (!rq) { ret = -ENOMEM; goto err_unreserve; @@ -768,6 +625,7 @@ i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx) rq->ring = ce->ring; rq->timeline = ce->ring->timeline; GEM_BUG_ON(rq->timeline == &engine->timeline); + rq->hwsp_seqno = rq->timeline->hwsp_seqno; spin_lock_init(&rq->lock); dma_fence_init(&rq->fence, @@ -778,13 +636,11 @@ i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx) /* We bump the ref for the fence chain */ i915_sw_fence_init(&i915_request_get(rq)->submit, submit_notify); - init_waitqueue_head(&rq->execute); i915_sched_node_init(&rq->sched); /* No zalloc, must clear what we need by hand */ rq->global_seqno = 0; - rq->signaling.wait.seqno = 0; rq->file_priv = NULL; rq->batch = NULL; rq->capture_list = NULL; @@ -796,9 +652,13 @@ i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx) * i915_request_add() call can't fail. Note that the reserve may need * to be redone if the request is not actually submitted straight * away, e.g. because a GPU scheduler has deferred it. + * + * Note that due to how we add reserved_space to intel_ring_begin() + * we need to double our request to ensure that if we need to wrap + * around inside i915_request_add() there is sufficient space at + * the beginning of the ring as well. */ - rq->reserved_space = MIN_SPACE_FOR_ADD_REQUEST; - GEM_BUG_ON(rq->reserved_space < engine->emit_breadcrumb_sz); + rq->reserved_space = 2 * engine->emit_fini_breadcrumb_dw * sizeof(u32); /* * Record the position of the start of the request so that @@ -808,8 +668,7 @@ i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx) */ rq->head = rq->ring->emit; - /* Unconditionally invalidate GPU caches and TLBs. */ - ret = engine->emit_flush(rq, EMIT_INVALIDATE); + ret = add_timeline_barrier(rq); if (ret) goto err_unwind; @@ -837,7 +696,6 @@ err_unwind: kmem_cache_free(i915->requests, rq); err_unreserve: unreserve_gt(i915); -err_unpin: intel_context_unpin(ce); return ERR_PTR(ret); } @@ -865,34 +723,12 @@ i915_request_await_request(struct i915_request *to, struct i915_request *from) ret = i915_sw_fence_await_sw_fence_gfp(&to->submit, &from->submit, I915_FENCE_GFP); - return ret < 0 ? ret : 0; - } - - if (to->engine->semaphore.sync_to) { - u32 seqno; - - GEM_BUG_ON(!from->engine->semaphore.signal); - - seqno = i915_request_global_seqno(from); - if (!seqno) - goto await_dma_fence; - - if (seqno <= to->timeline->global_sync[from->engine->id]) - return 0; - - trace_i915_gem_ring_sync_to(to, from); - ret = to->engine->semaphore.sync_to(to, from); - if (ret) - return ret; - - to->timeline->global_sync[from->engine->id] = seqno; - return 0; + } else { + ret = i915_sw_fence_await_dma_fence(&to->submit, + &from->fence, 0, + I915_FENCE_GFP); } -await_dma_fence: - ret = i915_sw_fence_await_dma_fence(&to->submit, - &from->fence, 0, - I915_FENCE_GFP); return ret < 0 ? ret : 0; } @@ -1050,7 +886,7 @@ void i915_request_add(struct i915_request *request) struct i915_request *prev; u32 *cs; - GEM_TRACE("%s fence %llx:%d\n", + GEM_TRACE("%s fence %llx:%lld\n", engine->name, request->fence.context, request->fence.seqno); lockdep_assert_held(&request->i915->drm.struct_mutex); @@ -1068,8 +904,8 @@ void i915_request_add(struct i915_request *request) * should already have been reserved in the ring buffer. Let the ring * know that it is time to use that space up. */ + GEM_BUG_ON(request->reserved_space > request->ring->space); request->reserved_space = 0; - engine->emit_flush(request, EMIT_FLUSH); /* * Record the position of the start of the breadcrumb so that @@ -1077,7 +913,7 @@ void i915_request_add(struct i915_request *request) * GPU processing the request, we never over-estimate the * position of the ring's HEAD. */ - cs = intel_ring_begin(request, engine->emit_breadcrumb_sz); + cs = intel_ring_begin(request, engine->emit_fini_breadcrumb_dw); GEM_BUG_ON(IS_ERR(cs)); request->postfix = intel_ring_offset(request, cs); @@ -1088,8 +924,8 @@ void i915_request_add(struct i915_request *request) * see a more recent value in the hws than we are tracking. */ - prev = i915_gem_active_raw(&timeline->last_request, - &request->i915->drm.struct_mutex); + prev = i915_active_request_raw(&timeline->last_request, + &request->i915->drm.struct_mutex); if (prev && !i915_request_completed(prev)) { i915_sw_fence_await_sw_fence(&request->submit, &prev->submit, &request->submitq); @@ -1105,7 +941,7 @@ void i915_request_add(struct i915_request *request) spin_unlock_irq(&timeline->lock); GEM_BUG_ON(timeline->seqno != request->fence.seqno); - i915_gem_active_set(&timeline->last_request, request); + __i915_active_request_set(&timeline->last_request, request); list_add_tail(&request->ring_link, &ring->request_list); if (list_is_first(&request->ring_link, &ring->request_list)) { @@ -1127,8 +963,20 @@ void i915_request_add(struct i915_request *request) */ local_bh_disable(); rcu_read_lock(); /* RCU serialisation for set-wedged protection */ - if (engine->schedule) - engine->schedule(request, &request->gem_context->sched); + if (engine->schedule) { + struct i915_sched_attr attr = request->gem_context->sched; + + /* + * Boost priorities to new clients (new request flows). + * + * Allow interactive/synchronous clients to jump ahead of + * the bulk clients. (FQ_CODEL) + */ + if (list_empty(&request->sched.signalers_list)) + attr.priority |= I915_PRIORITY_NEWCLIENT; + + engine->schedule(request, &attr); + } rcu_read_unlock(); i915_sw_fence_commit(&request->submit); local_bh_enable(); /* Kick the execlists tasklet if just scheduled */ @@ -1187,13 +1035,10 @@ static bool busywait_stop(unsigned long timeout, unsigned int cpu) return this_cpu != cpu; } -static bool __i915_spin_request(const struct i915_request *rq, - u32 seqno, int state, unsigned long timeout_us) +static bool __i915_spin_request(const struct i915_request * const rq, + int state, unsigned long timeout_us) { - struct intel_engine_cs *engine = rq->engine; - unsigned int irq, cpu; - - GEM_BUG_ON(!seqno); + unsigned int cpu; /* * Only wait for the request if we know it is likely to complete. @@ -1201,12 +1046,12 @@ static bool __i915_spin_request(const struct i915_request *rq, * We don't track the timestamps around requests, nor the average * request length, so we do not have a good indicator that this * request will complete within the timeout. What we do know is the - * order in which requests are executed by the engine and so we can - * tell if the request has started. If the request hasn't started yet, - * it is a fair assumption that it will not complete within our - * relatively short timeout. + * order in which requests are executed by the context and so we can + * tell if the request has been started. If the request is not even + * running yet, it is a fair assumption that it will not complete + * within our relatively short timeout. */ - if (!intel_engine_has_started(engine, seqno)) + if (!i915_request_is_running(rq)) return false; /* @@ -1220,20 +1065,10 @@ static bool __i915_spin_request(const struct i915_request *rq, * takes to sleep on a request, on the order of a microsecond. */ - irq = READ_ONCE(engine->breadcrumbs.irq_count); timeout_us += local_clock_us(&cpu); do { - if (intel_engine_has_completed(engine, seqno)) - return seqno == i915_request_global_seqno(rq); - - /* - * Seqno are meant to be ordered *before* the interrupt. If - * we see an interrupt without a corresponding seqno advance, - * assume we won't see one in the near future but require - * the engine->seqno_barrier() to fixup coherency. - */ - if (READ_ONCE(engine->breadcrumbs.irq_count) != irq) - break; + if (i915_request_completed(rq)) + return true; if (signal_pending_state(state, current)) break; @@ -1247,16 +1082,16 @@ static bool __i915_spin_request(const struct i915_request *rq, return false; } -static bool __i915_wait_request_check_and_reset(struct i915_request *request) -{ - struct i915_gpu_error *error = &request->i915->gpu_error; +struct request_wait { + struct dma_fence_cb cb; + struct task_struct *tsk; +}; - if (likely(!i915_reset_handoff(error))) - return false; +static void request_wait_wake(struct dma_fence *fence, struct dma_fence_cb *cb) +{ + struct request_wait *wait = container_of(cb, typeof(*wait), cb); - __set_current_state(TASK_RUNNING); - i915_reset(request->i915, error->stalled_mask, error->reason); - return true; + wake_up_process(wait->tsk); } /** @@ -1284,17 +1119,9 @@ long i915_request_wait(struct i915_request *rq, { const int state = flags & I915_WAIT_INTERRUPTIBLE ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE; - wait_queue_head_t *errq = &rq->i915->gpu_error.wait_queue; - DEFINE_WAIT_FUNC(reset, default_wake_function); - DEFINE_WAIT_FUNC(exec, default_wake_function); - struct intel_wait wait; + struct request_wait wait; might_sleep(); -#if IS_ENABLED(CONFIG_LOCKDEP) - GEM_BUG_ON(debug_locks && - !!lockdep_is_held(&rq->i915->drm.struct_mutex) != - !!(flags & I915_WAIT_LOCKED)); -#endif GEM_BUG_ON(timeout < 0); if (i915_request_completed(rq)) @@ -1305,55 +1132,23 @@ long i915_request_wait(struct i915_request *rq, trace_i915_request_wait_begin(rq, flags); - add_wait_queue(&rq->execute, &exec); - if (flags & I915_WAIT_LOCKED) - add_wait_queue(errq, &reset); - - intel_wait_init(&wait); - -restart: - do { - set_current_state(state); - if (intel_wait_update_request(&wait, rq)) - break; - - if (flags & I915_WAIT_LOCKED && - __i915_wait_request_check_and_reset(rq)) - continue; - - if (signal_pending_state(state, current)) { - timeout = -ERESTARTSYS; - goto complete; - } - - if (!timeout) { - timeout = -ETIME; - goto complete; - } - - timeout = io_schedule_timeout(timeout); - } while (1); - - GEM_BUG_ON(!intel_wait_has_seqno(&wait)); - GEM_BUG_ON(!i915_sw_fence_signaled(&rq->submit)); - /* Optimistic short spin before touching IRQs */ - if (__i915_spin_request(rq, wait.seqno, state, 5)) - goto complete; + if (__i915_spin_request(rq, state, 5)) + goto out; - set_current_state(state); - if (intel_engine_add_wait(rq->engine, &wait)) - /* - * In order to check that we haven't missed the interrupt - * as we enabled it, we need to kick ourselves to do a - * coherent check on the seqno before we sleep. - */ - goto wakeup; + if (flags & I915_WAIT_PRIORITY) + i915_schedule_bump_priority(rq, I915_PRIORITY_WAIT); - if (flags & I915_WAIT_LOCKED) - __i915_wait_request_check_and_reset(rq); + wait.tsk = current; + if (dma_fence_add_callback(&rq->fence, &wait.cb, request_wait_wake)) + goto out; for (;;) { + set_current_state(state); + + if (i915_request_completed(rq)) + break; + if (signal_pending_state(state, current)) { timeout = -ERESTARTSYS; break; @@ -1365,70 +1160,14 @@ restart: } timeout = io_schedule_timeout(timeout); - - if (intel_wait_complete(&wait) && - intel_wait_check_request(&wait, rq)) - break; - - set_current_state(state); - -wakeup: - /* - * Carefully check if the request is complete, giving time - * for the seqno to be visible following the interrupt. - * We also have to check in case we are kicked by the GPU - * reset in order to drop the struct_mutex. - */ - if (__i915_request_irq_complete(rq)) - break; - - /* - * If the GPU is hung, and we hold the lock, reset the GPU - * and then check for completion. On a full reset, the engine's - * HW seqno will be advanced passed us and we are complete. - * If we do a partial reset, we have to wait for the GPU to - * resume and update the breadcrumb. - * - * If we don't hold the mutex, we can just wait for the worker - * to come along and update the breadcrumb (either directly - * itself, or indirectly by recovering the GPU). - */ - if (flags & I915_WAIT_LOCKED && - __i915_wait_request_check_and_reset(rq)) - continue; - - /* Only spin if we know the GPU is processing this request */ - if (__i915_spin_request(rq, wait.seqno, state, 2)) - break; - - if (!intel_wait_check_request(&wait, rq)) { - intel_engine_remove_wait(rq->engine, &wait); - goto restart; - } } - - intel_engine_remove_wait(rq->engine, &wait); -complete: __set_current_state(TASK_RUNNING); - if (flags & I915_WAIT_LOCKED) - remove_wait_queue(errq, &reset); - remove_wait_queue(&rq->execute, &exec); - trace_i915_request_wait_end(rq); - - return timeout; -} -static void ring_retire_requests(struct intel_ring *ring) -{ - struct i915_request *request, *next; + dma_fence_remove_callback(&rq->fence, &wait.cb); - list_for_each_entry_safe(request, next, - &ring->request_list, ring_link) { - if (!i915_request_completed(request)) - break; - - i915_request_retire(request); - } +out: + trace_i915_request_wait_end(rq); + return timeout; } void i915_retire_requests(struct drm_i915_private *i915) diff --git a/drivers/gpu/drm/i915/i915_request.h b/drivers/gpu/drm/i915/i915_request.h index 7fa94b024968..40f3e8dcbdd5 100644 --- a/drivers/gpu/drm/i915/i915_request.h +++ b/drivers/gpu/drm/i915/i915_request.h @@ -30,7 +30,6 @@ #include "i915_gem.h" #include "i915_scheduler.h" #include "i915_sw_fence.h" -#include "i915_scheduler.h" #include <uapi/drm/i915_drm.h> @@ -39,23 +38,34 @@ struct drm_i915_gem_object; struct i915_request; struct i915_timeline; -struct intel_wait { - struct rb_node node; - struct task_struct *tsk; - struct i915_request *request; - u32 seqno; -}; - -struct intel_signal_node { - struct intel_wait wait; - struct list_head link; -}; - struct i915_capture_list { struct i915_capture_list *next; struct i915_vma *vma; }; +enum { + /* + * I915_FENCE_FLAG_ACTIVE - this request is currently submitted to HW. + * + * Set by __i915_request_submit() on handing over to HW, and cleared + * by __i915_request_unsubmit() if we preempt this request. + * + * Finally cleared for consistency on retiring the request, when + * we know the HW is no longer running this request. + * + * See i915_request_is_active() + */ + I915_FENCE_FLAG_ACTIVE = DMA_FENCE_FLAG_USER_BITS, + + /* + * I915_FENCE_FLAG_SIGNAL - this request is currently on signal_list + * + * Internal bookkeeping used by the breadcrumb code to track when + * a request is on the various signal_list. + */ + I915_FENCE_FLAG_SIGNAL, +}; + /** * Request queue structure. * @@ -98,7 +108,7 @@ struct i915_request { struct intel_context *hw_context; struct intel_ring *ring; struct i915_timeline *timeline; - struct intel_signal_node signaling; + struct list_head signal_link; /* * The rcu epoch of when this request was allocated. Used to judiciously @@ -117,7 +127,6 @@ struct i915_request { */ struct i915_sw_fence submit; wait_queue_entry_t submitq; - wait_queue_head_t execute; /* * A list of everyone we wait upon, and everyone who waits upon us. @@ -131,6 +140,13 @@ struct i915_request { struct i915_sched_node sched; struct i915_dependency dep; + /* + * A convenience pointer to the current breadcrumb value stored in + * the HW status page (or our timeline's local equivalent). The full + * path would be rq->hw_context->ring->timeline->hwsp_seqno. + */ + const u32 *hwsp_seqno; + /** * GEM sequence number associated with this request on the * global execution timeline. It is zero when the request is not @@ -249,7 +265,7 @@ i915_request_put(struct i915_request *rq) * that it has passed the global seqno and the global seqno is unchanged * after the read, it is indeed complete). */ -static u32 +static inline u32 i915_request_global_seqno(const struct i915_request *request) { return READ_ONCE(request->global_seqno); @@ -271,458 +287,120 @@ void i915_request_skip(struct i915_request *request, int error); void __i915_request_unsubmit(struct i915_request *request); void i915_request_unsubmit(struct i915_request *request); +/* Note: part of the intel_breadcrumbs family */ +bool i915_request_enable_breadcrumb(struct i915_request *request); +void i915_request_cancel_breadcrumb(struct i915_request *request); + long i915_request_wait(struct i915_request *rq, unsigned int flags, long timeout) __attribute__((nonnull(1))); #define I915_WAIT_INTERRUPTIBLE BIT(0) #define I915_WAIT_LOCKED BIT(1) /* struct_mutex held, handle GPU reset */ -#define I915_WAIT_ALL BIT(2) /* used by i915_gem_object_wait() */ -#define I915_WAIT_FOR_IDLE_BOOST BIT(3) - -static inline bool intel_engine_has_started(struct intel_engine_cs *engine, - u32 seqno); -static inline bool intel_engine_has_completed(struct intel_engine_cs *engine, - u32 seqno); - -/** - * Returns true if seq1 is later than seq2. - */ -static inline bool i915_seqno_passed(u32 seq1, u32 seq2) -{ - return (s32)(seq1 - seq2) >= 0; -} - -/** - * i915_request_started - check if the request has begun being executed - * @rq: the request - * - * Returns true if the request has been submitted to hardware, and the hardware - * has advanced passed the end of the previous request and so should be either - * currently processing the request (though it may be preempted and so - * not necessarily the next request to complete) or have completed the request. - */ -static inline bool i915_request_started(const struct i915_request *rq) -{ - u32 seqno; - - seqno = i915_request_global_seqno(rq); - if (!seqno) /* not yet submitted to HW */ - return false; - - return intel_engine_has_started(rq->engine, seqno); -} - -static inline bool -__i915_request_completed(const struct i915_request *rq, u32 seqno) -{ - GEM_BUG_ON(!seqno); - return intel_engine_has_completed(rq->engine, seqno) && - seqno == i915_request_global_seqno(rq); -} - -static inline bool i915_request_completed(const struct i915_request *rq) -{ - u32 seqno; - - seqno = i915_request_global_seqno(rq); - if (!seqno) - return false; +#define I915_WAIT_PRIORITY BIT(2) /* small priority bump for the request */ +#define I915_WAIT_ALL BIT(3) /* used by i915_gem_object_wait() */ +#define I915_WAIT_FOR_IDLE_BOOST BIT(4) - return __i915_request_completed(rq, seqno); -} - -static inline bool i915_sched_node_signaled(const struct i915_sched_node *node) +static inline bool i915_request_signaled(const struct i915_request *rq) { - const struct i915_request *rq = - container_of(node, const struct i915_request, sched); - - return i915_request_completed(rq); + /* The request may live longer than its HWSP, so check flags first! */ + return test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags); } -void i915_retire_requests(struct drm_i915_private *i915); - -/* - * We treat requests as fences. This is not be to confused with our - * "fence registers" but pipeline synchronisation objects ala GL_ARB_sync. - * We use the fences to synchronize access from the CPU with activity on the - * GPU, for example, we should not rewrite an object's PTE whilst the GPU - * is reading them. We also track fences at a higher level to provide - * implicit synchronisation around GEM objects, e.g. set-domain will wait - * for outstanding GPU rendering before marking the object ready for CPU - * access, or a pageflip will wait until the GPU is complete before showing - * the frame on the scanout. - * - * In order to use a fence, the object must track the fence it needs to - * serialise with. For example, GEM objects want to track both read and - * write access so that we can perform concurrent read operations between - * the CPU and GPU engines, as well as waiting for all rendering to - * complete, or waiting for the last GPU user of a "fence register". The - * object then embeds a #i915_gem_active to track the most recent (in - * retirement order) request relevant for the desired mode of access. - * The #i915_gem_active is updated with i915_gem_active_set() to track the - * most recent fence request, typically this is done as part of - * i915_vma_move_to_active(). - * - * When the #i915_gem_active completes (is retired), it will - * signal its completion to the owner through a callback as well as mark - * itself as idle (i915_gem_active.request == NULL). The owner - * can then perform any action, such as delayed freeing of an active - * resource including itself. - */ -struct i915_gem_active; - -typedef void (*i915_gem_retire_fn)(struct i915_gem_active *, - struct i915_request *); - -struct i915_gem_active { - struct i915_request __rcu *request; - struct list_head link; - i915_gem_retire_fn retire; -}; - -void i915_gem_retire_noop(struct i915_gem_active *, - struct i915_request *request); - -/** - * init_request_active - prepares the activity tracker for use - * @active - the active tracker - * @func - a callback when then the tracker is retired (becomes idle), - * can be NULL - * - * init_request_active() prepares the embedded @active struct for use as - * an activity tracker, that is for tracking the last known active request - * associated with it. When the last request becomes idle, when it is retired - * after completion, the optional callback @func is invoked. - */ -static inline void -init_request_active(struct i915_gem_active *active, - i915_gem_retire_fn retire) +static inline bool i915_request_is_active(const struct i915_request *rq) { - RCU_INIT_POINTER(active->request, NULL); - INIT_LIST_HEAD(&active->link); - active->retire = retire ?: i915_gem_retire_noop; + return test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags); } /** - * i915_gem_active_set - updates the tracker to watch the current request - * @active - the active tracker - * @request - the request to watch - * - * i915_gem_active_set() watches the given @request for completion. Whilst - * that @request is busy, the @active reports busy. When that @request is - * retired, the @active tracker is updated to report idle. - */ -static inline void -i915_gem_active_set(struct i915_gem_active *active, - struct i915_request *request) -{ - list_move(&active->link, &request->active_list); - rcu_assign_pointer(active->request, request); -} - -/** - * i915_gem_active_set_retire_fn - updates the retirement callback - * @active - the active tracker - * @fn - the routine called when the request is retired - * @mutex - struct_mutex used to guard retirements - * - * i915_gem_active_set_retire_fn() updates the function pointer that - * is called when the final request associated with the @active tracker - * is retired. + * Returns true if seq1 is later than seq2. */ -static inline void -i915_gem_active_set_retire_fn(struct i915_gem_active *active, - i915_gem_retire_fn fn, - struct mutex *mutex) +static inline bool i915_seqno_passed(u32 seq1, u32 seq2) { - lockdep_assert_held(mutex); - active->retire = fn ?: i915_gem_retire_noop; + return (s32)(seq1 - seq2) >= 0; } -static inline struct i915_request * -__i915_gem_active_peek(const struct i915_gem_active *active) +static inline u32 __hwsp_seqno(const struct i915_request *rq) { - /* - * Inside the error capture (running with the driver in an unknown - * state), we want to bend the rules slightly (a lot). - * - * Work is in progress to make it safer, in the meantime this keeps - * the known issue from spamming the logs. - */ - return rcu_dereference_protected(active->request, 1); + return READ_ONCE(*rq->hwsp_seqno); } /** - * i915_gem_active_raw - return the active request - * @active - the active tracker + * hwsp_seqno - the current breadcrumb value in the HW status page + * @rq: the request, to chase the relevant HW status page * - * i915_gem_active_raw() returns the current request being tracked, or NULL. - * It does not obtain a reference on the request for the caller, so the caller - * must hold struct_mutex. - */ -static inline struct i915_request * -i915_gem_active_raw(const struct i915_gem_active *active, struct mutex *mutex) -{ - return rcu_dereference_protected(active->request, - lockdep_is_held(mutex)); -} - -/** - * i915_gem_active_peek - report the active request being monitored - * @active - the active tracker + * The emphasis in naming here is that hwsp_seqno() is not a property of the + * request, but an indication of the current HW state (associated with this + * request). Its value will change as the GPU executes more requests. * - * i915_gem_active_peek() returns the current request being tracked if - * still active, or NULL. It does not obtain a reference on the request - * for the caller, so the caller must hold struct_mutex. + * Returns the current breadcrumb value in the associated HW status page (or + * the local timeline's equivalent) for this request. The request itself + * has the associated breadcrumb value of rq->fence.seqno, when the HW + * status page has that breadcrumb or later, this request is complete. */ -static inline struct i915_request * -i915_gem_active_peek(const struct i915_gem_active *active, struct mutex *mutex) +static inline u32 hwsp_seqno(const struct i915_request *rq) { - struct i915_request *request; + u32 seqno; - request = i915_gem_active_raw(active, mutex); - if (!request || i915_request_completed(request)) - return NULL; + rcu_read_lock(); /* the HWSP may be freed at runtime */ + seqno = __hwsp_seqno(rq); + rcu_read_unlock(); - return request; + return seqno; } -/** - * i915_gem_active_get - return a reference to the active request - * @active - the active tracker - * - * i915_gem_active_get() returns a reference to the active request, or NULL - * if the active tracker is idle. The caller must hold struct_mutex. - */ -static inline struct i915_request * -i915_gem_active_get(const struct i915_gem_active *active, struct mutex *mutex) +static inline bool __i915_request_has_started(const struct i915_request *rq) { - return i915_request_get(i915_gem_active_peek(active, mutex)); + return i915_seqno_passed(hwsp_seqno(rq), rq->fence.seqno - 1); } /** - * __i915_gem_active_get_rcu - return a reference to the active request - * @active - the active tracker - * - * __i915_gem_active_get() returns a reference to the active request, or NULL - * if the active tracker is idle. The caller must hold the RCU read lock, but - * the returned pointer is safe to use outside of RCU. - */ -static inline struct i915_request * -__i915_gem_active_get_rcu(const struct i915_gem_active *active) -{ - /* - * Performing a lockless retrieval of the active request is super - * tricky. SLAB_TYPESAFE_BY_RCU merely guarantees that the backing - * slab of request objects will not be freed whilst we hold the - * RCU read lock. It does not guarantee that the request itself - * will not be freed and then *reused*. Viz, - * - * Thread A Thread B - * - * rq = active.request - * retire(rq) -> free(rq); - * (rq is now first on the slab freelist) - * active.request = NULL - * - * rq = new submission on a new object - * ref(rq) - * - * To prevent the request from being reused whilst the caller - * uses it, we take a reference like normal. Whilst acquiring - * the reference we check that it is not in a destroyed state - * (refcnt == 0). That prevents the request being reallocated - * whilst the caller holds on to it. To check that the request - * was not reallocated as we acquired the reference we have to - * check that our request remains the active request across - * the lookup, in the same manner as a seqlock. The visibility - * of the pointer versus the reference counting is controlled - * by using RCU barriers (rcu_dereference and rcu_assign_pointer). - * - * In the middle of all that, we inspect whether the request is - * complete. Retiring is lazy so the request may be completed long - * before the active tracker is updated. Querying whether the - * request is complete is far cheaper (as it involves no locked - * instructions setting cachelines to exclusive) than acquiring - * the reference, so we do it first. The RCU read lock ensures the - * pointer dereference is valid, but does not ensure that the - * seqno nor HWS is the right one! However, if the request was - * reallocated, that means the active tracker's request was complete. - * If the new request is also complete, then both are and we can - * just report the active tracker is idle. If the new request is - * incomplete, then we acquire a reference on it and check that - * it remained the active request. - * - * It is then imperative that we do not zero the request on - * reallocation, so that we can chase the dangling pointers! - * See i915_request_alloc(). - */ - do { - struct i915_request *request; - - request = rcu_dereference(active->request); - if (!request || i915_request_completed(request)) - return NULL; - - /* - * An especially silly compiler could decide to recompute the - * result of i915_request_completed, more specifically - * re-emit the load for request->fence.seqno. A race would catch - * a later seqno value, which could flip the result from true to - * false. Which means part of the instructions below might not - * be executed, while later on instructions are executed. Due to - * barriers within the refcounting the inconsistency can't reach - * past the call to i915_request_get_rcu, but not executing - * that while still executing i915_request_put() creates - * havoc enough. Prevent this with a compiler barrier. - */ - barrier(); - - request = i915_request_get_rcu(request); - - /* - * What stops the following rcu_access_pointer() from occurring - * before the above i915_request_get_rcu()? If we were - * to read the value before pausing to get the reference to - * the request, we may not notice a change in the active - * tracker. - * - * The rcu_access_pointer() is a mere compiler barrier, which - * means both the CPU and compiler are free to perform the - * memory read without constraint. The compiler only has to - * ensure that any operations after the rcu_access_pointer() - * occur afterwards in program order. This means the read may - * be performed earlier by an out-of-order CPU, or adventurous - * compiler. - * - * The atomic operation at the heart of - * i915_request_get_rcu(), see dma_fence_get_rcu(), is - * atomic_inc_not_zero() which is only a full memory barrier - * when successful. That is, if i915_request_get_rcu() - * returns the request (and so with the reference counted - * incremented) then the following read for rcu_access_pointer() - * must occur after the atomic operation and so confirm - * that this request is the one currently being tracked. - * - * The corresponding write barrier is part of - * rcu_assign_pointer(). - */ - if (!request || request == rcu_access_pointer(active->request)) - return rcu_pointer_handoff(request); - - i915_request_put(request); - } while (1); -} - -/** - * i915_gem_active_get_unlocked - return a reference to the active request - * @active - the active tracker - * - * i915_gem_active_get_unlocked() returns a reference to the active request, - * or NULL if the active tracker is idle. The reference is obtained under RCU, - * so no locking is required by the caller. + * i915_request_started - check if the request has begun being executed + * @rq: the request * - * The reference should be freed with i915_request_put(). + * Returns true if the request has been submitted to hardware, and the hardware + * has advanced passed the end of the previous request and so should be either + * currently processing the request (though it may be preempted and so + * not necessarily the next request to complete) or have completed the request. */ -static inline struct i915_request * -i915_gem_active_get_unlocked(const struct i915_gem_active *active) +static inline bool i915_request_started(const struct i915_request *rq) { - struct i915_request *request; + if (i915_request_signaled(rq)) + return true; - rcu_read_lock(); - request = __i915_gem_active_get_rcu(active); - rcu_read_unlock(); - - return request; + /* Remember: started but may have since been preempted! */ + return __i915_request_has_started(rq); } /** - * i915_gem_active_isset - report whether the active tracker is assigned - * @active - the active tracker + * i915_request_is_running - check if the request may actually be executing + * @rq: the request * - * i915_gem_active_isset() returns true if the active tracker is currently - * assigned to a request. Due to the lazy retiring, that request may be idle - * and this may report stale information. + * Returns true if the request is currently submitted to hardware, has passed + * its start point (i.e. the context is setup and not busywaiting). Note that + * it may no longer be running by the time the function returns! */ -static inline bool -i915_gem_active_isset(const struct i915_gem_active *active) +static inline bool i915_request_is_running(const struct i915_request *rq) { - return rcu_access_pointer(active->request); + if (!i915_request_is_active(rq)) + return false; + + return __i915_request_has_started(rq); } -/** - * i915_gem_active_wait - waits until the request is completed - * @active - the active request on which to wait - * @flags - how to wait - * @timeout - how long to wait at most - * @rps - userspace client to charge for a waitboost - * - * i915_gem_active_wait() waits until the request is completed before - * returning, without requiring any locks to be held. Note that it does not - * retire any requests before returning. - * - * This function relies on RCU in order to acquire the reference to the active - * request without holding any locks. See __i915_gem_active_get_rcu() for the - * glory details on how that is managed. Once the reference is acquired, we - * can then wait upon the request, and afterwards release our reference, - * free of any locking. - * - * This function wraps i915_request_wait(), see it for the full details on - * the arguments. - * - * Returns 0 if successful, or a negative error code. - */ -static inline int -i915_gem_active_wait(const struct i915_gem_active *active, unsigned int flags) +static inline bool i915_request_completed(const struct i915_request *rq) { - struct i915_request *request; - long ret = 0; - - request = i915_gem_active_get_unlocked(active); - if (request) { - ret = i915_request_wait(request, flags, MAX_SCHEDULE_TIMEOUT); - i915_request_put(request); - } + if (i915_request_signaled(rq)) + return true; - return ret < 0 ? ret : 0; + return i915_seqno_passed(hwsp_seqno(rq), rq->fence.seqno); } -/** - * i915_gem_active_retire - waits until the request is retired - * @active - the active request on which to wait - * - * i915_gem_active_retire() waits until the request is completed, - * and then ensures that at least the retirement handler for this - * @active tracker is called before returning. If the @active - * tracker is idle, the function returns immediately. - */ -static inline int __must_check -i915_gem_active_retire(struct i915_gem_active *active, - struct mutex *mutex) +static inline void i915_request_mark_complete(struct i915_request *rq) { - struct i915_request *request; - long ret; - - request = i915_gem_active_raw(active, mutex); - if (!request) - return 0; - - ret = i915_request_wait(request, - I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED, - MAX_SCHEDULE_TIMEOUT); - if (ret < 0) - return ret; - - list_del_init(&active->link); - RCU_INIT_POINTER(active->request, NULL); - - active->retire(active, request); - - return 0; + rq->hwsp_seqno = (u32 *)&rq->fence.seqno; /* decouple from HWSP */ } -#define for_each_active(mask, idx) \ - for (; mask ? idx = ffs(mask) - 1, 1 : 0; mask &= ~BIT(idx)) +void i915_retire_requests(struct drm_i915_private *i915); #endif /* I915_REQUEST_H */ diff --git a/drivers/gpu/drm/i915/i915_reset.c b/drivers/gpu/drm/i915/i915_reset.c new file mode 100644 index 000000000000..0e0ddf2e6815 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_reset.c @@ -0,0 +1,1349 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright © 2008-2018 Intel Corporation + */ + +#include <linux/sched/mm.h> +#include <linux/stop_machine.h> + +#include "i915_drv.h" +#include "i915_gpu_error.h" +#include "i915_reset.h" + +#include "intel_guc.h" + +#define RESET_MAX_RETRIES 3 + +/* XXX How to handle concurrent GGTT updates using tiling registers? */ +#define RESET_UNDER_STOP_MACHINE 0 + +static void engine_skip_context(struct i915_request *rq) +{ + struct intel_engine_cs *engine = rq->engine; + struct i915_gem_context *hung_ctx = rq->gem_context; + struct i915_timeline *timeline = rq->timeline; + + lockdep_assert_held(&engine->timeline.lock); + GEM_BUG_ON(timeline == &engine->timeline); + + spin_lock(&timeline->lock); + + if (i915_request_is_active(rq)) { + list_for_each_entry_continue(rq, + &engine->timeline.requests, link) + if (rq->gem_context == hung_ctx) + i915_request_skip(rq, -EIO); + } + + list_for_each_entry(rq, &timeline->requests, link) + i915_request_skip(rq, -EIO); + + spin_unlock(&timeline->lock); +} + +static void client_mark_guilty(struct drm_i915_file_private *file_priv, + const struct i915_gem_context *ctx) +{ + unsigned int score; + unsigned long prev_hang; + + if (i915_gem_context_is_banned(ctx)) + score = I915_CLIENT_SCORE_CONTEXT_BAN; + else + score = 0; + + prev_hang = xchg(&file_priv->hang_timestamp, jiffies); + if (time_before(jiffies, prev_hang + I915_CLIENT_FAST_HANG_JIFFIES)) + score += I915_CLIENT_SCORE_HANG_FAST; + + if (score) { + atomic_add(score, &file_priv->ban_score); + + DRM_DEBUG_DRIVER("client %s: gained %u ban score, now %u\n", + ctx->name, score, + atomic_read(&file_priv->ban_score)); + } +} + +static bool context_mark_guilty(struct i915_gem_context *ctx) +{ + unsigned int score; + bool banned, bannable; + + atomic_inc(&ctx->guilty_count); + + bannable = i915_gem_context_is_bannable(ctx); + score = atomic_add_return(CONTEXT_SCORE_GUILTY, &ctx->ban_score); + banned = score >= CONTEXT_SCORE_BAN_THRESHOLD; + + /* Cool contexts don't accumulate client ban score */ + if (!bannable) + return false; + + if (banned) { + DRM_DEBUG_DRIVER("context %s: guilty %d, score %u, banned\n", + ctx->name, atomic_read(&ctx->guilty_count), + score); + i915_gem_context_set_banned(ctx); + } + + if (!IS_ERR_OR_NULL(ctx->file_priv)) + client_mark_guilty(ctx->file_priv, ctx); + + return banned; +} + +static void context_mark_innocent(struct i915_gem_context *ctx) +{ + atomic_inc(&ctx->active_count); +} + +void i915_reset_request(struct i915_request *rq, bool guilty) +{ + lockdep_assert_held(&rq->engine->timeline.lock); + GEM_BUG_ON(i915_request_completed(rq)); + + if (guilty) { + i915_request_skip(rq, -EIO); + if (context_mark_guilty(rq->gem_context)) + engine_skip_context(rq); + } else { + dma_fence_set_error(&rq->fence, -EAGAIN); + context_mark_innocent(rq->gem_context); + } +} + +static void gen3_stop_engine(struct intel_engine_cs *engine) +{ + struct drm_i915_private *dev_priv = engine->i915; + const u32 base = engine->mmio_base; + + if (intel_engine_stop_cs(engine)) + DRM_DEBUG_DRIVER("%s: timed out on STOP_RING\n", engine->name); + + I915_WRITE_FW(RING_HEAD(base), I915_READ_FW(RING_TAIL(base))); + POSTING_READ_FW(RING_HEAD(base)); /* paranoia */ + + I915_WRITE_FW(RING_HEAD(base), 0); + I915_WRITE_FW(RING_TAIL(base), 0); + POSTING_READ_FW(RING_TAIL(base)); + + /* The ring must be empty before it is disabled */ + I915_WRITE_FW(RING_CTL(base), 0); + + /* Check acts as a post */ + if (I915_READ_FW(RING_HEAD(base)) != 0) + DRM_DEBUG_DRIVER("%s: ring head not parked\n", + engine->name); +} + +static void i915_stop_engines(struct drm_i915_private *i915, + unsigned int engine_mask) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + + if (INTEL_GEN(i915) < 3) + return; + + for_each_engine_masked(engine, i915, engine_mask, id) + gen3_stop_engine(engine); +} + +static bool i915_in_reset(struct pci_dev *pdev) +{ + u8 gdrst; + + pci_read_config_byte(pdev, I915_GDRST, &gdrst); + return gdrst & GRDOM_RESET_STATUS; +} + +static int i915_do_reset(struct drm_i915_private *i915, + unsigned int engine_mask, + unsigned int retry) +{ + struct pci_dev *pdev = i915->drm.pdev; + int err; + + /* Assert reset for at least 20 usec, and wait for acknowledgement. */ + pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE); + udelay(50); + err = wait_for_atomic(i915_in_reset(pdev), 50); + + /* Clear the reset request. */ + pci_write_config_byte(pdev, I915_GDRST, 0); + udelay(50); + if (!err) + err = wait_for_atomic(!i915_in_reset(pdev), 50); + + return err; +} + +static bool g4x_reset_complete(struct pci_dev *pdev) +{ + u8 gdrst; + + pci_read_config_byte(pdev, I915_GDRST, &gdrst); + return (gdrst & GRDOM_RESET_ENABLE) == 0; +} + +static int g33_do_reset(struct drm_i915_private *i915, + unsigned int engine_mask, + unsigned int retry) +{ + struct pci_dev *pdev = i915->drm.pdev; + + pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE); + return wait_for_atomic(g4x_reset_complete(pdev), 50); +} + +static int g4x_do_reset(struct drm_i915_private *dev_priv, + unsigned int engine_mask, + unsigned int retry) +{ + struct pci_dev *pdev = dev_priv->drm.pdev; + int ret; + + /* WaVcpClkGateDisableForMediaReset:ctg,elk */ + I915_WRITE_FW(VDECCLK_GATE_D, + I915_READ(VDECCLK_GATE_D) | VCP_UNIT_CLOCK_GATE_DISABLE); + POSTING_READ_FW(VDECCLK_GATE_D); + + pci_write_config_byte(pdev, I915_GDRST, + GRDOM_MEDIA | GRDOM_RESET_ENABLE); + ret = wait_for_atomic(g4x_reset_complete(pdev), 50); + if (ret) { + DRM_DEBUG_DRIVER("Wait for media reset failed\n"); + goto out; + } + + pci_write_config_byte(pdev, I915_GDRST, + GRDOM_RENDER | GRDOM_RESET_ENABLE); + ret = wait_for_atomic(g4x_reset_complete(pdev), 50); + if (ret) { + DRM_DEBUG_DRIVER("Wait for render reset failed\n"); + goto out; + } + +out: + pci_write_config_byte(pdev, I915_GDRST, 0); + + I915_WRITE_FW(VDECCLK_GATE_D, + I915_READ(VDECCLK_GATE_D) & ~VCP_UNIT_CLOCK_GATE_DISABLE); + POSTING_READ_FW(VDECCLK_GATE_D); + + return ret; +} + +static int ironlake_do_reset(struct drm_i915_private *dev_priv, + unsigned int engine_mask, + unsigned int retry) +{ + int ret; + + I915_WRITE_FW(ILK_GDSR, ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE); + ret = __intel_wait_for_register_fw(dev_priv, ILK_GDSR, + ILK_GRDOM_RESET_ENABLE, 0, + 5000, 0, + NULL); + if (ret) { + DRM_DEBUG_DRIVER("Wait for render reset failed\n"); + goto out; + } + + I915_WRITE_FW(ILK_GDSR, ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE); + ret = __intel_wait_for_register_fw(dev_priv, ILK_GDSR, + ILK_GRDOM_RESET_ENABLE, 0, + 5000, 0, + NULL); + if (ret) { + DRM_DEBUG_DRIVER("Wait for media reset failed\n"); + goto out; + } + +out: + I915_WRITE_FW(ILK_GDSR, 0); + POSTING_READ_FW(ILK_GDSR); + return ret; +} + +/* Reset the hardware domains (GENX_GRDOM_*) specified by mask */ +static int gen6_hw_domain_reset(struct drm_i915_private *dev_priv, + u32 hw_domain_mask) +{ + int err; + + /* + * GEN6_GDRST is not in the gt power well, no need to check + * for fifo space for the write or forcewake the chip for + * the read + */ + I915_WRITE_FW(GEN6_GDRST, hw_domain_mask); + + /* Wait for the device to ack the reset requests */ + err = __intel_wait_for_register_fw(dev_priv, + GEN6_GDRST, hw_domain_mask, 0, + 500, 0, + NULL); + if (err) + DRM_DEBUG_DRIVER("Wait for 0x%08x engines reset failed\n", + hw_domain_mask); + + return err; +} + +static int gen6_reset_engines(struct drm_i915_private *i915, + unsigned int engine_mask, + unsigned int retry) +{ + struct intel_engine_cs *engine; + const u32 hw_engine_mask[I915_NUM_ENGINES] = { + [RCS] = GEN6_GRDOM_RENDER, + [BCS] = GEN6_GRDOM_BLT, + [VCS] = GEN6_GRDOM_MEDIA, + [VCS2] = GEN8_GRDOM_MEDIA2, + [VECS] = GEN6_GRDOM_VECS, + }; + u32 hw_mask; + + if (engine_mask == ALL_ENGINES) { + hw_mask = GEN6_GRDOM_FULL; + } else { + unsigned int tmp; + + hw_mask = 0; + for_each_engine_masked(engine, i915, engine_mask, tmp) + hw_mask |= hw_engine_mask[engine->id]; + } + + return gen6_hw_domain_reset(i915, hw_mask); +} + +static u32 gen11_lock_sfc(struct drm_i915_private *dev_priv, + struct intel_engine_cs *engine) +{ + u8 vdbox_sfc_access = RUNTIME_INFO(dev_priv)->vdbox_sfc_access; + i915_reg_t sfc_forced_lock, sfc_forced_lock_ack; + u32 sfc_forced_lock_bit, sfc_forced_lock_ack_bit; + i915_reg_t sfc_usage; + u32 sfc_usage_bit; + u32 sfc_reset_bit; + + switch (engine->class) { + case VIDEO_DECODE_CLASS: + if ((BIT(engine->instance) & vdbox_sfc_access) == 0) + return 0; + + sfc_forced_lock = GEN11_VCS_SFC_FORCED_LOCK(engine); + sfc_forced_lock_bit = GEN11_VCS_SFC_FORCED_LOCK_BIT; + + sfc_forced_lock_ack = GEN11_VCS_SFC_LOCK_STATUS(engine); + sfc_forced_lock_ack_bit = GEN11_VCS_SFC_LOCK_ACK_BIT; + + sfc_usage = GEN11_VCS_SFC_LOCK_STATUS(engine); + sfc_usage_bit = GEN11_VCS_SFC_USAGE_BIT; + sfc_reset_bit = GEN11_VCS_SFC_RESET_BIT(engine->instance); + break; + + case VIDEO_ENHANCEMENT_CLASS: + sfc_forced_lock = GEN11_VECS_SFC_FORCED_LOCK(engine); + sfc_forced_lock_bit = GEN11_VECS_SFC_FORCED_LOCK_BIT; + + sfc_forced_lock_ack = GEN11_VECS_SFC_LOCK_ACK(engine); + sfc_forced_lock_ack_bit = GEN11_VECS_SFC_LOCK_ACK_BIT; + + sfc_usage = GEN11_VECS_SFC_USAGE(engine); + sfc_usage_bit = GEN11_VECS_SFC_USAGE_BIT; + sfc_reset_bit = GEN11_VECS_SFC_RESET_BIT(engine->instance); + break; + + default: + return 0; + } + + /* + * Tell the engine that a software reset is going to happen. The engine + * will then try to force lock the SFC (if currently locked, it will + * remain so until we tell the engine it is safe to unlock; if currently + * unlocked, it will ignore this and all new lock requests). If SFC + * ends up being locked to the engine we want to reset, we have to reset + * it as well (we will unlock it once the reset sequence is completed). + */ + I915_WRITE_FW(sfc_forced_lock, + I915_READ_FW(sfc_forced_lock) | sfc_forced_lock_bit); + + if (__intel_wait_for_register_fw(dev_priv, + sfc_forced_lock_ack, + sfc_forced_lock_ack_bit, + sfc_forced_lock_ack_bit, + 1000, 0, NULL)) { + DRM_DEBUG_DRIVER("Wait for SFC forced lock ack failed\n"); + return 0; + } + + if (I915_READ_FW(sfc_usage) & sfc_usage_bit) + return sfc_reset_bit; + + return 0; +} + +static void gen11_unlock_sfc(struct drm_i915_private *dev_priv, + struct intel_engine_cs *engine) +{ + u8 vdbox_sfc_access = RUNTIME_INFO(dev_priv)->vdbox_sfc_access; + i915_reg_t sfc_forced_lock; + u32 sfc_forced_lock_bit; + + switch (engine->class) { + case VIDEO_DECODE_CLASS: + if ((BIT(engine->instance) & vdbox_sfc_access) == 0) + return; + + sfc_forced_lock = GEN11_VCS_SFC_FORCED_LOCK(engine); + sfc_forced_lock_bit = GEN11_VCS_SFC_FORCED_LOCK_BIT; + break; + + case VIDEO_ENHANCEMENT_CLASS: + sfc_forced_lock = GEN11_VECS_SFC_FORCED_LOCK(engine); + sfc_forced_lock_bit = GEN11_VECS_SFC_FORCED_LOCK_BIT; + break; + + default: + return; + } + + I915_WRITE_FW(sfc_forced_lock, + I915_READ_FW(sfc_forced_lock) & ~sfc_forced_lock_bit); +} + +static int gen11_reset_engines(struct drm_i915_private *i915, + unsigned int engine_mask, + unsigned int retry) +{ + const u32 hw_engine_mask[I915_NUM_ENGINES] = { + [RCS] = GEN11_GRDOM_RENDER, + [BCS] = GEN11_GRDOM_BLT, + [VCS] = GEN11_GRDOM_MEDIA, + [VCS2] = GEN11_GRDOM_MEDIA2, + [VCS3] = GEN11_GRDOM_MEDIA3, + [VCS4] = GEN11_GRDOM_MEDIA4, + [VECS] = GEN11_GRDOM_VECS, + [VECS2] = GEN11_GRDOM_VECS2, + }; + struct intel_engine_cs *engine; + unsigned int tmp; + u32 hw_mask; + int ret; + + BUILD_BUG_ON(VECS2 + 1 != I915_NUM_ENGINES); + + if (engine_mask == ALL_ENGINES) { + hw_mask = GEN11_GRDOM_FULL; + } else { + hw_mask = 0; + for_each_engine_masked(engine, i915, engine_mask, tmp) { + hw_mask |= hw_engine_mask[engine->id]; + hw_mask |= gen11_lock_sfc(i915, engine); + } + } + + ret = gen6_hw_domain_reset(i915, hw_mask); + + if (engine_mask != ALL_ENGINES) + for_each_engine_masked(engine, i915, engine_mask, tmp) + gen11_unlock_sfc(i915, engine); + + return ret; +} + +static int gen8_engine_reset_prepare(struct intel_engine_cs *engine) +{ + struct drm_i915_private *dev_priv = engine->i915; + int ret; + + I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base), + _MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET)); + + ret = __intel_wait_for_register_fw(dev_priv, + RING_RESET_CTL(engine->mmio_base), + RESET_CTL_READY_TO_RESET, + RESET_CTL_READY_TO_RESET, + 700, 0, + NULL); + if (ret) + DRM_ERROR("%s: reset request timeout\n", engine->name); + + return ret; +} + +static void gen8_engine_reset_cancel(struct intel_engine_cs *engine) +{ + struct drm_i915_private *dev_priv = engine->i915; + + I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base), + _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET)); +} + +static int gen8_reset_engines(struct drm_i915_private *i915, + unsigned int engine_mask, + unsigned int retry) +{ + struct intel_engine_cs *engine; + const bool reset_non_ready = retry >= 1; + unsigned int tmp; + int ret; + + for_each_engine_masked(engine, i915, engine_mask, tmp) { + ret = gen8_engine_reset_prepare(engine); + if (ret && !reset_non_ready) + goto skip_reset; + + /* + * If this is not the first failed attempt to prepare, + * we decide to proceed anyway. + * + * By doing so we risk context corruption and with + * some gens (kbl), possible system hang if reset + * happens during active bb execution. + * + * We rather take context corruption instead of + * failed reset with a wedged driver/gpu. And + * active bb execution case should be covered by + * i915_stop_engines we have before the reset. + */ + } + + if (INTEL_GEN(i915) >= 11) + ret = gen11_reset_engines(i915, engine_mask, retry); + else + ret = gen6_reset_engines(i915, engine_mask, retry); + +skip_reset: + for_each_engine_masked(engine, i915, engine_mask, tmp) + gen8_engine_reset_cancel(engine); + + return ret; +} + +typedef int (*reset_func)(struct drm_i915_private *, + unsigned int engine_mask, + unsigned int retry); + +static reset_func intel_get_gpu_reset(struct drm_i915_private *i915) +{ + if (!i915_modparams.reset) + return NULL; + + if (INTEL_GEN(i915) >= 8) + return gen8_reset_engines; + else if (INTEL_GEN(i915) >= 6) + return gen6_reset_engines; + else if (INTEL_GEN(i915) >= 5) + return ironlake_do_reset; + else if (IS_G4X(i915)) + return g4x_do_reset; + else if (IS_G33(i915) || IS_PINEVIEW(i915)) + return g33_do_reset; + else if (INTEL_GEN(i915) >= 3) + return i915_do_reset; + else + return NULL; +} + +int intel_gpu_reset(struct drm_i915_private *i915, unsigned int engine_mask) +{ + const int retries = engine_mask == ALL_ENGINES ? RESET_MAX_RETRIES : 1; + reset_func reset; + int ret = -ETIMEDOUT; + int retry; + + reset = intel_get_gpu_reset(i915); + if (!reset) + return -ENODEV; + + /* + * If the power well sleeps during the reset, the reset + * request may be dropped and never completes (causing -EIO). + */ + intel_uncore_forcewake_get(i915, FORCEWAKE_ALL); + for (retry = 0; ret == -ETIMEDOUT && retry < retries; retry++) { + /* + * We stop engines, otherwise we might get failed reset and a + * dead gpu (on elk). Also as modern gpu as kbl can suffer + * from system hang if batchbuffer is progressing when + * the reset is issued, regardless of READY_TO_RESET ack. + * Thus assume it is best to stop engines on all gens + * where we have a gpu reset. + * + * WaKBLVECSSemaphoreWaitPoll:kbl (on ALL_ENGINES) + * + * WaMediaResetMainRingCleanup:ctg,elk (presumably) + * + * FIXME: Wa for more modern gens needs to be validated + */ + i915_stop_engines(i915, engine_mask); + + GEM_TRACE("engine_mask=%x\n", engine_mask); + preempt_disable(); + ret = reset(i915, engine_mask, retry); + preempt_enable(); + } + intel_uncore_forcewake_put(i915, FORCEWAKE_ALL); + + return ret; +} + +bool intel_has_gpu_reset(struct drm_i915_private *i915) +{ + if (USES_GUC(i915)) + return false; + + return intel_get_gpu_reset(i915); +} + +bool intel_has_reset_engine(struct drm_i915_private *i915) +{ + return INTEL_INFO(i915)->has_reset_engine && i915_modparams.reset >= 2; +} + +int intel_reset_guc(struct drm_i915_private *i915) +{ + u32 guc_domain = + INTEL_GEN(i915) >= 11 ? GEN11_GRDOM_GUC : GEN9_GRDOM_GUC; + int ret; + + GEM_BUG_ON(!HAS_GUC(i915)); + + intel_uncore_forcewake_get(i915, FORCEWAKE_ALL); + ret = gen6_hw_domain_reset(i915, guc_domain); + intel_uncore_forcewake_put(i915, FORCEWAKE_ALL); + + return ret; +} + +/* + * Ensure irq handler finishes, and not run again. + * Also return the active request so that we only search for it once. + */ +static void reset_prepare_engine(struct intel_engine_cs *engine) +{ + /* + * During the reset sequence, we must prevent the engine from + * entering RC6. As the context state is undefined until we restart + * the engine, if it does enter RC6 during the reset, the state + * written to the powercontext is undefined and so we may lose + * GPU state upon resume, i.e. fail to restart after a reset. + */ + intel_uncore_forcewake_get(engine->i915, FORCEWAKE_ALL); + engine->reset.prepare(engine); +} + +static void reset_prepare(struct drm_i915_private *i915) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + + for_each_engine(engine, i915, id) + reset_prepare_engine(engine); + + intel_uc_sanitize(i915); +} + +static int gt_reset(struct drm_i915_private *i915, unsigned int stalled_mask) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + int err; + + /* + * Everything depends on having the GTT running, so we need to start + * there. + */ + err = i915_ggtt_enable_hw(i915); + if (err) + return err; + + for_each_engine(engine, i915, id) + intel_engine_reset(engine, stalled_mask & ENGINE_MASK(id)); + + i915_gem_restore_fences(i915); + + return err; +} + +static void reset_finish_engine(struct intel_engine_cs *engine) +{ + engine->reset.finish(engine); + intel_uncore_forcewake_put(engine->i915, FORCEWAKE_ALL); +} + +struct i915_gpu_restart { + struct work_struct work; + struct drm_i915_private *i915; +}; + +static void restart_work(struct work_struct *work) +{ + struct i915_gpu_restart *arg = container_of(work, typeof(*arg), work); + struct drm_i915_private *i915 = arg->i915; + struct intel_engine_cs *engine; + enum intel_engine_id id; + intel_wakeref_t wakeref; + + wakeref = intel_runtime_pm_get(i915); + mutex_lock(&i915->drm.struct_mutex); + WRITE_ONCE(i915->gpu_error.restart, NULL); + + for_each_engine(engine, i915, id) { + struct i915_request *rq; + + /* + * Ostensibily, we always want a context loaded for powersaving, + * so if the engine is idle after the reset, send a request + * to load our scratch kernel_context. + */ + if (!intel_engine_is_idle(engine)) + continue; + + rq = i915_request_alloc(engine, i915->kernel_context); + if (!IS_ERR(rq)) + i915_request_add(rq); + } + + mutex_unlock(&i915->drm.struct_mutex); + intel_runtime_pm_put(i915, wakeref); + + kfree(arg); +} + +static void reset_finish(struct drm_i915_private *i915) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + + for_each_engine(engine, i915, id) + reset_finish_engine(engine); +} + +static void reset_restart(struct drm_i915_private *i915) +{ + struct i915_gpu_restart *arg; + + /* + * Following the reset, ensure that we always reload context for + * powersaving, and to correct engine->last_retired_context. Since + * this requires us to submit a request, queue a worker to do that + * task for us to evade any locking here. + */ + if (READ_ONCE(i915->gpu_error.restart)) + return; + + arg = kmalloc(sizeof(*arg), GFP_KERNEL); + if (arg) { + arg->i915 = i915; + INIT_WORK(&arg->work, restart_work); + + WRITE_ONCE(i915->gpu_error.restart, arg); + queue_work(i915->wq, &arg->work); + } +} + +static void nop_submit_request(struct i915_request *request) +{ + struct intel_engine_cs *engine = request->engine; + unsigned long flags; + + GEM_TRACE("%s fence %llx:%lld -> -EIO\n", + engine->name, request->fence.context, request->fence.seqno); + dma_fence_set_error(&request->fence, -EIO); + + spin_lock_irqsave(&engine->timeline.lock, flags); + __i915_request_submit(request); + i915_request_mark_complete(request); + intel_engine_write_global_seqno(engine, request->global_seqno); + spin_unlock_irqrestore(&engine->timeline.lock, flags); + + intel_engine_queue_breadcrumbs(engine); +} + +void i915_gem_set_wedged(struct drm_i915_private *i915) +{ + struct i915_gpu_error *error = &i915->gpu_error; + struct intel_engine_cs *engine; + enum intel_engine_id id; + + mutex_lock(&error->wedge_mutex); + if (test_bit(I915_WEDGED, &error->flags)) { + mutex_unlock(&error->wedge_mutex); + return; + } + + if (GEM_SHOW_DEBUG() && !intel_engines_are_idle(i915)) { + struct drm_printer p = drm_debug_printer(__func__); + + for_each_engine(engine, i915, id) + intel_engine_dump(engine, &p, "%s\n", engine->name); + } + + GEM_TRACE("start\n"); + + /* + * First, stop submission to hw, but do not yet complete requests by + * rolling the global seqno forward (since this would complete requests + * for which we haven't set the fence error to EIO yet). + */ + for_each_engine(engine, i915, id) + reset_prepare_engine(engine); + + /* Even if the GPU reset fails, it should still stop the engines */ + if (INTEL_GEN(i915) >= 5) + intel_gpu_reset(i915, ALL_ENGINES); + + for_each_engine(engine, i915, id) { + engine->submit_request = nop_submit_request; + engine->schedule = NULL; + } + i915->caps.scheduler = 0; + + /* + * Make sure no request can slip through without getting completed by + * either this call here to intel_engine_write_global_seqno, or the one + * in nop_submit_request. + */ + synchronize_rcu(); + + /* Mark all executing requests as skipped */ + for_each_engine(engine, i915, id) + engine->cancel_requests(engine); + + for_each_engine(engine, i915, id) { + reset_finish_engine(engine); + intel_engine_signal_breadcrumbs(engine); + } + + smp_mb__before_atomic(); + set_bit(I915_WEDGED, &error->flags); + + GEM_TRACE("end\n"); + mutex_unlock(&error->wedge_mutex); + + wake_up_all(&error->reset_queue); +} + +bool i915_gem_unset_wedged(struct drm_i915_private *i915) +{ + struct i915_gpu_error *error = &i915->gpu_error; + struct i915_timeline *tl; + bool ret = false; + + if (!test_bit(I915_WEDGED, &error->flags)) + return true; + + if (!i915->gt.scratch) /* Never full initialised, recovery impossible */ + return false; + + mutex_lock(&error->wedge_mutex); + + GEM_TRACE("start\n"); + + /* + * Before unwedging, make sure that all pending operations + * are flushed and errored out - we may have requests waiting upon + * third party fences. We marked all inflight requests as EIO, and + * every execbuf since returned EIO, for consistency we want all + * the currently pending requests to also be marked as EIO, which + * is done inside our nop_submit_request - and so we must wait. + * + * No more can be submitted until we reset the wedged bit. + */ + mutex_lock(&i915->gt.timelines.mutex); + list_for_each_entry(tl, &i915->gt.timelines.active_list, link) { + struct i915_request *rq; + long timeout; + + rq = i915_active_request_get_unlocked(&tl->last_request); + if (!rq) + continue; + + /* + * We can't use our normal waiter as we want to + * avoid recursively trying to handle the current + * reset. The basic dma_fence_default_wait() installs + * a callback for dma_fence_signal(), which is + * triggered by our nop handler (indirectly, the + * callback enables the signaler thread which is + * woken by the nop_submit_request() advancing the seqno + * and when the seqno passes the fence, the signaler + * then signals the fence waking us up). + */ + timeout = dma_fence_default_wait(&rq->fence, true, + MAX_SCHEDULE_TIMEOUT); + i915_request_put(rq); + if (timeout < 0) { + mutex_unlock(&i915->gt.timelines.mutex); + goto unlock; + } + } + mutex_unlock(&i915->gt.timelines.mutex); + + intel_engines_sanitize(i915, false); + + /* + * Undo nop_submit_request. We prevent all new i915 requests from + * being queued (by disallowing execbuf whilst wedged) so having + * waited for all active requests above, we know the system is idle + * and do not have to worry about a thread being inside + * engine->submit_request() as we swap over. So unlike installing + * the nop_submit_request on reset, we can do this from normal + * context and do not require stop_machine(). + */ + intel_engines_reset_default_submission(i915); + + GEM_TRACE("end\n"); + + smp_mb__before_atomic(); /* complete takeover before enabling execbuf */ + clear_bit(I915_WEDGED, &i915->gpu_error.flags); + ret = true; +unlock: + mutex_unlock(&i915->gpu_error.wedge_mutex); + + return ret; +} + +struct __i915_reset { + struct drm_i915_private *i915; + unsigned int stalled_mask; +}; + +static int __i915_reset__BKL(void *data) +{ + struct __i915_reset *arg = data; + int err; + + err = intel_gpu_reset(arg->i915, ALL_ENGINES); + if (err) + return err; + + return gt_reset(arg->i915, arg->stalled_mask); +} + +#if RESET_UNDER_STOP_MACHINE +/* + * XXX An alternative to using stop_machine would be to park only the + * processes that have a GGTT mmap. By remote parking the threads (SIGSTOP) + * we should be able to prevent their memmory accesses via the lost fence + * registers over the course of the reset without the potential recursive + * of mutexes between the pagefault handler and reset. + * + * See igt/gem_mmap_gtt/hang + */ +#define __do_reset(fn, arg) stop_machine(fn, arg, NULL) +#else +#define __do_reset(fn, arg) fn(arg) +#endif + +static int do_reset(struct drm_i915_private *i915, unsigned int stalled_mask) +{ + struct __i915_reset arg = { i915, stalled_mask }; + int err, i; + + err = __do_reset(__i915_reset__BKL, &arg); + for (i = 0; err && i < RESET_MAX_RETRIES; i++) { + msleep(100); + err = __do_reset(__i915_reset__BKL, &arg); + } + + return err; +} + +/** + * i915_reset - reset chip after a hang + * @i915: #drm_i915_private to reset + * @stalled_mask: mask of the stalled engines with the guilty requests + * @reason: user error message for why we are resetting + * + * Reset the chip. Useful if a hang is detected. Marks the device as wedged + * on failure. + * + * Caller must hold the struct_mutex. + * + * Procedure is fairly simple: + * - reset the chip using the reset reg + * - re-init context state + * - re-init hardware status page + * - re-init ring buffer + * - re-init interrupt state + * - re-init display + */ +void i915_reset(struct drm_i915_private *i915, + unsigned int stalled_mask, + const char *reason) +{ + struct i915_gpu_error *error = &i915->gpu_error; + int ret; + + GEM_TRACE("flags=%lx\n", error->flags); + + might_sleep(); + assert_rpm_wakelock_held(i915); + GEM_BUG_ON(!test_bit(I915_RESET_BACKOFF, &error->flags)); + + /* Clear any previous failed attempts at recovery. Time to try again. */ + if (!i915_gem_unset_wedged(i915)) + return; + + if (reason) + dev_notice(i915->drm.dev, "Resetting chip for %s\n", reason); + error->reset_count++; + + reset_prepare(i915); + + if (!intel_has_gpu_reset(i915)) { + if (i915_modparams.reset) + dev_err(i915->drm.dev, "GPU reset not supported\n"); + else + DRM_DEBUG_DRIVER("GPU reset disabled\n"); + goto error; + } + + if (do_reset(i915, stalled_mask)) { + dev_err(i915->drm.dev, "Failed to reset chip\n"); + goto taint; + } + + intel_overlay_reset(i915); + + /* + * Next we need to restore the context, but we don't use those + * yet either... + * + * Ring buffer needs to be re-initialized in the KMS case, or if X + * was running at the time of the reset (i.e. we weren't VT + * switched away). + */ + ret = i915_gem_init_hw(i915); + if (ret) { + DRM_ERROR("Failed to initialise HW following reset (%d)\n", + ret); + goto error; + } + + i915_queue_hangcheck(i915); + +finish: + reset_finish(i915); + if (!i915_terminally_wedged(error)) + reset_restart(i915); + return; + +taint: + /* + * History tells us that if we cannot reset the GPU now, we + * never will. This then impacts everything that is run + * subsequently. On failing the reset, we mark the driver + * as wedged, preventing further execution on the GPU. + * We also want to go one step further and add a taint to the + * kernel so that any subsequent faults can be traced back to + * this failure. This is important for CI, where if the + * GPU/driver fails we would like to reboot and restart testing + * rather than continue on into oblivion. For everyone else, + * the system should still plod along, but they have been warned! + */ + add_taint(TAINT_WARN, LOCKDEP_STILL_OK); +error: + i915_gem_set_wedged(i915); + goto finish; +} + +static inline int intel_gt_reset_engine(struct drm_i915_private *i915, + struct intel_engine_cs *engine) +{ + return intel_gpu_reset(i915, intel_engine_flag(engine)); +} + +/** + * i915_reset_engine - reset GPU engine to recover from a hang + * @engine: engine to reset + * @msg: reason for GPU reset; or NULL for no dev_notice() + * + * Reset a specific GPU engine. Useful if a hang is detected. + * Returns zero on successful reset or otherwise an error code. + * + * Procedure is: + * - identifies the request that caused the hang and it is dropped + * - reset engine (which will force the engine to idle) + * - re-init/configure engine + */ +int i915_reset_engine(struct intel_engine_cs *engine, const char *msg) +{ + struct i915_gpu_error *error = &engine->i915->gpu_error; + int ret; + + GEM_TRACE("%s flags=%lx\n", engine->name, error->flags); + GEM_BUG_ON(!test_bit(I915_RESET_ENGINE + engine->id, &error->flags)); + + reset_prepare_engine(engine); + + if (msg) + dev_notice(engine->i915->drm.dev, + "Resetting %s for %s\n", engine->name, msg); + error->reset_engine_count[engine->id]++; + + if (!engine->i915->guc.execbuf_client) + ret = intel_gt_reset_engine(engine->i915, engine); + else + ret = intel_guc_reset_engine(&engine->i915->guc, engine); + if (ret) { + /* If we fail here, we expect to fallback to a global reset */ + DRM_DEBUG_DRIVER("%sFailed to reset %s, ret=%d\n", + engine->i915->guc.execbuf_client ? "GuC " : "", + engine->name, ret); + goto out; + } + + /* + * The request that caused the hang is stuck on elsp, we know the + * active request and can drop it, adjust head to skip the offending + * request to resume executing remaining requests in the queue. + */ + intel_engine_reset(engine, true); + + /* + * The engine and its registers (and workarounds in case of render) + * have been reset to their default values. Follow the init_ring + * process to program RING_MODE, HWSP and re-enable submission. + */ + ret = engine->init_hw(engine); + if (ret) + goto out; + +out: + intel_engine_cancel_stop_cs(engine); + reset_finish_engine(engine); + return ret; +} + +static void i915_reset_device(struct drm_i915_private *i915, + u32 engine_mask, + const char *reason) +{ + struct i915_gpu_error *error = &i915->gpu_error; + struct kobject *kobj = &i915->drm.primary->kdev->kobj; + char *error_event[] = { I915_ERROR_UEVENT "=1", NULL }; + char *reset_event[] = { I915_RESET_UEVENT "=1", NULL }; + char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL }; + struct i915_wedge_me w; + + kobject_uevent_env(kobj, KOBJ_CHANGE, error_event); + + DRM_DEBUG_DRIVER("resetting chip\n"); + kobject_uevent_env(kobj, KOBJ_CHANGE, reset_event); + + /* Use a watchdog to ensure that our reset completes */ + i915_wedge_on_timeout(&w, i915, 5 * HZ) { + intel_prepare_reset(i915); + + i915_reset(i915, engine_mask, reason); + + intel_finish_reset(i915); + } + + if (!test_bit(I915_WEDGED, &error->flags)) + kobject_uevent_env(kobj, KOBJ_CHANGE, reset_done_event); +} + +void i915_clear_error_registers(struct drm_i915_private *dev_priv) +{ + u32 eir; + + if (!IS_GEN(dev_priv, 2)) + I915_WRITE(PGTBL_ER, I915_READ(PGTBL_ER)); + + if (INTEL_GEN(dev_priv) < 4) + I915_WRITE(IPEIR, I915_READ(IPEIR)); + else + I915_WRITE(IPEIR_I965, I915_READ(IPEIR_I965)); + + I915_WRITE(EIR, I915_READ(EIR)); + eir = I915_READ(EIR); + if (eir) { + /* + * some errors might have become stuck, + * mask them. + */ + DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir); + I915_WRITE(EMR, I915_READ(EMR) | eir); + I915_WRITE(IIR, I915_MASTER_ERROR_INTERRUPT); + } + + if (INTEL_GEN(dev_priv) >= 8) { + I915_WRITE(GEN8_RING_FAULT_REG, + I915_READ(GEN8_RING_FAULT_REG) & ~RING_FAULT_VALID); + POSTING_READ(GEN8_RING_FAULT_REG); + } else if (INTEL_GEN(dev_priv) >= 6) { + struct intel_engine_cs *engine; + enum intel_engine_id id; + + for_each_engine(engine, dev_priv, id) { + I915_WRITE(RING_FAULT_REG(engine), + I915_READ(RING_FAULT_REG(engine)) & + ~RING_FAULT_VALID); + } + POSTING_READ(RING_FAULT_REG(dev_priv->engine[RCS])); + } +} + +/** + * i915_handle_error - handle a gpu error + * @i915: i915 device private + * @engine_mask: mask representing engines that are hung + * @flags: control flags + * @fmt: Error message format string + * + * Do some basic checking of register state at error time and + * dump it to the syslog. Also call i915_capture_error_state() to make + * sure we get a record and make it available in debugfs. Fire a uevent + * so userspace knows something bad happened (should trigger collection + * of a ring dump etc.). + */ +void i915_handle_error(struct drm_i915_private *i915, + u32 engine_mask, + unsigned long flags, + const char *fmt, ...) +{ + struct intel_engine_cs *engine; + intel_wakeref_t wakeref; + unsigned int tmp; + char error_msg[80]; + char *msg = NULL; + + if (fmt) { + va_list args; + + va_start(args, fmt); + vscnprintf(error_msg, sizeof(error_msg), fmt, args); + va_end(args); + + msg = error_msg; + } + + /* + * In most cases it's guaranteed that we get here with an RPM + * reference held, for example because there is a pending GPU + * request that won't finish until the reset is done. This + * isn't the case at least when we get here by doing a + * simulated reset via debugfs, so get an RPM reference. + */ + wakeref = intel_runtime_pm_get(i915); + + engine_mask &= INTEL_INFO(i915)->ring_mask; + + if (flags & I915_ERROR_CAPTURE) { + i915_capture_error_state(i915, engine_mask, msg); + i915_clear_error_registers(i915); + } + + /* + * Try engine reset when available. We fall back to full reset if + * single reset fails. + */ + if (intel_has_reset_engine(i915) && + !i915_terminally_wedged(&i915->gpu_error)) { + for_each_engine_masked(engine, i915, engine_mask, tmp) { + BUILD_BUG_ON(I915_RESET_MODESET >= I915_RESET_ENGINE); + if (test_and_set_bit(I915_RESET_ENGINE + engine->id, + &i915->gpu_error.flags)) + continue; + + if (i915_reset_engine(engine, msg) == 0) + engine_mask &= ~intel_engine_flag(engine); + + clear_bit(I915_RESET_ENGINE + engine->id, + &i915->gpu_error.flags); + wake_up_bit(&i915->gpu_error.flags, + I915_RESET_ENGINE + engine->id); + } + } + + if (!engine_mask) + goto out; + + /* Full reset needs the mutex, stop any other user trying to do so. */ + if (test_and_set_bit(I915_RESET_BACKOFF, &i915->gpu_error.flags)) { + wait_event(i915->gpu_error.reset_queue, + !test_bit(I915_RESET_BACKOFF, + &i915->gpu_error.flags)); + goto out; + } + + /* Prevent any other reset-engine attempt. */ + for_each_engine(engine, i915, tmp) { + while (test_and_set_bit(I915_RESET_ENGINE + engine->id, + &i915->gpu_error.flags)) + wait_on_bit(&i915->gpu_error.flags, + I915_RESET_ENGINE + engine->id, + TASK_UNINTERRUPTIBLE); + } + + i915_reset_device(i915, engine_mask, msg); + + for_each_engine(engine, i915, tmp) { + clear_bit(I915_RESET_ENGINE + engine->id, + &i915->gpu_error.flags); + } + + clear_bit(I915_RESET_BACKOFF, &i915->gpu_error.flags); + wake_up_all(&i915->gpu_error.reset_queue); + +out: + intel_runtime_pm_put(i915, wakeref); +} + +bool i915_reset_flush(struct drm_i915_private *i915) +{ + int err; + + cancel_delayed_work_sync(&i915->gpu_error.hangcheck_work); + + flush_workqueue(i915->wq); + GEM_BUG_ON(READ_ONCE(i915->gpu_error.restart)); + + mutex_lock(&i915->drm.struct_mutex); + err = i915_gem_wait_for_idle(i915, + I915_WAIT_LOCKED | + I915_WAIT_FOR_IDLE_BOOST, + MAX_SCHEDULE_TIMEOUT); + mutex_unlock(&i915->drm.struct_mutex); + + return !err; +} + +static void i915_wedge_me(struct work_struct *work) +{ + struct i915_wedge_me *w = container_of(work, typeof(*w), work.work); + + dev_err(w->i915->drm.dev, + "%s timed out, cancelling all in-flight rendering.\n", + w->name); + i915_gem_set_wedged(w->i915); +} + +void __i915_init_wedge(struct i915_wedge_me *w, + struct drm_i915_private *i915, + long timeout, + const char *name) +{ + w->i915 = i915; + w->name = name; + + INIT_DELAYED_WORK_ONSTACK(&w->work, i915_wedge_me); + schedule_delayed_work(&w->work, timeout); +} + +void __i915_fini_wedge(struct i915_wedge_me *w) +{ + cancel_delayed_work_sync(&w->work); + destroy_delayed_work_on_stack(&w->work); + w->i915 = NULL; +} diff --git a/drivers/gpu/drm/i915/i915_reset.h b/drivers/gpu/drm/i915/i915_reset.h new file mode 100644 index 000000000000..f2d347f319df --- /dev/null +++ b/drivers/gpu/drm/i915/i915_reset.h @@ -0,0 +1,59 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright © 2008-2018 Intel Corporation + */ + +#ifndef I915_RESET_H +#define I915_RESET_H + +#include <linux/compiler.h> +#include <linux/types.h> + +struct drm_i915_private; +struct intel_engine_cs; +struct intel_guc; + +__printf(4, 5) +void i915_handle_error(struct drm_i915_private *i915, + u32 engine_mask, + unsigned long flags, + const char *fmt, ...); +#define I915_ERROR_CAPTURE BIT(0) + +void i915_clear_error_registers(struct drm_i915_private *i915); + +void i915_reset(struct drm_i915_private *i915, + unsigned int stalled_mask, + const char *reason); +int i915_reset_engine(struct intel_engine_cs *engine, + const char *reason); + +void i915_reset_request(struct i915_request *rq, bool guilty); +bool i915_reset_flush(struct drm_i915_private *i915); + +bool intel_has_gpu_reset(struct drm_i915_private *i915); +bool intel_has_reset_engine(struct drm_i915_private *i915); + +int intel_gpu_reset(struct drm_i915_private *i915, u32 engine_mask); + +int intel_reset_guc(struct drm_i915_private *i915); + +struct i915_wedge_me { + struct delayed_work work; + struct drm_i915_private *i915; + const char *name; +}; + +void __i915_init_wedge(struct i915_wedge_me *w, + struct drm_i915_private *i915, + long timeout, + const char *name); +void __i915_fini_wedge(struct i915_wedge_me *w); + +#define i915_wedge_on_timeout(W, DEV, TIMEOUT) \ + for (__i915_init_wedge((W), (DEV), (TIMEOUT), __func__); \ + (W)->i915; \ + __i915_fini_wedge((W))) + +#endif /* I915_RESET_H */ diff --git a/drivers/gpu/drm/i915/i915_scheduler.c b/drivers/gpu/drm/i915/i915_scheduler.c new file mode 100644 index 000000000000..8bc042551692 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_scheduler.c @@ -0,0 +1,417 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright © 2018 Intel Corporation + */ + +#include <linux/mutex.h> + +#include "i915_drv.h" +#include "i915_request.h" +#include "i915_scheduler.h" + +static DEFINE_SPINLOCK(schedule_lock); + +static const struct i915_request * +node_to_request(const struct i915_sched_node *node) +{ + return container_of(node, const struct i915_request, sched); +} + +static inline bool node_signaled(const struct i915_sched_node *node) +{ + return i915_request_completed(node_to_request(node)); +} + +void i915_sched_node_init(struct i915_sched_node *node) +{ + INIT_LIST_HEAD(&node->signalers_list); + INIT_LIST_HEAD(&node->waiters_list); + INIT_LIST_HEAD(&node->link); + node->attr.priority = I915_PRIORITY_INVALID; +} + +static struct i915_dependency * +i915_dependency_alloc(struct drm_i915_private *i915) +{ + return kmem_cache_alloc(i915->dependencies, GFP_KERNEL); +} + +static void +i915_dependency_free(struct drm_i915_private *i915, + struct i915_dependency *dep) +{ + kmem_cache_free(i915->dependencies, dep); +} + +bool __i915_sched_node_add_dependency(struct i915_sched_node *node, + struct i915_sched_node *signal, + struct i915_dependency *dep, + unsigned long flags) +{ + bool ret = false; + + spin_lock(&schedule_lock); + + if (!node_signaled(signal)) { + INIT_LIST_HEAD(&dep->dfs_link); + list_add(&dep->wait_link, &signal->waiters_list); + list_add(&dep->signal_link, &node->signalers_list); + dep->signaler = signal; + dep->flags = flags; + + ret = true; + } + + spin_unlock(&schedule_lock); + + return ret; +} + +int i915_sched_node_add_dependency(struct drm_i915_private *i915, + struct i915_sched_node *node, + struct i915_sched_node *signal) +{ + struct i915_dependency *dep; + + dep = i915_dependency_alloc(i915); + if (!dep) + return -ENOMEM; + + if (!__i915_sched_node_add_dependency(node, signal, dep, + I915_DEPENDENCY_ALLOC)) + i915_dependency_free(i915, dep); + + return 0; +} + +void i915_sched_node_fini(struct drm_i915_private *i915, + struct i915_sched_node *node) +{ + struct i915_dependency *dep, *tmp; + + GEM_BUG_ON(!list_empty(&node->link)); + + spin_lock(&schedule_lock); + + /* + * Everyone we depended upon (the fences we wait to be signaled) + * should retire before us and remove themselves from our list. + * However, retirement is run independently on each timeline and + * so we may be called out-of-order. + */ + list_for_each_entry_safe(dep, tmp, &node->signalers_list, signal_link) { + GEM_BUG_ON(!node_signaled(dep->signaler)); + GEM_BUG_ON(!list_empty(&dep->dfs_link)); + + list_del(&dep->wait_link); + if (dep->flags & I915_DEPENDENCY_ALLOC) + i915_dependency_free(i915, dep); + } + + /* Remove ourselves from everyone who depends upon us */ + list_for_each_entry_safe(dep, tmp, &node->waiters_list, wait_link) { + GEM_BUG_ON(dep->signaler != node); + GEM_BUG_ON(!list_empty(&dep->dfs_link)); + + list_del(&dep->signal_link); + if (dep->flags & I915_DEPENDENCY_ALLOC) + i915_dependency_free(i915, dep); + } + + spin_unlock(&schedule_lock); +} + +static inline struct i915_priolist *to_priolist(struct rb_node *rb) +{ + return rb_entry(rb, struct i915_priolist, node); +} + +static void assert_priolists(struct intel_engine_execlists * const execlists) +{ + struct rb_node *rb; + long last_prio, i; + + if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) + return; + + GEM_BUG_ON(rb_first_cached(&execlists->queue) != + rb_first(&execlists->queue.rb_root)); + + last_prio = (INT_MAX >> I915_USER_PRIORITY_SHIFT) + 1; + for (rb = rb_first_cached(&execlists->queue); rb; rb = rb_next(rb)) { + const struct i915_priolist *p = to_priolist(rb); + + GEM_BUG_ON(p->priority >= last_prio); + last_prio = p->priority; + + GEM_BUG_ON(!p->used); + for (i = 0; i < ARRAY_SIZE(p->requests); i++) { + if (list_empty(&p->requests[i])) + continue; + + GEM_BUG_ON(!(p->used & BIT(i))); + } + } +} + +struct list_head * +i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio) +{ + struct intel_engine_execlists * const execlists = &engine->execlists; + struct i915_priolist *p; + struct rb_node **parent, *rb; + bool first = true; + int idx, i; + + lockdep_assert_held(&engine->timeline.lock); + assert_priolists(execlists); + + /* buckets sorted from highest [in slot 0] to lowest priority */ + idx = I915_PRIORITY_COUNT - (prio & I915_PRIORITY_MASK) - 1; + prio >>= I915_USER_PRIORITY_SHIFT; + if (unlikely(execlists->no_priolist)) + prio = I915_PRIORITY_NORMAL; + +find_priolist: + /* most positive priority is scheduled first, equal priorities fifo */ + rb = NULL; + parent = &execlists->queue.rb_root.rb_node; + while (*parent) { + rb = *parent; + p = to_priolist(rb); + if (prio > p->priority) { + parent = &rb->rb_left; + } else if (prio < p->priority) { + parent = &rb->rb_right; + first = false; + } else { + goto out; + } + } + + if (prio == I915_PRIORITY_NORMAL) { + p = &execlists->default_priolist; + } else { + p = kmem_cache_alloc(engine->i915->priorities, GFP_ATOMIC); + /* Convert an allocation failure to a priority bump */ + if (unlikely(!p)) { + prio = I915_PRIORITY_NORMAL; /* recurses just once */ + + /* To maintain ordering with all rendering, after an + * allocation failure we have to disable all scheduling. + * Requests will then be executed in fifo, and schedule + * will ensure that dependencies are emitted in fifo. + * There will be still some reordering with existing + * requests, so if userspace lied about their + * dependencies that reordering may be visible. + */ + execlists->no_priolist = true; + goto find_priolist; + } + } + + p->priority = prio; + for (i = 0; i < ARRAY_SIZE(p->requests); i++) + INIT_LIST_HEAD(&p->requests[i]); + rb_link_node(&p->node, rb, parent); + rb_insert_color_cached(&p->node, &execlists->queue, first); + p->used = 0; + +out: + p->used |= BIT(idx); + return &p->requests[idx]; +} + +struct sched_cache { + struct list_head *priolist; +}; + +static struct intel_engine_cs * +sched_lock_engine(const struct i915_sched_node *node, + struct intel_engine_cs *locked, + struct sched_cache *cache) +{ + struct intel_engine_cs *engine = node_to_request(node)->engine; + + GEM_BUG_ON(!locked); + + if (engine != locked) { + spin_unlock(&locked->timeline.lock); + memset(cache, 0, sizeof(*cache)); + spin_lock(&engine->timeline.lock); + } + + return engine; +} + +static bool inflight(const struct i915_request *rq, + const struct intel_engine_cs *engine) +{ + const struct i915_request *active; + + if (!i915_request_is_active(rq)) + return false; + + active = port_request(engine->execlists.port); + return active->hw_context == rq->hw_context; +} + +static void __i915_schedule(struct i915_request *rq, + const struct i915_sched_attr *attr) +{ + struct intel_engine_cs *engine; + struct i915_dependency *dep, *p; + struct i915_dependency stack; + const int prio = attr->priority; + struct sched_cache cache; + LIST_HEAD(dfs); + + /* Needed in order to use the temporary link inside i915_dependency */ + lockdep_assert_held(&schedule_lock); + GEM_BUG_ON(prio == I915_PRIORITY_INVALID); + + if (i915_request_completed(rq)) + return; + + if (prio <= READ_ONCE(rq->sched.attr.priority)) + return; + + stack.signaler = &rq->sched; + list_add(&stack.dfs_link, &dfs); + + /* + * Recursively bump all dependent priorities to match the new request. + * + * A naive approach would be to use recursion: + * static void update_priorities(struct i915_sched_node *node, prio) { + * list_for_each_entry(dep, &node->signalers_list, signal_link) + * update_priorities(dep->signal, prio) + * queue_request(node); + * } + * but that may have unlimited recursion depth and so runs a very + * real risk of overunning the kernel stack. Instead, we build + * a flat list of all dependencies starting with the current request. + * As we walk the list of dependencies, we add all of its dependencies + * to the end of the list (this may include an already visited + * request) and continue to walk onwards onto the new dependencies. The + * end result is a topological list of requests in reverse order, the + * last element in the list is the request we must execute first. + */ + list_for_each_entry(dep, &dfs, dfs_link) { + struct i915_sched_node *node = dep->signaler; + + /* + * Within an engine, there can be no cycle, but we may + * refer to the same dependency chain multiple times + * (redundant dependencies are not eliminated) and across + * engines. + */ + list_for_each_entry(p, &node->signalers_list, signal_link) { + GEM_BUG_ON(p == dep); /* no cycles! */ + + if (node_signaled(p->signaler)) + continue; + + GEM_BUG_ON(p->signaler->attr.priority < node->attr.priority); + if (prio > READ_ONCE(p->signaler->attr.priority)) + list_move_tail(&p->dfs_link, &dfs); + } + } + + /* + * If we didn't need to bump any existing priorities, and we haven't + * yet submitted this request (i.e. there is no potential race with + * execlists_submit_request()), we can set our own priority and skip + * acquiring the engine locks. + */ + if (rq->sched.attr.priority == I915_PRIORITY_INVALID) { + GEM_BUG_ON(!list_empty(&rq->sched.link)); + rq->sched.attr = *attr; + + if (stack.dfs_link.next == stack.dfs_link.prev) + return; + + __list_del_entry(&stack.dfs_link); + } + + memset(&cache, 0, sizeof(cache)); + engine = rq->engine; + spin_lock_irq(&engine->timeline.lock); + + /* Fifo and depth-first replacement ensure our deps execute before us */ + list_for_each_entry_safe_reverse(dep, p, &dfs, dfs_link) { + struct i915_sched_node *node = dep->signaler; + + INIT_LIST_HEAD(&dep->dfs_link); + + engine = sched_lock_engine(node, engine, &cache); + lockdep_assert_held(&engine->timeline.lock); + + /* Recheck after acquiring the engine->timeline.lock */ + if (prio <= node->attr.priority || node_signaled(node)) + continue; + + node->attr.priority = prio; + if (!list_empty(&node->link)) { + if (!cache.priolist) + cache.priolist = + i915_sched_lookup_priolist(engine, + prio); + list_move_tail(&node->link, cache.priolist); + } else { + /* + * If the request is not in the priolist queue because + * it is not yet runnable, then it doesn't contribute + * to our preemption decisions. On the other hand, + * if the request is on the HW, it too is not in the + * queue; but in that case we may still need to reorder + * the inflight requests. + */ + if (!i915_sw_fence_done(&node_to_request(node)->submit)) + continue; + } + + if (prio <= engine->execlists.queue_priority_hint) + continue; + + engine->execlists.queue_priority_hint = prio; + + /* + * If we are already the currently executing context, don't + * bother evaluating if we should preempt ourselves. + */ + if (inflight(node_to_request(node), engine)) + continue; + + /* Defer (tasklet) submission until after all of our updates. */ + tasklet_hi_schedule(&engine->execlists.tasklet); + } + + spin_unlock_irq(&engine->timeline.lock); +} + +void i915_schedule(struct i915_request *rq, const struct i915_sched_attr *attr) +{ + spin_lock(&schedule_lock); + __i915_schedule(rq, attr); + spin_unlock(&schedule_lock); +} + +void i915_schedule_bump_priority(struct i915_request *rq, unsigned int bump) +{ + struct i915_sched_attr attr; + + GEM_BUG_ON(bump & ~I915_PRIORITY_MASK); + + if (READ_ONCE(rq->sched.attr.priority) == I915_PRIORITY_INVALID) + return; + + spin_lock_bh(&schedule_lock); + + attr = rq->sched.attr; + attr.priority |= bump; + __i915_schedule(rq, &attr); + + spin_unlock_bh(&schedule_lock); +} diff --git a/drivers/gpu/drm/i915/i915_scheduler.h b/drivers/gpu/drm/i915/i915_scheduler.h index 70a42220358d..dbe9cb7ecd82 100644 --- a/drivers/gpu/drm/i915/i915_scheduler.h +++ b/drivers/gpu/drm/i915/i915_scheduler.h @@ -8,9 +8,14 @@ #define _I915_SCHEDULER_H_ #include <linux/bitops.h> +#include <linux/kernel.h> #include <uapi/drm/i915_drm.h> +struct drm_i915_private; +struct i915_request; +struct intel_engine_cs; + enum { I915_PRIORITY_MIN = I915_CONTEXT_MIN_USER_PRIORITY - 1, I915_PRIORITY_NORMAL = I915_CONTEXT_DEFAULT_PRIORITY, @@ -19,6 +24,15 @@ enum { I915_PRIORITY_INVALID = INT_MIN }; +#define I915_USER_PRIORITY_SHIFT 2 +#define I915_USER_PRIORITY(x) ((x) << I915_USER_PRIORITY_SHIFT) + +#define I915_PRIORITY_COUNT BIT(I915_USER_PRIORITY_SHIFT) +#define I915_PRIORITY_MASK (I915_PRIORITY_COUNT - 1) + +#define I915_PRIORITY_WAIT ((u8)BIT(0)) +#define I915_PRIORITY_NEWCLIENT ((u8)BIT(1)) + struct i915_sched_attr { /** * @priority: execution and service priority @@ -69,4 +83,26 @@ struct i915_dependency { #define I915_DEPENDENCY_ALLOC BIT(0) }; +void i915_sched_node_init(struct i915_sched_node *node); + +bool __i915_sched_node_add_dependency(struct i915_sched_node *node, + struct i915_sched_node *signal, + struct i915_dependency *dep, + unsigned long flags); + +int i915_sched_node_add_dependency(struct drm_i915_private *i915, + struct i915_sched_node *node, + struct i915_sched_node *signal); + +void i915_sched_node_fini(struct drm_i915_private *i915, + struct i915_sched_node *node); + +void i915_schedule(struct i915_request *request, + const struct i915_sched_attr *attr); + +void i915_schedule_bump_priority(struct i915_request *rq, unsigned int bump); + +struct list_head * +i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio); + #endif /* _I915_SCHEDULER_H_ */ diff --git a/drivers/gpu/drm/i915/i915_selftest.h b/drivers/gpu/drm/i915/i915_selftest.h index a73472dd12fd..207e21b478f2 100644 --- a/drivers/gpu/drm/i915/i915_selftest.h +++ b/drivers/gpu/drm/i915/i915_selftest.h @@ -31,6 +31,7 @@ struct i915_selftest { unsigned long timeout_jiffies; unsigned int timeout_ms; unsigned int random_seed; + char *filter; int mock; int live; }; diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c index 8f3aa4dc0c98..d2f2a9c2fabd 100644 --- a/drivers/gpu/drm/i915/i915_suspend.c +++ b/drivers/gpu/drm/i915/i915_suspend.c @@ -24,7 +24,6 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include <drm/drmP.h> #include <drm/i915_drm.h> #include "intel_drv.h" #include "i915_reg.h" @@ -65,7 +64,7 @@ int i915_save_state(struct drm_i915_private *dev_priv) i915_save_display(dev_priv); - if (IS_GEN4(dev_priv)) + if (IS_GEN(dev_priv, 4)) pci_read_config_word(pdev, GCDGMBUS, &dev_priv->regfile.saveGCDGMBUS); @@ -77,17 +76,17 @@ int i915_save_state(struct drm_i915_private *dev_priv) dev_priv->regfile.saveMI_ARB_STATE = I915_READ(MI_ARB_STATE); /* Scratch space */ - if (IS_GEN2(dev_priv) && IS_MOBILE(dev_priv)) { + if (IS_GEN(dev_priv, 2) && IS_MOBILE(dev_priv)) { for (i = 0; i < 7; i++) { dev_priv->regfile.saveSWF0[i] = I915_READ(SWF0(i)); dev_priv->regfile.saveSWF1[i] = I915_READ(SWF1(i)); } for (i = 0; i < 3; i++) dev_priv->regfile.saveSWF3[i] = I915_READ(SWF3(i)); - } else if (IS_GEN2(dev_priv)) { + } else if (IS_GEN(dev_priv, 2)) { for (i = 0; i < 7; i++) dev_priv->regfile.saveSWF1[i] = I915_READ(SWF1(i)); - } else if (HAS_GMCH_DISPLAY(dev_priv)) { + } else if (HAS_GMCH(dev_priv)) { for (i = 0; i < 16; i++) { dev_priv->regfile.saveSWF0[i] = I915_READ(SWF0(i)); dev_priv->regfile.saveSWF1[i] = I915_READ(SWF1(i)); @@ -108,7 +107,7 @@ int i915_restore_state(struct drm_i915_private *dev_priv) mutex_lock(&dev_priv->drm.struct_mutex); - if (IS_GEN4(dev_priv)) + if (IS_GEN(dev_priv, 4)) pci_write_config_word(pdev, GCDGMBUS, dev_priv->regfile.saveGCDGMBUS); i915_restore_display(dev_priv); @@ -122,17 +121,17 @@ int i915_restore_state(struct drm_i915_private *dev_priv) I915_WRITE(MI_ARB_STATE, dev_priv->regfile.saveMI_ARB_STATE | 0xffff0000); /* Scratch space */ - if (IS_GEN2(dev_priv) && IS_MOBILE(dev_priv)) { + if (IS_GEN(dev_priv, 2) && IS_MOBILE(dev_priv)) { for (i = 0; i < 7; i++) { I915_WRITE(SWF0(i), dev_priv->regfile.saveSWF0[i]); I915_WRITE(SWF1(i), dev_priv->regfile.saveSWF1[i]); } for (i = 0; i < 3; i++) I915_WRITE(SWF3(i), dev_priv->regfile.saveSWF3[i]); - } else if (IS_GEN2(dev_priv)) { + } else if (IS_GEN(dev_priv, 2)) { for (i = 0; i < 7; i++) I915_WRITE(SWF1(i), dev_priv->regfile.saveSWF1[i]); - } else if (HAS_GMCH_DISPLAY(dev_priv)) { + } else if (HAS_GMCH(dev_priv)) { for (i = 0; i < 16; i++) { I915_WRITE(SWF0(i), dev_priv->regfile.saveSWF0[i]); I915_WRITE(SWF1(i), dev_priv->regfile.saveSWF1[i]); diff --git a/drivers/gpu/drm/i915/i915_sw_fence.c b/drivers/gpu/drm/i915/i915_sw_fence.c index 6dbeed079ae5..7c58b049ecb5 100644 --- a/drivers/gpu/drm/i915/i915_sw_fence.c +++ b/drivers/gpu/drm/i915/i915_sw_fence.c @@ -1,10 +1,7 @@ /* - * (C) Copyright 2016 Intel Corporation + * SPDX-License-Identifier: MIT * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; version 2 - * of the License. + * (C) Copyright 2016 Intel Corporation */ #include <linux/slab.h> @@ -393,7 +390,7 @@ static void timer_i915_sw_fence_wake(struct timer_list *t) if (!fence) return; - pr_notice("Asynchronous wait on fence %s:%s:%x timed out (hint:%pS)\n", + pr_notice("Asynchronous wait on fence %s:%s:%llx timed out (hint:%pS)\n", cb->dma->ops->get_driver_name(cb->dma), cb->dma->ops->get_timeline_name(cb->dma), cb->dma->seqno, diff --git a/drivers/gpu/drm/i915/i915_sw_fence.h b/drivers/gpu/drm/i915/i915_sw_fence.h index fe2ef4dadfc6..0e055ea0179f 100644 --- a/drivers/gpu/drm/i915/i915_sw_fence.h +++ b/drivers/gpu/drm/i915/i915_sw_fence.h @@ -1,10 +1,9 @@ /* + * SPDX-License-Identifier: MIT + * * i915_sw_fence.h - library routines for N:M synchronisation points * * Copyright (C) 2016 Intel Corporation - * - * This file is released under the GPLv2. - * */ #ifndef _I915_SW_FENCE_H_ diff --git a/drivers/gpu/drm/i915/i915_syncmap.c b/drivers/gpu/drm/i915/i915_syncmap.c index 58f8d0cc125c..60404dbb2e9f 100644 --- a/drivers/gpu/drm/i915/i915_syncmap.c +++ b/drivers/gpu/drm/i915/i915_syncmap.c @@ -92,7 +92,7 @@ void i915_syncmap_init(struct i915_syncmap **root) { BUILD_BUG_ON_NOT_POWER_OF_2(KSYNCMAP); BUILD_BUG_ON_NOT_POWER_OF_2(SHIFT); - BUILD_BUG_ON(KSYNCMAP > BITS_PER_BYTE * sizeof((*root)->bitmap)); + BUILD_BUG_ON(KSYNCMAP > BITS_PER_TYPE((*root)->bitmap)); *root = NULL; } diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c index e5e6f6bb2b05..41313005af42 100644 --- a/drivers/gpu/drm/i915/i915_sysfs.c +++ b/drivers/gpu/drm/i915/i915_sysfs.c @@ -42,11 +42,11 @@ static inline struct drm_i915_private *kdev_minor_to_i915(struct device *kdev) static u32 calc_residency(struct drm_i915_private *dev_priv, i915_reg_t reg) { - u64 res; + intel_wakeref_t wakeref; + u64 res = 0; - intel_runtime_pm_get(dev_priv); - res = intel_rc6_residency_us(dev_priv, reg); - intel_runtime_pm_put(dev_priv); + with_intel_runtime_pm(dev_priv, wakeref) + res = intel_rc6_residency_us(dev_priv, reg); return DIV_ROUND_CLOSEST_ULL(res, 1000); } @@ -258,9 +258,10 @@ static ssize_t gt_act_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf) { struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); + intel_wakeref_t wakeref; int ret; - intel_runtime_pm_get(dev_priv); + wakeref = intel_runtime_pm_get(dev_priv); mutex_lock(&dev_priv->pcu_lock); if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { @@ -274,7 +275,7 @@ static ssize_t gt_act_freq_mhz_show(struct device *kdev, } mutex_unlock(&dev_priv->pcu_lock); - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, wakeref); return snprintf(buf, PAGE_SIZE, "%d\n", ret); } @@ -354,6 +355,7 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev, { struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); struct intel_rps *rps = &dev_priv->gt_pm.rps; + intel_wakeref_t wakeref; u32 val; ssize_t ret; @@ -361,7 +363,7 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev, if (ret) return ret; - intel_runtime_pm_get(dev_priv); + wakeref = intel_runtime_pm_get(dev_priv); mutex_lock(&dev_priv->pcu_lock); @@ -371,7 +373,7 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev, val > rps->max_freq || val < rps->min_freq_softlimit) { mutex_unlock(&dev_priv->pcu_lock); - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, wakeref); return -EINVAL; } @@ -392,7 +394,7 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev, mutex_unlock(&dev_priv->pcu_lock); - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, wakeref); return ret ?: count; } @@ -412,6 +414,7 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev, { struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); struct intel_rps *rps = &dev_priv->gt_pm.rps; + intel_wakeref_t wakeref; u32 val; ssize_t ret; @@ -419,7 +422,7 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev, if (ret) return ret; - intel_runtime_pm_get(dev_priv); + wakeref = intel_runtime_pm_get(dev_priv); mutex_lock(&dev_priv->pcu_lock); @@ -429,7 +432,7 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev, val > rps->max_freq || val > rps->max_freq_softlimit) { mutex_unlock(&dev_priv->pcu_lock); - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, wakeref); return -EINVAL; } @@ -446,7 +449,7 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev, mutex_unlock(&dev_priv->pcu_lock); - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, wakeref); return ret ?: count; } @@ -483,7 +486,7 @@ static ssize_t gt_rp_mhz_show(struct device *kdev, struct device_attribute *attr return snprintf(buf, PAGE_SIZE, "%d\n", val); } -static const struct attribute *gen6_attrs[] = { +static const struct attribute * const gen6_attrs[] = { &dev_attr_gt_act_freq_mhz.attr, &dev_attr_gt_cur_freq_mhz.attr, &dev_attr_gt_boost_freq_mhz.attr, @@ -495,7 +498,7 @@ static const struct attribute *gen6_attrs[] = { NULL, }; -static const struct attribute *vlv_attrs[] = { +static const struct attribute * const vlv_attrs[] = { &dev_attr_gt_act_freq_mhz.attr, &dev_attr_gt_cur_freq_mhz.attr, &dev_attr_gt_boost_freq_mhz.attr, @@ -516,26 +519,23 @@ static ssize_t error_state_read(struct file *filp, struct kobject *kobj, { struct device *kdev = kobj_to_dev(kobj); - struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev); - struct drm_i915_error_state_buf error_str; + struct drm_i915_private *i915 = kdev_minor_to_i915(kdev); struct i915_gpu_state *gpu; ssize_t ret; - ret = i915_error_state_buf_init(&error_str, dev_priv, count, off); - if (ret) - return ret; - - gpu = i915_first_error_state(dev_priv); - ret = i915_error_state_to_str(&error_str, gpu); - if (ret) - goto out; - - ret = count < error_str.bytes ? count : error_str.bytes; - memcpy(buf, error_str.buf, ret); + gpu = i915_first_error_state(i915); + if (IS_ERR(gpu)) { + ret = PTR_ERR(gpu); + } else if (gpu) { + ret = i915_gpu_state_copy_to_buffer(gpu, buf, off, count); + i915_gpu_state_put(gpu); + } else { + const char *str = "No error state collected\n"; + size_t len = strlen(str); -out: - i915_gpu_state_put(gpu); - i915_error_state_buf_release(&error_str); + ret = min_t(size_t, count, len - off); + memcpy(buf, str + off, ret); + } return ret; } diff --git a/drivers/gpu/drm/i915/i915_timeline.c b/drivers/gpu/drm/i915/i915_timeline.c index 4667cc08c416..b2202d2e58a2 100644 --- a/drivers/gpu/drm/i915/i915_timeline.c +++ b/drivers/gpu/drm/i915/i915_timeline.c @@ -9,34 +9,199 @@ #include "i915_timeline.h" #include "i915_syncmap.h" -void i915_timeline_init(struct drm_i915_private *i915, - struct i915_timeline *timeline, - const char *name) +struct i915_timeline_hwsp { + struct i915_vma *vma; + struct list_head free_link; + u64 free_bitmap; +}; + +static inline struct i915_timeline_hwsp * +i915_timeline_hwsp(const struct i915_timeline *tl) +{ + return tl->hwsp_ggtt->private; +} + +static struct i915_vma *__hwsp_alloc(struct drm_i915_private *i915) +{ + struct drm_i915_gem_object *obj; + struct i915_vma *vma; + + obj = i915_gem_object_create_internal(i915, PAGE_SIZE); + if (IS_ERR(obj)) + return ERR_CAST(obj); + + i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC); + + vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); + if (IS_ERR(vma)) + i915_gem_object_put(obj); + + return vma; +} + +static struct i915_vma * +hwsp_alloc(struct i915_timeline *timeline, unsigned int *cacheline) { - lockdep_assert_held(&i915->drm.struct_mutex); + struct drm_i915_private *i915 = timeline->i915; + struct i915_gt_timelines *gt = &i915->gt.timelines; + struct i915_timeline_hwsp *hwsp; + + BUILD_BUG_ON(BITS_PER_TYPE(u64) * CACHELINE_BYTES > PAGE_SIZE); + + spin_lock(>->hwsp_lock); + + /* hwsp_free_list only contains HWSP that have available cachelines */ + hwsp = list_first_entry_or_null(>->hwsp_free_list, + typeof(*hwsp), free_link); + if (!hwsp) { + struct i915_vma *vma; + + spin_unlock(>->hwsp_lock); + + hwsp = kmalloc(sizeof(*hwsp), GFP_KERNEL); + if (!hwsp) + return ERR_PTR(-ENOMEM); + + vma = __hwsp_alloc(i915); + if (IS_ERR(vma)) { + kfree(hwsp); + return vma; + } + + vma->private = hwsp; + hwsp->vma = vma; + hwsp->free_bitmap = ~0ull; + + spin_lock(>->hwsp_lock); + list_add(&hwsp->free_link, >->hwsp_free_list); + } + + GEM_BUG_ON(!hwsp->free_bitmap); + *cacheline = __ffs64(hwsp->free_bitmap); + hwsp->free_bitmap &= ~BIT_ULL(*cacheline); + if (!hwsp->free_bitmap) + list_del(&hwsp->free_link); + + spin_unlock(>->hwsp_lock); + + GEM_BUG_ON(hwsp->vma->private != hwsp); + return hwsp->vma; +} + +static void hwsp_free(struct i915_timeline *timeline) +{ + struct i915_gt_timelines *gt = &timeline->i915->gt.timelines; + struct i915_timeline_hwsp *hwsp; + + hwsp = i915_timeline_hwsp(timeline); + if (!hwsp) /* leave global HWSP alone! */ + return; + + spin_lock(>->hwsp_lock); + + /* As a cacheline becomes available, publish the HWSP on the freelist */ + if (!hwsp->free_bitmap) + list_add_tail(&hwsp->free_link, >->hwsp_free_list); + + hwsp->free_bitmap |= BIT_ULL(timeline->hwsp_offset / CACHELINE_BYTES); + + /* And if no one is left using it, give the page back to the system */ + if (hwsp->free_bitmap == ~0ull) { + i915_vma_put(hwsp->vma); + list_del(&hwsp->free_link); + kfree(hwsp); + } + + spin_unlock(>->hwsp_lock); +} + +int i915_timeline_init(struct drm_i915_private *i915, + struct i915_timeline *timeline, + const char *name, + struct i915_vma *hwsp) +{ + void *vaddr; /* * Ideally we want a set of engines on a single leaf as we expect * to mostly be tracking synchronisation between engines. It is not * a huge issue if this is not the case, but we may want to mitigate * any page crossing penalties if they become an issue. + * + * Called during early_init before we know how many engines there are. */ BUILD_BUG_ON(KSYNCMAP < I915_NUM_ENGINES); + timeline->i915 = i915; timeline->name = name; + timeline->pin_count = 0; + timeline->has_initial_breadcrumb = !hwsp; - list_add(&timeline->link, &i915->gt.timelines); + timeline->hwsp_offset = I915_GEM_HWS_SEQNO_ADDR; + if (!hwsp) { + unsigned int cacheline; + + hwsp = hwsp_alloc(timeline, &cacheline); + if (IS_ERR(hwsp)) + return PTR_ERR(hwsp); + + timeline->hwsp_offset = cacheline * CACHELINE_BYTES; + } + timeline->hwsp_ggtt = i915_vma_get(hwsp); + + vaddr = i915_gem_object_pin_map(hwsp->obj, I915_MAP_WB); + if (IS_ERR(vaddr)) { + hwsp_free(timeline); + i915_vma_put(hwsp); + return PTR_ERR(vaddr); + } - /* Called during early_init before we know how many engines there are */ + timeline->hwsp_seqno = + memset(vaddr + timeline->hwsp_offset, 0, CACHELINE_BYTES); timeline->fence_context = dma_fence_context_alloc(1); spin_lock_init(&timeline->lock); - init_request_active(&timeline->last_request, NULL); + INIT_ACTIVE_REQUEST(&timeline->barrier); + INIT_ACTIVE_REQUEST(&timeline->last_request); INIT_LIST_HEAD(&timeline->requests); i915_syncmap_init(&timeline->sync); + + return 0; +} + +void i915_timelines_init(struct drm_i915_private *i915) +{ + struct i915_gt_timelines *gt = &i915->gt.timelines; + + mutex_init(>->mutex); + INIT_LIST_HEAD(>->active_list); + + spin_lock_init(>->hwsp_lock); + INIT_LIST_HEAD(>->hwsp_free_list); + + /* via i915_gem_wait_for_idle() */ + i915_gem_shrinker_taints_mutex(i915, >->mutex); +} + +static void timeline_add_to_active(struct i915_timeline *tl) +{ + struct i915_gt_timelines *gt = &tl->i915->gt.timelines; + + mutex_lock(>->mutex); + list_add(&tl->link, >->active_list); + mutex_unlock(>->mutex); +} + +static void timeline_remove_from_active(struct i915_timeline *tl) +{ + struct i915_gt_timelines *gt = &tl->i915->gt.timelines; + + mutex_lock(>->mutex); + list_del(&tl->link); + mutex_unlock(>->mutex); } /** @@ -51,11 +216,11 @@ void i915_timeline_init(struct drm_i915_private *i915, */ void i915_timelines_park(struct drm_i915_private *i915) { + struct i915_gt_timelines *gt = &i915->gt.timelines; struct i915_timeline *timeline; - lockdep_assert_held(&i915->drm.struct_mutex); - - list_for_each_entry(timeline, &i915->gt.timelines, link) { + mutex_lock(>->mutex); + list_for_each_entry(timeline, >->active_list, link) { /* * All known fences are completed so we can scrap * the current sync point tracking and start afresh, @@ -64,32 +229,88 @@ void i915_timelines_park(struct drm_i915_private *i915) */ i915_syncmap_free(&timeline->sync); } + mutex_unlock(>->mutex); } void i915_timeline_fini(struct i915_timeline *timeline) { + GEM_BUG_ON(timeline->pin_count); GEM_BUG_ON(!list_empty(&timeline->requests)); + GEM_BUG_ON(i915_active_request_isset(&timeline->barrier)); i915_syncmap_free(&timeline->sync); + hwsp_free(timeline); - list_del(&timeline->link); + i915_gem_object_unpin_map(timeline->hwsp_ggtt->obj); + i915_vma_put(timeline->hwsp_ggtt); } struct i915_timeline * -i915_timeline_create(struct drm_i915_private *i915, const char *name) +i915_timeline_create(struct drm_i915_private *i915, + const char *name, + struct i915_vma *global_hwsp) { struct i915_timeline *timeline; + int err; timeline = kzalloc(sizeof(*timeline), GFP_KERNEL); if (!timeline) return ERR_PTR(-ENOMEM); - i915_timeline_init(i915, timeline, name); + err = i915_timeline_init(i915, timeline, name, global_hwsp); + if (err) { + kfree(timeline); + return ERR_PTR(err); + } + kref_init(&timeline->kref); return timeline; } +int i915_timeline_pin(struct i915_timeline *tl) +{ + int err; + + if (tl->pin_count++) + return 0; + GEM_BUG_ON(!tl->pin_count); + + err = i915_vma_pin(tl->hwsp_ggtt, 0, 0, PIN_GLOBAL | PIN_HIGH); + if (err) + goto unpin; + + tl->hwsp_offset = + i915_ggtt_offset(tl->hwsp_ggtt) + + offset_in_page(tl->hwsp_offset); + + timeline_add_to_active(tl); + + return 0; + +unpin: + tl->pin_count = 0; + return err; +} + +void i915_timeline_unpin(struct i915_timeline *tl) +{ + GEM_BUG_ON(!tl->pin_count); + if (--tl->pin_count) + return; + + timeline_remove_from_active(tl); + + /* + * Since this timeline is idle, all bariers upon which we were waiting + * must also be complete and so we can discard the last used barriers + * without loss of information. + */ + i915_syncmap_free(&tl->sync); + + __i915_vma_unpin(tl->hwsp_ggtt); +} + void __i915_timeline_free(struct kref *kref) { struct i915_timeline *timeline = @@ -99,6 +320,16 @@ void __i915_timeline_free(struct kref *kref) kfree(timeline); } +void i915_timelines_fini(struct drm_i915_private *i915) +{ + struct i915_gt_timelines *gt = &i915->gt.timelines; + + GEM_BUG_ON(!list_empty(>->active_list)); + GEM_BUG_ON(!list_empty(>->hwsp_free_list)); + + mutex_destroy(>->mutex); +} + #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) #include "selftests/mock_timeline.c" #include "selftests/i915_timeline.c" diff --git a/drivers/gpu/drm/i915/i915_timeline.h b/drivers/gpu/drm/i915/i915_timeline.h index a2c2c3ab5fb0..7bec7d2e45bf 100644 --- a/drivers/gpu/drm/i915/i915_timeline.h +++ b/drivers/gpu/drm/i915/i915_timeline.h @@ -28,10 +28,14 @@ #include <linux/list.h> #include <linux/kref.h> +#include "i915_active.h" #include "i915_request.h" #include "i915_syncmap.h" #include "i915_utils.h" +struct i915_vma; +struct i915_timeline_hwsp; + struct i915_timeline { u64 fence_context; u32 seqno; @@ -40,6 +44,13 @@ struct i915_timeline { #define TIMELINE_CLIENT 0 /* default subclass */ #define TIMELINE_ENGINE 1 + unsigned int pin_count; + const u32 *hwsp_seqno; + struct i915_vma *hwsp_ggtt; + u32 hwsp_offset; + + bool has_initial_breadcrumb; + /** * List of breadcrumbs associated with GPU requests currently * outstanding. @@ -48,10 +59,10 @@ struct i915_timeline { /* Contains an RCU guarded pointer to the last request. No reference is * held to the request, users must carefully acquire a reference to - * the request using i915_gem_active_get_request_rcu(), or hold the + * the request using i915_active_request_get_request_rcu(), or hold the * struct_mutex. */ - struct i915_gem_active last_request; + struct i915_active_request last_request; /** * We track the most recent seqno that we wait on in every context so @@ -63,28 +74,53 @@ struct i915_timeline { * redundant and we can discard it without loss of generality. */ struct i915_syncmap *sync; + /** - * Separately to the inter-context seqno map above, we track the last - * barrier (e.g. semaphore wait) to the global engine timelines. Note - * that this tracks global_seqno rather than the context.seqno, and - * so it is subject to the limitations of hw wraparound and that we - * may need to revoke global_seqno (on pre-emption). + * Barrier provides the ability to serialize ordering between different + * timelines. + * + * Users can call i915_timeline_set_barrier which will make all + * subsequent submissions to this timeline be executed only after the + * barrier has been completed. */ - u32 global_sync[I915_NUM_ENGINES]; + struct i915_active_request barrier; struct list_head link; const char *name; + struct drm_i915_private *i915; struct kref kref; }; -void i915_timeline_init(struct drm_i915_private *i915, - struct i915_timeline *tl, - const char *name); +int i915_timeline_init(struct drm_i915_private *i915, + struct i915_timeline *tl, + const char *name, + struct i915_vma *hwsp); void i915_timeline_fini(struct i915_timeline *tl); +static inline void +i915_timeline_set_subclass(struct i915_timeline *timeline, + unsigned int subclass) +{ + lockdep_set_subclass(&timeline->lock, subclass); + + /* + * Due to an interesting quirk in lockdep's internal debug tracking, + * after setting a subclass we must ensure the lock is used. Otherwise, + * nr_unused_locks is incremented once too often. + */ +#ifdef CONFIG_DEBUG_LOCK_ALLOC + local_irq_disable(); + lock_map_acquire(&timeline->lock.dep_map); + lock_map_release(&timeline->lock.dep_map); + local_irq_enable(); +#endif +} + struct i915_timeline * -i915_timeline_create(struct drm_i915_private *i915, const char *name); +i915_timeline_create(struct drm_i915_private *i915, + const char *name, + struct i915_vma *global_hwsp); static inline struct i915_timeline * i915_timeline_get(struct i915_timeline *timeline) @@ -123,6 +159,26 @@ static inline bool i915_timeline_sync_is_later(struct i915_timeline *tl, return __i915_timeline_sync_is_later(tl, fence->context, fence->seqno); } +int i915_timeline_pin(struct i915_timeline *tl); +void i915_timeline_unpin(struct i915_timeline *tl); + +void i915_timelines_init(struct drm_i915_private *i915); void i915_timelines_park(struct drm_i915_private *i915); +void i915_timelines_fini(struct drm_i915_private *i915); + +/** + * i915_timeline_set_barrier - orders submission between different timelines + * @timeline: timeline to set the barrier on + * @rq: request after which new submissions can proceed + * + * Sets the passed in request as the serialization point for all subsequent + * submissions on @timeline. Subsequent requests will not be submitted to GPU + * until the barrier has been completed. + */ +static inline int +i915_timeline_set_barrier(struct i915_timeline *tl, struct i915_request *rq) +{ + return i915_active_request_set(&tl->barrier, rq); +} #endif diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h index b50c6b829715..eab313c3163c 100644 --- a/drivers/gpu/drm/i915/i915_trace.h +++ b/drivers/gpu/drm/i915/i915_trace.h @@ -6,7 +6,8 @@ #include <linux/types.h> #include <linux/tracepoint.h> -#include <drm/drmP.h> +#include <drm/drm_drv.h> + #include "i915_drv.h" #include "intel_drv.h" #include "intel_ringbuffer.h" @@ -585,35 +586,6 @@ TRACE_EVENT(i915_gem_evict_vm, TP_printk("dev=%d, vm=%p", __entry->dev, __entry->vm) ); -TRACE_EVENT(i915_gem_ring_sync_to, - TP_PROTO(struct i915_request *to, struct i915_request *from), - TP_ARGS(to, from), - - TP_STRUCT__entry( - __field(u32, dev) - __field(u32, from_class) - __field(u32, from_instance) - __field(u32, to_class) - __field(u32, to_instance) - __field(u32, seqno) - ), - - TP_fast_assign( - __entry->dev = from->i915->drm.primary->index; - __entry->from_class = from->engine->uabi_class; - __entry->from_instance = from->engine->instance; - __entry->to_class = to->engine->uabi_class; - __entry->to_instance = to->engine->instance; - __entry->seqno = from->global_seqno; - ), - - TP_printk("dev=%u, sync-from=%u:%u, sync-to=%u:%u, seqno=%u", - __entry->dev, - __entry->from_class, __entry->from_instance, - __entry->to_class, __entry->to_instance, - __entry->seqno) -); - TRACE_EVENT(i915_request_queue, TP_PROTO(struct i915_request *rq, u32 flags), TP_ARGS(rq, flags), @@ -780,31 +752,6 @@ trace_i915_request_out(struct i915_request *rq) #endif #endif -TRACE_EVENT(intel_engine_notify, - TP_PROTO(struct intel_engine_cs *engine, bool waiters), - TP_ARGS(engine, waiters), - - TP_STRUCT__entry( - __field(u32, dev) - __field(u16, class) - __field(u16, instance) - __field(u32, seqno) - __field(bool, waiters) - ), - - TP_fast_assign( - __entry->dev = engine->i915->drm.primary->index; - __entry->class = engine->uabi_class; - __entry->instance = engine->instance; - __entry->seqno = intel_engine_get_seqno(engine); - __entry->waiters = waiters; - ), - - TP_printk("dev=%u, engine=%u:%u, seqno=%u, waiters=%u", - __entry->dev, __entry->class, __entry->instance, - __entry->seqno, __entry->waiters) -); - DEFINE_EVENT(i915_request, i915_request_retire, TP_PROTO(struct i915_request *rq), TP_ARGS(rq) diff --git a/drivers/gpu/drm/i915/i915_utils.h b/drivers/gpu/drm/i915/i915_utils.h index 395dd2511568..540e20eb032c 100644 --- a/drivers/gpu/drm/i915/i915_utils.h +++ b/drivers/gpu/drm/i915/i915_utils.h @@ -44,16 +44,19 @@ __stringify(x), (long)(x)) #if defined(GCC_VERSION) && GCC_VERSION >= 70000 -#define add_overflows(A, B) \ - __builtin_add_overflow_p((A), (B), (typeof((A) + (B)))0) +#define add_overflows_t(T, A, B) \ + __builtin_add_overflow_p((A), (B), (T)0) #else -#define add_overflows(A, B) ({ \ +#define add_overflows_t(T, A, B) ({ \ typeof(A) a = (A); \ typeof(B) b = (B); \ - a + b < a; \ + (T)(a + b) < a; \ }) #endif +#define add_overflows(A, B) \ + add_overflows_t(typeof((A) + (B)), (A), (B)) + #define range_overflows(start, size, max) ({ \ typeof(start) start__ = (start); \ typeof(size) size__ = (size); \ @@ -68,7 +71,7 @@ /* Note we don't consider signbits :| */ #define overflows_type(x, T) \ - (sizeof(x) > sizeof(T) && (x) >> (sizeof(T) * BITS_PER_BYTE)) + (sizeof(x) > sizeof(T) && (x) >> BITS_PER_TYPE(T)) #define ptr_mask_bits(ptr, n) ({ \ unsigned long __v = (unsigned long)(ptr); \ @@ -120,12 +123,6 @@ static inline u64 ptr_to_u64(const void *ptr) #include <linux/list.h> -static inline int list_is_first(const struct list_head *list, - const struct list_head *head) -{ - return head->next == list; -} - static inline void __list_del_many(struct list_head *head, struct list_head *first) { diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c index 31efc971a3a8..b713bed20c38 100644 --- a/drivers/gpu/drm/i915/i915_vma.c +++ b/drivers/gpu/drm/i915/i915_vma.c @@ -63,24 +63,22 @@ static void vma_print_allocator(struct i915_vma *vma, const char *reason) #endif -struct i915_vma_active { - struct i915_gem_active base; - struct i915_vma *vma; - struct rb_node node; - u64 timeline; -}; - -static void -__i915_vma_retire(struct i915_vma *vma, struct i915_request *rq) +static void obj_bump_mru(struct drm_i915_gem_object *obj) { - struct drm_i915_gem_object *obj = vma->obj; + struct drm_i915_private *i915 = to_i915(obj->base.dev); - GEM_BUG_ON(!i915_vma_is_active(vma)); - if (--vma->active_count) - return; + spin_lock(&i915->mm.obj_lock); + if (obj->bind_count) + list_move_tail(&obj->mm.link, &i915->mm.bound_list); + spin_unlock(&i915->mm.obj_lock); - GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); - list_move_tail(&vma->vm_link, &vma->vm->inactive_list); + obj->mm.dirty = true; /* be paranoid */ +} + +static void __i915_vma_retire(struct i915_active *ref) +{ + struct i915_vma *vma = container_of(ref, typeof(*vma), active); + struct drm_i915_gem_object *obj = vma->obj; GEM_BUG_ON(!i915_gem_object_is_active(obj)); if (--obj->active_count) @@ -93,16 +91,12 @@ __i915_vma_retire(struct i915_vma *vma, struct i915_request *rq) reservation_object_unlock(obj->resv); } - /* Bump our place on the bound list to keep it roughly in LRU order + /* + * Bump our place on the bound list to keep it roughly in LRU order * so that we don't steal from recently used but inactive objects * (unless we are forced to ofc!) */ - spin_lock(&rq->i915->mm.obj_lock); - if (obj->bind_count) - list_move_tail(&obj->mm.link, &rq->i915->mm.bound_list); - spin_unlock(&rq->i915->mm.obj_lock); - - obj->mm.dirty = true; /* be paranoid */ + obj_bump_mru(obj); if (i915_gem_object_has_active_reference(obj)) { i915_gem_object_clear_active_reference(obj); @@ -110,21 +104,6 @@ __i915_vma_retire(struct i915_vma *vma, struct i915_request *rq) } } -static void -i915_vma_retire(struct i915_gem_active *base, struct i915_request *rq) -{ - struct i915_vma_active *active = - container_of(base, typeof(*active), base); - - __i915_vma_retire(active->vma, rq); -} - -static void -i915_vma_last_retire(struct i915_gem_active *base, struct i915_request *rq) -{ - __i915_vma_retire(container_of(base, struct i915_vma, last_active), rq); -} - static struct i915_vma * vma_create(struct drm_i915_gem_object *obj, struct i915_address_space *vm, @@ -140,10 +119,9 @@ vma_create(struct drm_i915_gem_object *obj, if (vma == NULL) return ERR_PTR(-ENOMEM); - vma->active = RB_ROOT; + i915_active_init(vm->i915, &vma->active, __i915_vma_retire); + INIT_ACTIVE_REQUEST(&vma->last_fence); - init_request_active(&vma->last_active, i915_vma_last_retire); - init_request_active(&vma->last_fence, NULL); vma->vm = vm; vma->ops = &vm->vma_ops; vma->obj = obj; @@ -190,33 +168,56 @@ vma_create(struct drm_i915_gem_object *obj, i915_gem_object_get_stride(obj)); GEM_BUG_ON(!is_power_of_2(vma->fence_alignment)); - /* - * We put the GGTT vma at the start of the vma-list, followed - * by the ppGGTT vma. This allows us to break early when - * iterating over only the GGTT vma for an object, see - * for_each_ggtt_vma() - */ vma->flags |= I915_VMA_GGTT; - list_add(&vma->obj_link, &obj->vma_list); - } else { - list_add_tail(&vma->obj_link, &obj->vma_list); } + spin_lock(&obj->vma.lock); + rb = NULL; - p = &obj->vma_tree.rb_node; + p = &obj->vma.tree.rb_node; while (*p) { struct i915_vma *pos; + long cmp; rb = *p; pos = rb_entry(rb, struct i915_vma, obj_node); - if (i915_vma_compare(pos, vm, view) < 0) + + /* + * If the view already exists in the tree, another thread + * already created a matching vma, so return the older instance + * and dispose of ours. + */ + cmp = i915_vma_compare(pos, vm, view); + if (cmp == 0) { + spin_unlock(&obj->vma.lock); + kmem_cache_free(vm->i915->vmas, vma); + return pos; + } + + if (cmp < 0) p = &rb->rb_right; else p = &rb->rb_left; } rb_link_node(&vma->obj_node, rb, p); - rb_insert_color(&vma->obj_node, &obj->vma_tree); + rb_insert_color(&vma->obj_node, &obj->vma.tree); + + if (i915_vma_is_ggtt(vma)) + /* + * We put the GGTT vma at the start of the vma-list, followed + * by the ppGGTT vma. This allows us to break early when + * iterating over only the GGTT vma for an object, see + * for_each_ggtt_vma() + */ + list_add(&vma->obj_link, &obj->vma.list); + else + list_add_tail(&vma->obj_link, &obj->vma.list); + + spin_unlock(&obj->vma.lock); + + mutex_lock(&vm->mutex); list_add(&vma->vm_link, &vm->unbound_list); + mutex_unlock(&vm->mutex); return vma; @@ -232,7 +233,7 @@ vma_lookup(struct drm_i915_gem_object *obj, { struct rb_node *rb; - rb = obj->vma_tree.rb_node; + rb = obj->vma.tree.rb_node; while (rb) { struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node); long cmp; @@ -272,16 +273,18 @@ i915_vma_instance(struct drm_i915_gem_object *obj, { struct i915_vma *vma; - lockdep_assert_held(&obj->base.dev->struct_mutex); GEM_BUG_ON(view && !i915_is_ggtt(vm)); GEM_BUG_ON(vm->closed); + spin_lock(&obj->vma.lock); vma = vma_lookup(obj, vm, view); - if (!vma) + spin_unlock(&obj->vma.lock); + + /* vma_create() will resolve the race if another creates the vma */ + if (unlikely(!vma)) vma = vma_create(obj, vm, view); GEM_BUG_ON(!IS_ERR(vma) && i915_vma_compare(vma, vm, view)); - GEM_BUG_ON(!IS_ERR(vma) && vma_lookup(obj, vm, view) != vma); return vma; } @@ -305,12 +308,12 @@ int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level, GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); GEM_BUG_ON(vma->size > vma->node.size); - if (GEM_WARN_ON(range_overflows(vma->node.start, - vma->node.size, - vma->vm->total))) + if (GEM_DEBUG_WARN_ON(range_overflows(vma->node.start, + vma->node.size, + vma->vm->total))) return -ENODEV; - if (GEM_WARN_ON(!flags)) + if (GEM_DEBUG_WARN_ON(!flags)) return -EINVAL; bind_flags = 0; @@ -659,7 +662,9 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, cache_level)); - list_move_tail(&vma->vm_link, &vma->vm->inactive_list); + mutex_lock(&vma->vm->mutex); + list_move_tail(&vma->vm_link, &vma->vm->bound_list); + mutex_unlock(&vma->vm->mutex); if (vma->obj) { struct drm_i915_gem_object *obj = vma->obj; @@ -692,8 +697,10 @@ i915_vma_remove(struct i915_vma *vma) vma->ops->clear_pages(vma); + mutex_lock(&vma->vm->mutex); drm_mm_remove_node(&vma->node); list_move_tail(&vma->vm_link, &vma->vm->unbound_list); + mutex_unlock(&vma->vm->mutex); /* * Since the unbound list is global, only move to that list if @@ -797,23 +804,27 @@ void i915_vma_reopen(struct i915_vma *vma) static void __i915_vma_destroy(struct i915_vma *vma) { struct drm_i915_private *i915 = vma->vm->i915; - struct i915_vma_active *iter, *n; GEM_BUG_ON(vma->node.allocated); GEM_BUG_ON(vma->fence); - GEM_BUG_ON(i915_gem_active_isset(&vma->last_fence)); + GEM_BUG_ON(i915_active_request_isset(&vma->last_fence)); - list_del(&vma->obj_link); + mutex_lock(&vma->vm->mutex); list_del(&vma->vm_link); - if (vma->obj) - rb_erase(&vma->obj_node, &vma->obj->vma_tree); + mutex_unlock(&vma->vm->mutex); + + if (vma->obj) { + struct drm_i915_gem_object *obj = vma->obj; - rbtree_postorder_for_each_entry_safe(iter, n, &vma->active, node) { - GEM_BUG_ON(i915_gem_active_isset(&iter->base)); - kfree(iter); + spin_lock(&obj->vma.lock); + list_del(&vma->obj_link); + rb_erase(&vma->obj_node, &vma->obj->vma.tree); + spin_unlock(&obj->vma.lock); } + i915_active_fini(&vma->active); + kmem_cache_free(i915->vmas, vma); } @@ -892,109 +903,20 @@ static void export_fence(struct i915_vma *vma, reservation_object_lock(resv, NULL); if (flags & EXEC_OBJECT_WRITE) reservation_object_add_excl_fence(resv, &rq->fence); - else if (reservation_object_reserve_shared(resv) == 0) + else if (reservation_object_reserve_shared(resv, 1) == 0) reservation_object_add_shared_fence(resv, &rq->fence); reservation_object_unlock(resv); } -static struct i915_gem_active *active_instance(struct i915_vma *vma, u64 idx) -{ - struct i915_vma_active *active; - struct rb_node **p, *parent; - struct i915_request *old; - - /* - * We track the most recently used timeline to skip a rbtree search - * for the common case, under typical loads we never need the rbtree - * at all. We can reuse the last_active slot if it is empty, that is - * after the previous activity has been retired, or if the active - * matches the current timeline. - * - * Note that we allow the timeline to be active simultaneously in - * the rbtree and the last_active cache. We do this to avoid having - * to search and replace the rbtree element for a new timeline, with - * the cost being that we must be aware that the vma may be retired - * twice for the same timeline (as the older rbtree element will be - * retired before the new request added to last_active). - */ - old = i915_gem_active_raw(&vma->last_active, - &vma->vm->i915->drm.struct_mutex); - if (!old || old->fence.context == idx) - goto out; - - /* Move the currently active fence into the rbtree */ - idx = old->fence.context; - - parent = NULL; - p = &vma->active.rb_node; - while (*p) { - parent = *p; - - active = rb_entry(parent, struct i915_vma_active, node); - if (active->timeline == idx) - goto replace; - - if (active->timeline < idx) - p = &parent->rb_right; - else - p = &parent->rb_left; - } - - active = kmalloc(sizeof(*active), GFP_KERNEL); - - /* kmalloc may retire the vma->last_active request (thanks shrinker)! */ - if (unlikely(!i915_gem_active_raw(&vma->last_active, - &vma->vm->i915->drm.struct_mutex))) { - kfree(active); - goto out; - } - - if (unlikely(!active)) - return ERR_PTR(-ENOMEM); - - init_request_active(&active->base, i915_vma_retire); - active->vma = vma; - active->timeline = idx; - - rb_link_node(&active->node, parent, p); - rb_insert_color(&active->node, &vma->active); - -replace: - /* - * Overwrite the previous active slot in the rbtree with last_active, - * leaving last_active zeroed. If the previous slot is still active, - * we must be careful as we now only expect to receive one retire - * callback not two, and so much undo the active counting for the - * overwritten slot. - */ - if (i915_gem_active_isset(&active->base)) { - /* Retire ourselves from the old rq->active_list */ - __list_del_entry(&active->base.link); - vma->active_count--; - GEM_BUG_ON(!vma->active_count); - } - GEM_BUG_ON(list_empty(&vma->last_active.link)); - list_replace_init(&vma->last_active.link, &active->base.link); - active->base.request = fetch_and_zero(&vma->last_active.request); - -out: - return &vma->last_active; -} - int i915_vma_move_to_active(struct i915_vma *vma, struct i915_request *rq, unsigned int flags) { struct drm_i915_gem_object *obj = vma->obj; - struct i915_gem_active *active; lockdep_assert_held(&rq->i915->drm.struct_mutex); GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); - active = active_instance(vma, rq->fence.context); - if (IS_ERR(active)) - return PTR_ERR(active); - /* * Add a reference if we're newly entering the active list. * The order in which we add operations to the retirement queue is @@ -1003,11 +925,15 @@ int i915_vma_move_to_active(struct i915_vma *vma, * add the active reference first and queue for it to be dropped * *last*. */ - if (!i915_gem_active_isset(active) && !vma->active_count++) { - list_move_tail(&vma->vm_link, &vma->vm->active_list); + if (!vma->active.count) obj->active_count++; + + if (unlikely(i915_active_ref(&vma->active, rq->fence.context, rq))) { + if (!vma->active.count) + obj->active_count--; + return -ENOMEM; } - i915_gem_active_set(active, rq); + GEM_BUG_ON(!i915_vma_is_active(vma)); GEM_BUG_ON(!obj->active_count); @@ -1016,14 +942,14 @@ int i915_vma_move_to_active(struct i915_vma *vma, obj->write_domain = I915_GEM_DOMAIN_RENDER; if (intel_fb_obj_invalidate(obj, ORIGIN_CS)) - i915_gem_active_set(&obj->frontbuffer_write, rq); + __i915_active_request_set(&obj->frontbuffer_write, rq); obj->read_domains = 0; } obj->read_domains |= I915_GEM_GPU_DOMAINS; if (flags & EXEC_OBJECT_NEEDS_FENCE) - i915_gem_active_set(&vma->last_fence, rq); + __i915_active_request_set(&vma->last_fence, rq); export_fence(vma, rq, flags); return 0; @@ -1041,8 +967,6 @@ int i915_vma_unbind(struct i915_vma *vma) */ might_sleep(); if (i915_vma_is_active(vma)) { - struct i915_vma_active *active, *n; - /* * When a closed VMA is retired, it is unbound - eek. * In order to prevent it from being recursively closed, @@ -1058,21 +982,12 @@ int i915_vma_unbind(struct i915_vma *vma) */ __i915_vma_pin(vma); - ret = i915_gem_active_retire(&vma->last_active, - &vma->vm->i915->drm.struct_mutex); + ret = i915_active_wait(&vma->active); if (ret) goto unpin; - rbtree_postorder_for_each_entry_safe(active, n, - &vma->active, node) { - ret = i915_gem_active_retire(&active->base, - &vma->vm->i915->drm.struct_mutex); - if (ret) - goto unpin; - } - - ret = i915_gem_active_retire(&vma->last_fence, - &vma->vm->i915->drm.struct_mutex); + ret = i915_active_request_retire(&vma->last_fence, + &vma->vm->i915->drm.struct_mutex); unpin: __i915_vma_unpin(vma); if (ret) diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h index 4f7c1c7599f4..7c742027f866 100644 --- a/drivers/gpu/drm/i915/i915_vma.h +++ b/drivers/gpu/drm/i915/i915_vma.h @@ -34,6 +34,7 @@ #include "i915_gem_fence_reg.h" #include "i915_gem_object.h" +#include "i915_active.h" #include "i915_request.h" enum i915_cache_level; @@ -71,34 +72,45 @@ struct i915_vma { unsigned int open_count; unsigned long flags; /** - * How many users have pinned this object in GTT space. The following - * users can each hold at most one reference: pwrite/pread, execbuffer - * (objects are not allowed multiple times for the same batchbuffer), - * and the framebuffer code. When switching/pageflipping, the - * framebuffer code has at most two buffers pinned per crtc. + * How many users have pinned this object in GTT space. * - * In the worst case this is 1 + 1 + 1 + 2*2 = 7. That would fit into 3 - * bits with absolutely no headroom. So use 4 bits. + * This is a tightly bound, fairly small number of users, so we + * stuff inside the flags field so that we can both check for overflow + * and detect a no-op i915_vma_pin() in a single check, while also + * pinning the vma. + * + * The worst case display setup would have the same vma pinned for + * use on each plane on each crtc, while also building the next atomic + * state and holding a pin for the length of the cleanup queue. In the + * future, the flip queue may be increased from 1. + * Estimated worst case: 3 [qlen] * 4 [max crtcs] * 7 [max planes] = 84 + * + * For GEM, the number of concurrent users for pwrite/pread is + * unbounded. For execbuffer, it is currently one but will in future + * be extended to allow multiple clients to pin vma concurrently. + * + * We also use suballocated pages, with each suballocation claiming + * its own pin on the shared vma. At present, this is limited to + * exclusive cachelines of a single page, so a maximum of 64 possible + * users. */ -#define I915_VMA_PIN_MASK 0xf -#define I915_VMA_PIN_OVERFLOW BIT(5) +#define I915_VMA_PIN_MASK 0xff +#define I915_VMA_PIN_OVERFLOW BIT(8) /** Flags and address space this VMA is bound to */ -#define I915_VMA_GLOBAL_BIND BIT(6) -#define I915_VMA_LOCAL_BIND BIT(7) +#define I915_VMA_GLOBAL_BIND BIT(9) +#define I915_VMA_LOCAL_BIND BIT(10) #define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND | I915_VMA_PIN_OVERFLOW) -#define I915_VMA_GGTT BIT(8) -#define I915_VMA_CAN_FENCE BIT(9) -#define I915_VMA_CLOSED BIT(10) -#define I915_VMA_USERFAULT_BIT 11 +#define I915_VMA_GGTT BIT(11) +#define I915_VMA_CAN_FENCE BIT(12) +#define I915_VMA_CLOSED BIT(13) +#define I915_VMA_USERFAULT_BIT 14 #define I915_VMA_USERFAULT BIT(I915_VMA_USERFAULT_BIT) -#define I915_VMA_GGTT_WRITE BIT(12) +#define I915_VMA_GGTT_WRITE BIT(15) - unsigned int active_count; - struct rb_root active; - struct i915_gem_active last_active; - struct i915_gem_active last_fence; + struct i915_active active; + struct i915_active_request last_fence; /** * Support different GGTT views into the same object. @@ -141,9 +153,9 @@ i915_vma_instance(struct drm_i915_gem_object *obj, void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags); #define I915_VMA_RELEASE_MAP BIT(0) -static inline bool i915_vma_is_active(struct i915_vma *vma) +static inline bool i915_vma_is_active(const struct i915_vma *vma) { - return vma->active_count; + return !i915_active_is_idle(&vma->active); } int __must_check i915_vma_move_to_active(struct i915_vma *vma, @@ -425,7 +437,7 @@ void i915_vma_parked(struct drm_i915_private *i915); * or the list is empty ofc. */ #define for_each_ggtt_vma(V, OBJ) \ - list_for_each_entry(V, &(OBJ)->vma_list, obj_link) \ + list_for_each_entry(V, &(OBJ)->vma.list, obj_link) \ for_each_until(!i915_vma_is_ggtt(V)) #endif diff --git a/drivers/gpu/drm/i915/icl_dsi.c b/drivers/gpu/drm/i915/icl_dsi.c index 13830e43a4d1..641e0778fa9c 100644 --- a/drivers/gpu/drm/i915/icl_dsi.c +++ b/drivers/gpu/drm/i915/icl_dsi.c @@ -25,8 +25,277 @@ * Jani Nikula <jani.nikula@intel.com> */ +#include <drm/drm_mipi_dsi.h> +#include <drm/drm_atomic_helper.h> #include "intel_dsi.h" +static inline int header_credits_available(struct drm_i915_private *dev_priv, + enum transcoder dsi_trans) +{ + return (I915_READ(DSI_CMD_TXCTL(dsi_trans)) & FREE_HEADER_CREDIT_MASK) + >> FREE_HEADER_CREDIT_SHIFT; +} + +static inline int payload_credits_available(struct drm_i915_private *dev_priv, + enum transcoder dsi_trans) +{ + return (I915_READ(DSI_CMD_TXCTL(dsi_trans)) & FREE_PLOAD_CREDIT_MASK) + >> FREE_PLOAD_CREDIT_SHIFT; +} + +static void wait_for_header_credits(struct drm_i915_private *dev_priv, + enum transcoder dsi_trans) +{ + if (wait_for_us(header_credits_available(dev_priv, dsi_trans) >= + MAX_HEADER_CREDIT, 100)) + DRM_ERROR("DSI header credits not released\n"); +} + +static void wait_for_payload_credits(struct drm_i915_private *dev_priv, + enum transcoder dsi_trans) +{ + if (wait_for_us(payload_credits_available(dev_priv, dsi_trans) >= + MAX_PLOAD_CREDIT, 100)) + DRM_ERROR("DSI payload credits not released\n"); +} + +static enum transcoder dsi_port_to_transcoder(enum port port) +{ + if (port == PORT_A) + return TRANSCODER_DSI_0; + else + return TRANSCODER_DSI_1; +} + +static void wait_for_cmds_dispatched_to_panel(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + struct mipi_dsi_device *dsi; + enum port port; + enum transcoder dsi_trans; + int ret; + + /* wait for header/payload credits to be released */ + for_each_dsi_port(port, intel_dsi->ports) { + dsi_trans = dsi_port_to_transcoder(port); + wait_for_header_credits(dev_priv, dsi_trans); + wait_for_payload_credits(dev_priv, dsi_trans); + } + + /* send nop DCS command */ + for_each_dsi_port(port, intel_dsi->ports) { + dsi = intel_dsi->dsi_hosts[port]->device; + dsi->mode_flags |= MIPI_DSI_MODE_LPM; + dsi->channel = 0; + ret = mipi_dsi_dcs_nop(dsi); + if (ret < 0) + DRM_ERROR("error sending DCS NOP command\n"); + } + + /* wait for header credits to be released */ + for_each_dsi_port(port, intel_dsi->ports) { + dsi_trans = dsi_port_to_transcoder(port); + wait_for_header_credits(dev_priv, dsi_trans); + } + + /* wait for LP TX in progress bit to be cleared */ + for_each_dsi_port(port, intel_dsi->ports) { + dsi_trans = dsi_port_to_transcoder(port); + if (wait_for_us(!(I915_READ(DSI_LP_MSG(dsi_trans)) & + LPTX_IN_PROGRESS), 20)) + DRM_ERROR("LPTX bit not cleared\n"); + } +} + +static bool add_payld_to_queue(struct intel_dsi_host *host, const u8 *data, + u32 len) +{ + struct intel_dsi *intel_dsi = host->intel_dsi; + struct drm_i915_private *dev_priv = to_i915(intel_dsi->base.base.dev); + enum transcoder dsi_trans = dsi_port_to_transcoder(host->port); + int free_credits; + int i, j; + + for (i = 0; i < len; i += 4) { + u32 tmp = 0; + + free_credits = payload_credits_available(dev_priv, dsi_trans); + if (free_credits < 1) { + DRM_ERROR("Payload credit not available\n"); + return false; + } + + for (j = 0; j < min_t(u32, len - i, 4); j++) + tmp |= *data++ << 8 * j; + + I915_WRITE(DSI_CMD_TXPYLD(dsi_trans), tmp); + } + + return true; +} + +static int dsi_send_pkt_hdr(struct intel_dsi_host *host, + struct mipi_dsi_packet pkt, bool enable_lpdt) +{ + struct intel_dsi *intel_dsi = host->intel_dsi; + struct drm_i915_private *dev_priv = to_i915(intel_dsi->base.base.dev); + enum transcoder dsi_trans = dsi_port_to_transcoder(host->port); + u32 tmp; + int free_credits; + + /* check if header credit available */ + free_credits = header_credits_available(dev_priv, dsi_trans); + if (free_credits < 1) { + DRM_ERROR("send pkt header failed, not enough hdr credits\n"); + return -1; + } + + tmp = I915_READ(DSI_CMD_TXHDR(dsi_trans)); + + if (pkt.payload) + tmp |= PAYLOAD_PRESENT; + else + tmp &= ~PAYLOAD_PRESENT; + + tmp &= ~VBLANK_FENCE; + + if (enable_lpdt) + tmp |= LP_DATA_TRANSFER; + + tmp &= ~(PARAM_WC_MASK | VC_MASK | DT_MASK); + tmp |= ((pkt.header[0] & VC_MASK) << VC_SHIFT); + tmp |= ((pkt.header[0] & DT_MASK) << DT_SHIFT); + tmp |= (pkt.header[1] << PARAM_WC_LOWER_SHIFT); + tmp |= (pkt.header[2] << PARAM_WC_UPPER_SHIFT); + I915_WRITE(DSI_CMD_TXHDR(dsi_trans), tmp); + + return 0; +} + +static int dsi_send_pkt_payld(struct intel_dsi_host *host, + struct mipi_dsi_packet pkt) +{ + /* payload queue can accept *256 bytes*, check limit */ + if (pkt.payload_length > MAX_PLOAD_CREDIT * 4) { + DRM_ERROR("payload size exceeds max queue limit\n"); + return -1; + } + + /* load data into command payload queue */ + if (!add_payld_to_queue(host, pkt.payload, + pkt.payload_length)) { + DRM_ERROR("adding payload to queue failed\n"); + return -1; + } + + return 0; +} + +static void dsi_program_swing_and_deemphasis(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + enum port port; + u32 tmp; + int lane; + + for_each_dsi_port(port, intel_dsi->ports) { + + /* + * Program voltage swing and pre-emphasis level values as per + * table in BSPEC under DDI buffer programing + */ + tmp = I915_READ(ICL_PORT_TX_DW5_LN0(port)); + tmp &= ~(SCALING_MODE_SEL_MASK | RTERM_SELECT_MASK); + tmp |= SCALING_MODE_SEL(0x2); + tmp |= TAP2_DISABLE | TAP3_DISABLE; + tmp |= RTERM_SELECT(0x6); + I915_WRITE(ICL_PORT_TX_DW5_GRP(port), tmp); + + tmp = I915_READ(ICL_PORT_TX_DW5_AUX(port)); + tmp &= ~(SCALING_MODE_SEL_MASK | RTERM_SELECT_MASK); + tmp |= SCALING_MODE_SEL(0x2); + tmp |= TAP2_DISABLE | TAP3_DISABLE; + tmp |= RTERM_SELECT(0x6); + I915_WRITE(ICL_PORT_TX_DW5_AUX(port), tmp); + + tmp = I915_READ(ICL_PORT_TX_DW2_LN0(port)); + tmp &= ~(SWING_SEL_LOWER_MASK | SWING_SEL_UPPER_MASK | + RCOMP_SCALAR_MASK); + tmp |= SWING_SEL_UPPER(0x2); + tmp |= SWING_SEL_LOWER(0x2); + tmp |= RCOMP_SCALAR(0x98); + I915_WRITE(ICL_PORT_TX_DW2_GRP(port), tmp); + + tmp = I915_READ(ICL_PORT_TX_DW2_AUX(port)); + tmp &= ~(SWING_SEL_LOWER_MASK | SWING_SEL_UPPER_MASK | + RCOMP_SCALAR_MASK); + tmp |= SWING_SEL_UPPER(0x2); + tmp |= SWING_SEL_LOWER(0x2); + tmp |= RCOMP_SCALAR(0x98); + I915_WRITE(ICL_PORT_TX_DW2_AUX(port), tmp); + + tmp = I915_READ(ICL_PORT_TX_DW4_AUX(port)); + tmp &= ~(POST_CURSOR_1_MASK | POST_CURSOR_2_MASK | + CURSOR_COEFF_MASK); + tmp |= POST_CURSOR_1(0x0); + tmp |= POST_CURSOR_2(0x0); + tmp |= CURSOR_COEFF(0x3f); + I915_WRITE(ICL_PORT_TX_DW4_AUX(port), tmp); + + for (lane = 0; lane <= 3; lane++) { + /* Bspec: must not use GRP register for write */ + tmp = I915_READ(ICL_PORT_TX_DW4_LN(port, lane)); + tmp &= ~(POST_CURSOR_1_MASK | POST_CURSOR_2_MASK | + CURSOR_COEFF_MASK); + tmp |= POST_CURSOR_1(0x0); + tmp |= POST_CURSOR_2(0x0); + tmp |= CURSOR_COEFF(0x3f); + I915_WRITE(ICL_PORT_TX_DW4_LN(port, lane), tmp); + } + } +} + +static void configure_dual_link_mode(struct intel_encoder *encoder, + const struct intel_crtc_state *pipe_config) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + u32 dss_ctl1; + + dss_ctl1 = I915_READ(DSS_CTL1); + dss_ctl1 |= SPLITTER_ENABLE; + dss_ctl1 &= ~OVERLAP_PIXELS_MASK; + dss_ctl1 |= OVERLAP_PIXELS(intel_dsi->pixel_overlap); + + if (intel_dsi->dual_link == DSI_DUAL_LINK_FRONT_BACK) { + const struct drm_display_mode *adjusted_mode = + &pipe_config->base.adjusted_mode; + u32 dss_ctl2; + u16 hactive = adjusted_mode->crtc_hdisplay; + u16 dl_buffer_depth; + + dss_ctl1 &= ~DUAL_LINK_MODE_INTERLEAVE; + dl_buffer_depth = hactive / 2 + intel_dsi->pixel_overlap; + + if (dl_buffer_depth > MAX_DL_BUFFER_TARGET_DEPTH) + DRM_ERROR("DL buffer depth exceed max value\n"); + + dss_ctl1 &= ~LEFT_DL_BUF_TARGET_DEPTH_MASK; + dss_ctl1 |= LEFT_DL_BUF_TARGET_DEPTH(dl_buffer_depth); + dss_ctl2 = I915_READ(DSS_CTL2); + dss_ctl2 &= ~RIGHT_DL_BUF_TARGET_DEPTH_MASK; + dss_ctl2 |= RIGHT_DL_BUF_TARGET_DEPTH(dl_buffer_depth); + I915_WRITE(DSS_CTL2, dss_ctl2); + } else { + /* Interleave */ + dss_ctl1 |= DUAL_LINK_MODE_INTERLEAVE; + } + + I915_WRITE(DSS_CTL1, dss_ctl1); +} + static void gen11_dsi_program_esc_clk_div(struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); @@ -54,6 +323,21 @@ static void gen11_dsi_program_esc_clk_div(struct intel_encoder *encoder) } } +static void get_dsi_io_power_domains(struct drm_i915_private *dev_priv, + struct intel_dsi *intel_dsi) +{ + enum port port; + + for_each_dsi_port(port, intel_dsi->ports) { + WARN_ON(intel_dsi->io_wakeref[port]); + intel_dsi->io_wakeref[port] = + intel_display_power_get(dev_priv, + port == PORT_A ? + POWER_DOMAIN_PORT_DDI_A_IO : + POWER_DOMAIN_PORT_DDI_B_IO); + } +} + static void gen11_dsi_enable_io_power(struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); @@ -67,11 +351,7 @@ static void gen11_dsi_enable_io_power(struct intel_encoder *encoder) I915_WRITE(ICL_DSI_IO_MODECTL(port), tmp); } - for_each_dsi_port(port, intel_dsi->ports) { - intel_display_power_get(dev_priv, port == PORT_A ? - POWER_DOMAIN_PORT_DDI_A_IO : - POWER_DOMAIN_PORT_DDI_B_IO); - } + get_dsi_io_power_domains(dev_priv, intel_dsi); } static void gen11_dsi_power_up_lanes(struct intel_encoder *encoder) @@ -105,23 +385,1087 @@ static void gen11_dsi_power_up_lanes(struct intel_encoder *encoder) } } -static void gen11_dsi_enable_port_and_phy(struct intel_encoder *encoder) +static void gen11_dsi_config_phy_lanes_sequence(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + enum port port; + u32 tmp; + int lane; + + /* Step 4b(i) set loadgen select for transmit and aux lanes */ + for_each_dsi_port(port, intel_dsi->ports) { + tmp = I915_READ(ICL_PORT_TX_DW4_AUX(port)); + tmp &= ~LOADGEN_SELECT; + I915_WRITE(ICL_PORT_TX_DW4_AUX(port), tmp); + for (lane = 0; lane <= 3; lane++) { + tmp = I915_READ(ICL_PORT_TX_DW4_LN(port, lane)); + tmp &= ~LOADGEN_SELECT; + if (lane != 2) + tmp |= LOADGEN_SELECT; + I915_WRITE(ICL_PORT_TX_DW4_LN(port, lane), tmp); + } + } + + /* Step 4b(ii) set latency optimization for transmit and aux lanes */ + for_each_dsi_port(port, intel_dsi->ports) { + tmp = I915_READ(ICL_PORT_TX_DW2_AUX(port)); + tmp &= ~FRC_LATENCY_OPTIM_MASK; + tmp |= FRC_LATENCY_OPTIM_VAL(0x5); + I915_WRITE(ICL_PORT_TX_DW2_AUX(port), tmp); + tmp = I915_READ(ICL_PORT_TX_DW2_LN0(port)); + tmp &= ~FRC_LATENCY_OPTIM_MASK; + tmp |= FRC_LATENCY_OPTIM_VAL(0x5); + I915_WRITE(ICL_PORT_TX_DW2_GRP(port), tmp); + } + +} + +static void gen11_dsi_voltage_swing_program_seq(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + u32 tmp; + enum port port; + + /* clear common keeper enable bit */ + for_each_dsi_port(port, intel_dsi->ports) { + tmp = I915_READ(ICL_PORT_PCS_DW1_LN0(port)); + tmp &= ~COMMON_KEEPER_EN; + I915_WRITE(ICL_PORT_PCS_DW1_GRP(port), tmp); + tmp = I915_READ(ICL_PORT_PCS_DW1_AUX(port)); + tmp &= ~COMMON_KEEPER_EN; + I915_WRITE(ICL_PORT_PCS_DW1_AUX(port), tmp); + } + + /* + * Set SUS Clock Config bitfield to 11b + * Note: loadgen select program is done + * as part of lane phy sequence configuration + */ + for_each_dsi_port(port, intel_dsi->ports) { + tmp = I915_READ(ICL_PORT_CL_DW5(port)); + tmp |= SUS_CLOCK_CONFIG; + I915_WRITE(ICL_PORT_CL_DW5(port), tmp); + } + + /* Clear training enable to change swing values */ + for_each_dsi_port(port, intel_dsi->ports) { + tmp = I915_READ(ICL_PORT_TX_DW5_LN0(port)); + tmp &= ~TX_TRAINING_EN; + I915_WRITE(ICL_PORT_TX_DW5_GRP(port), tmp); + tmp = I915_READ(ICL_PORT_TX_DW5_AUX(port)); + tmp &= ~TX_TRAINING_EN; + I915_WRITE(ICL_PORT_TX_DW5_AUX(port), tmp); + } + + /* Program swing and de-emphasis */ + dsi_program_swing_and_deemphasis(encoder); + + /* Set training enable to trigger update */ + for_each_dsi_port(port, intel_dsi->ports) { + tmp = I915_READ(ICL_PORT_TX_DW5_LN0(port)); + tmp |= TX_TRAINING_EN; + I915_WRITE(ICL_PORT_TX_DW5_GRP(port), tmp); + tmp = I915_READ(ICL_PORT_TX_DW5_AUX(port)); + tmp |= TX_TRAINING_EN; + I915_WRITE(ICL_PORT_TX_DW5_AUX(port), tmp); + } +} + +static void gen11_dsi_enable_ddi_buffer(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + u32 tmp; + enum port port; + + for_each_dsi_port(port, intel_dsi->ports) { + tmp = I915_READ(DDI_BUF_CTL(port)); + tmp |= DDI_BUF_CTL_ENABLE; + I915_WRITE(DDI_BUF_CTL(port), tmp); + + if (wait_for_us(!(I915_READ(DDI_BUF_CTL(port)) & + DDI_BUF_IS_IDLE), + 500)) + DRM_ERROR("DDI port:%c buffer idle\n", port_name(port)); + } +} + +static void gen11_dsi_setup_dphy_timings(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + u32 tmp; + enum port port; + + /* Program T-INIT master registers */ + for_each_dsi_port(port, intel_dsi->ports) { + tmp = I915_READ(ICL_DSI_T_INIT_MASTER(port)); + tmp &= ~MASTER_INIT_TIMER_MASK; + tmp |= intel_dsi->init_count; + I915_WRITE(ICL_DSI_T_INIT_MASTER(port), tmp); + } + + /* Program DPHY clock lanes timings */ + for_each_dsi_port(port, intel_dsi->ports) { + I915_WRITE(DPHY_CLK_TIMING_PARAM(port), intel_dsi->dphy_reg); + + /* shadow register inside display core */ + I915_WRITE(DSI_CLK_TIMING_PARAM(port), intel_dsi->dphy_reg); + } + + /* Program DPHY data lanes timings */ + for_each_dsi_port(port, intel_dsi->ports) { + I915_WRITE(DPHY_DATA_TIMING_PARAM(port), + intel_dsi->dphy_data_lane_reg); + + /* shadow register inside display core */ + I915_WRITE(DSI_DATA_TIMING_PARAM(port), + intel_dsi->dphy_data_lane_reg); + } + + /* + * If DSI link operating at or below an 800 MHz, + * TA_SURE should be override and programmed to + * a value '0' inside TA_PARAM_REGISTERS otherwise + * leave all fields at HW default values. + */ + if (intel_dsi_bitrate(intel_dsi) <= 800000) { + for_each_dsi_port(port, intel_dsi->ports) { + tmp = I915_READ(DPHY_TA_TIMING_PARAM(port)); + tmp &= ~TA_SURE_MASK; + tmp |= TA_SURE_OVERRIDE | TA_SURE(0); + I915_WRITE(DPHY_TA_TIMING_PARAM(port), tmp); + + /* shadow register inside display core */ + tmp = I915_READ(DSI_TA_TIMING_PARAM(port)); + tmp &= ~TA_SURE_MASK; + tmp |= TA_SURE_OVERRIDE | TA_SURE(0); + I915_WRITE(DSI_TA_TIMING_PARAM(port), tmp); + } + } +} + +static void gen11_dsi_gate_clocks(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + u32 tmp; + enum port port; + + mutex_lock(&dev_priv->dpll_lock); + tmp = I915_READ(DPCLKA_CFGCR0_ICL); + for_each_dsi_port(port, intel_dsi->ports) { + tmp |= DPCLKA_CFGCR0_DDI_CLK_OFF(port); + } + + I915_WRITE(DPCLKA_CFGCR0_ICL, tmp); + mutex_unlock(&dev_priv->dpll_lock); +} + +static void gen11_dsi_ungate_clocks(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + u32 tmp; + enum port port; + + mutex_lock(&dev_priv->dpll_lock); + tmp = I915_READ(DPCLKA_CFGCR0_ICL); + for_each_dsi_port(port, intel_dsi->ports) { + tmp &= ~DPCLKA_CFGCR0_DDI_CLK_OFF(port); + } + + I915_WRITE(DPCLKA_CFGCR0_ICL, tmp); + mutex_unlock(&dev_priv->dpll_lock); +} + +static void gen11_dsi_map_pll(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + struct intel_shared_dpll *pll = crtc_state->shared_dpll; + enum port port; + u32 val; + + mutex_lock(&dev_priv->dpll_lock); + + val = I915_READ(DPCLKA_CFGCR0_ICL); + for_each_dsi_port(port, intel_dsi->ports) { + val &= ~DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port); + val |= DPCLKA_CFGCR0_DDI_CLK_SEL(pll->info->id, port); + } + I915_WRITE(DPCLKA_CFGCR0_ICL, val); + + for_each_dsi_port(port, intel_dsi->ports) { + val &= ~DPCLKA_CFGCR0_DDI_CLK_OFF(port); + } + I915_WRITE(DPCLKA_CFGCR0_ICL, val); + + POSTING_READ(DPCLKA_CFGCR0_ICL); + + mutex_unlock(&dev_priv->dpll_lock); +} + +static void +gen11_dsi_configure_transcoder(struct intel_encoder *encoder, + const struct intel_crtc_state *pipe_config) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + struct intel_crtc *intel_crtc = to_intel_crtc(pipe_config->base.crtc); + enum pipe pipe = intel_crtc->pipe; + u32 tmp; + enum port port; + enum transcoder dsi_trans; + + for_each_dsi_port(port, intel_dsi->ports) { + dsi_trans = dsi_port_to_transcoder(port); + tmp = I915_READ(DSI_TRANS_FUNC_CONF(dsi_trans)); + + if (intel_dsi->eotp_pkt) + tmp &= ~EOTP_DISABLED; + else + tmp |= EOTP_DISABLED; + + /* enable link calibration if freq > 1.5Gbps */ + if (intel_dsi_bitrate(intel_dsi) >= 1500 * 1000) { + tmp &= ~LINK_CALIBRATION_MASK; + tmp |= CALIBRATION_ENABLED_INITIAL_ONLY; + } + + /* configure continuous clock */ + tmp &= ~CONTINUOUS_CLK_MASK; + if (intel_dsi->clock_stop) + tmp |= CLK_ENTER_LP_AFTER_DATA; + else + tmp |= CLK_HS_CONTINUOUS; + + /* configure buffer threshold limit to minimum */ + tmp &= ~PIX_BUF_THRESHOLD_MASK; + tmp |= PIX_BUF_THRESHOLD_1_4; + + /* set virtual channel to '0' */ + tmp &= ~PIX_VIRT_CHAN_MASK; + tmp |= PIX_VIRT_CHAN(0); + + /* program BGR transmission */ + if (intel_dsi->bgr_enabled) + tmp |= BGR_TRANSMISSION; + + /* select pixel format */ + tmp &= ~PIX_FMT_MASK; + switch (intel_dsi->pixel_format) { + default: + MISSING_CASE(intel_dsi->pixel_format); + /* fallthrough */ + case MIPI_DSI_FMT_RGB565: + tmp |= PIX_FMT_RGB565; + break; + case MIPI_DSI_FMT_RGB666_PACKED: + tmp |= PIX_FMT_RGB666_PACKED; + break; + case MIPI_DSI_FMT_RGB666: + tmp |= PIX_FMT_RGB666_LOOSE; + break; + case MIPI_DSI_FMT_RGB888: + tmp |= PIX_FMT_RGB888; + break; + } + + /* program DSI operation mode */ + if (is_vid_mode(intel_dsi)) { + tmp &= ~OP_MODE_MASK; + switch (intel_dsi->video_mode_format) { + default: + MISSING_CASE(intel_dsi->video_mode_format); + /* fallthrough */ + case VIDEO_MODE_NON_BURST_WITH_SYNC_EVENTS: + tmp |= VIDEO_MODE_SYNC_EVENT; + break; + case VIDEO_MODE_NON_BURST_WITH_SYNC_PULSE: + tmp |= VIDEO_MODE_SYNC_PULSE; + break; + } + } + + I915_WRITE(DSI_TRANS_FUNC_CONF(dsi_trans), tmp); + } + + /* enable port sync mode if dual link */ + if (intel_dsi->dual_link) { + for_each_dsi_port(port, intel_dsi->ports) { + dsi_trans = dsi_port_to_transcoder(port); + tmp = I915_READ(TRANS_DDI_FUNC_CTL2(dsi_trans)); + tmp |= PORT_SYNC_MODE_ENABLE; + I915_WRITE(TRANS_DDI_FUNC_CTL2(dsi_trans), tmp); + } + + /* configure stream splitting */ + configure_dual_link_mode(encoder, pipe_config); + } + + for_each_dsi_port(port, intel_dsi->ports) { + dsi_trans = dsi_port_to_transcoder(port); + + /* select data lane width */ + tmp = I915_READ(TRANS_DDI_FUNC_CTL(dsi_trans)); + tmp &= ~DDI_PORT_WIDTH_MASK; + tmp |= DDI_PORT_WIDTH(intel_dsi->lane_count); + + /* select input pipe */ + tmp &= ~TRANS_DDI_EDP_INPUT_MASK; + switch (pipe) { + default: + MISSING_CASE(pipe); + /* fallthrough */ + case PIPE_A: + tmp |= TRANS_DDI_EDP_INPUT_A_ON; + break; + case PIPE_B: + tmp |= TRANS_DDI_EDP_INPUT_B_ONOFF; + break; + case PIPE_C: + tmp |= TRANS_DDI_EDP_INPUT_C_ONOFF; + break; + } + + /* enable DDI buffer */ + tmp |= TRANS_DDI_FUNC_ENABLE; + I915_WRITE(TRANS_DDI_FUNC_CTL(dsi_trans), tmp); + } + + /* wait for link ready */ + for_each_dsi_port(port, intel_dsi->ports) { + dsi_trans = dsi_port_to_transcoder(port); + if (wait_for_us((I915_READ(DSI_TRANS_FUNC_CONF(dsi_trans)) & + LINK_READY), 2500)) + DRM_ERROR("DSI link not ready\n"); + } +} + +static void +gen11_dsi_set_transcoder_timings(struct intel_encoder *encoder, + const struct intel_crtc_state *pipe_config) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + const struct drm_display_mode *adjusted_mode = + &pipe_config->base.adjusted_mode; + enum port port; + enum transcoder dsi_trans; + /* horizontal timings */ + u16 htotal, hactive, hsync_start, hsync_end, hsync_size; + u16 hfront_porch, hback_porch; + /* vertical timings */ + u16 vtotal, vactive, vsync_start, vsync_end, vsync_shift; + + hactive = adjusted_mode->crtc_hdisplay; + htotal = adjusted_mode->crtc_htotal; + hsync_start = adjusted_mode->crtc_hsync_start; + hsync_end = adjusted_mode->crtc_hsync_end; + hsync_size = hsync_end - hsync_start; + hfront_porch = (adjusted_mode->crtc_hsync_start - + adjusted_mode->crtc_hdisplay); + hback_porch = (adjusted_mode->crtc_htotal - + adjusted_mode->crtc_hsync_end); + vactive = adjusted_mode->crtc_vdisplay; + vtotal = adjusted_mode->crtc_vtotal; + vsync_start = adjusted_mode->crtc_vsync_start; + vsync_end = adjusted_mode->crtc_vsync_end; + vsync_shift = hsync_start - htotal / 2; + + if (intel_dsi->dual_link) { + hactive /= 2; + if (intel_dsi->dual_link == DSI_DUAL_LINK_FRONT_BACK) + hactive += intel_dsi->pixel_overlap; + htotal /= 2; + } + + /* minimum hactive as per bspec: 256 pixels */ + if (adjusted_mode->crtc_hdisplay < 256) + DRM_ERROR("hactive is less then 256 pixels\n"); + + /* if RGB666 format, then hactive must be multiple of 4 pixels */ + if (intel_dsi->pixel_format == MIPI_DSI_FMT_RGB666 && hactive % 4 != 0) + DRM_ERROR("hactive pixels are not multiple of 4\n"); + + /* program TRANS_HTOTAL register */ + for_each_dsi_port(port, intel_dsi->ports) { + dsi_trans = dsi_port_to_transcoder(port); + I915_WRITE(HTOTAL(dsi_trans), + (hactive - 1) | ((htotal - 1) << 16)); + } + + /* TRANS_HSYNC register to be programmed only for video mode */ + if (intel_dsi->operation_mode == INTEL_DSI_VIDEO_MODE) { + if (intel_dsi->video_mode_format == + VIDEO_MODE_NON_BURST_WITH_SYNC_PULSE) { + /* BSPEC: hsync size should be atleast 16 pixels */ + if (hsync_size < 16) + DRM_ERROR("hsync size < 16 pixels\n"); + } + + if (hback_porch < 16) + DRM_ERROR("hback porch < 16 pixels\n"); + + if (intel_dsi->dual_link) { + hsync_start /= 2; + hsync_end /= 2; + } + + for_each_dsi_port(port, intel_dsi->ports) { + dsi_trans = dsi_port_to_transcoder(port); + I915_WRITE(HSYNC(dsi_trans), + (hsync_start - 1) | ((hsync_end - 1) << 16)); + } + } + + /* program TRANS_VTOTAL register */ + for_each_dsi_port(port, intel_dsi->ports) { + dsi_trans = dsi_port_to_transcoder(port); + /* + * FIXME: Programing this by assuming progressive mode, since + * non-interlaced info from VBT is not saved inside + * struct drm_display_mode. + * For interlace mode: program required pixel minus 2 + */ + I915_WRITE(VTOTAL(dsi_trans), + (vactive - 1) | ((vtotal - 1) << 16)); + } + + if (vsync_end < vsync_start || vsync_end > vtotal) + DRM_ERROR("Invalid vsync_end value\n"); + + if (vsync_start < vactive) + DRM_ERROR("vsync_start less than vactive\n"); + + /* program TRANS_VSYNC register */ + for_each_dsi_port(port, intel_dsi->ports) { + dsi_trans = dsi_port_to_transcoder(port); + I915_WRITE(VSYNC(dsi_trans), + (vsync_start - 1) | ((vsync_end - 1) << 16)); + } + + /* + * FIXME: It has to be programmed only for interlaced + * modes. Put the check condition here once interlaced + * info available as described above. + * program TRANS_VSYNCSHIFT register + */ + for_each_dsi_port(port, intel_dsi->ports) { + dsi_trans = dsi_port_to_transcoder(port); + I915_WRITE(VSYNCSHIFT(dsi_trans), vsync_shift); + } +} + +static void gen11_dsi_enable_transcoder(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + enum port port; + enum transcoder dsi_trans; + u32 tmp; + + for_each_dsi_port(port, intel_dsi->ports) { + dsi_trans = dsi_port_to_transcoder(port); + tmp = I915_READ(PIPECONF(dsi_trans)); + tmp |= PIPECONF_ENABLE; + I915_WRITE(PIPECONF(dsi_trans), tmp); + + /* wait for transcoder to be enabled */ + if (intel_wait_for_register(dev_priv, PIPECONF(dsi_trans), + I965_PIPECONF_ACTIVE, + I965_PIPECONF_ACTIVE, 10)) + DRM_ERROR("DSI transcoder not enabled\n"); + } +} + +static void gen11_dsi_setup_timeouts(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + enum port port; + enum transcoder dsi_trans; + u32 tmp, hs_tx_timeout, lp_rx_timeout, ta_timeout, divisor, mul; + + /* + * escape clock count calculation: + * BYTE_CLK_COUNT = TIME_NS/(8 * UI) + * UI (nsec) = (10^6)/Bitrate + * TIME_NS = (BYTE_CLK_COUNT * 8 * 10^6)/ Bitrate + * ESCAPE_CLK_COUNT = TIME_NS/ESC_CLK_NS + */ + divisor = intel_dsi_tlpx_ns(intel_dsi) * intel_dsi_bitrate(intel_dsi) * 1000; + mul = 8 * 1000000; + hs_tx_timeout = DIV_ROUND_UP(intel_dsi->hs_tx_timeout * mul, + divisor); + lp_rx_timeout = DIV_ROUND_UP(intel_dsi->lp_rx_timeout * mul, divisor); + ta_timeout = DIV_ROUND_UP(intel_dsi->turn_arnd_val * mul, divisor); + + for_each_dsi_port(port, intel_dsi->ports) { + dsi_trans = dsi_port_to_transcoder(port); + + /* program hst_tx_timeout */ + tmp = I915_READ(DSI_HSTX_TO(dsi_trans)); + tmp &= ~HSTX_TIMEOUT_VALUE_MASK; + tmp |= HSTX_TIMEOUT_VALUE(hs_tx_timeout); + I915_WRITE(DSI_HSTX_TO(dsi_trans), tmp); + + /* FIXME: DSI_CALIB_TO */ + + /* program lp_rx_host timeout */ + tmp = I915_READ(DSI_LPRX_HOST_TO(dsi_trans)); + tmp &= ~LPRX_TIMEOUT_VALUE_MASK; + tmp |= LPRX_TIMEOUT_VALUE(lp_rx_timeout); + I915_WRITE(DSI_LPRX_HOST_TO(dsi_trans), tmp); + + /* FIXME: DSI_PWAIT_TO */ + + /* program turn around timeout */ + tmp = I915_READ(DSI_TA_TO(dsi_trans)); + tmp &= ~TA_TIMEOUT_VALUE_MASK; + tmp |= TA_TIMEOUT_VALUE(ta_timeout); + I915_WRITE(DSI_TA_TO(dsi_trans), tmp); + } +} + +static void +gen11_dsi_enable_port_and_phy(struct intel_encoder *encoder, + const struct intel_crtc_state *pipe_config) { /* step 4a: power up all lanes of the DDI used by DSI */ gen11_dsi_power_up_lanes(encoder); + + /* step 4b: configure lane sequencing of the Combo-PHY transmitters */ + gen11_dsi_config_phy_lanes_sequence(encoder); + + /* step 4c: configure voltage swing and skew */ + gen11_dsi_voltage_swing_program_seq(encoder); + + /* enable DDI buffer */ + gen11_dsi_enable_ddi_buffer(encoder); + + /* setup D-PHY timings */ + gen11_dsi_setup_dphy_timings(encoder); + + /* step 4h: setup DSI protocol timeouts */ + gen11_dsi_setup_timeouts(encoder); + + /* Step (4h, 4i, 4j, 4k): Configure transcoder */ + gen11_dsi_configure_transcoder(encoder, pipe_config); + + /* Step 4l: Gate DDI clocks */ + gen11_dsi_gate_clocks(encoder); } -static void __attribute__((unused)) -gen11_dsi_pre_enable(struct intel_encoder *encoder, - const struct intel_crtc_state *pipe_config, - const struct drm_connector_state *conn_state) +static void gen11_dsi_powerup_panel(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + struct mipi_dsi_device *dsi; + enum port port; + enum transcoder dsi_trans; + u32 tmp; + int ret; + + /* set maximum return packet size */ + for_each_dsi_port(port, intel_dsi->ports) { + dsi_trans = dsi_port_to_transcoder(port); + + /* + * FIXME: This uses the number of DW's currently in the payload + * receive queue. This is probably not what we want here. + */ + tmp = I915_READ(DSI_CMD_RXCTL(dsi_trans)); + tmp &= NUMBER_RX_PLOAD_DW_MASK; + /* multiply "Number Rx Payload DW" by 4 to get max value */ + tmp = tmp * 4; + dsi = intel_dsi->dsi_hosts[port]->device; + ret = mipi_dsi_set_maximum_return_packet_size(dsi, tmp); + if (ret < 0) + DRM_ERROR("error setting max return pkt size%d\n", tmp); + } + + /* panel power on related mipi dsi vbt sequences */ + intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_POWER_ON); + intel_dsi_msleep(intel_dsi, intel_dsi->panel_on_delay); + intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_DEASSERT_RESET); + intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_INIT_OTP); + intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_DISPLAY_ON); + + /* ensure all panel commands dispatched before enabling transcoder */ + wait_for_cmds_dispatched_to_panel(encoder); +} + +static void gen11_dsi_pre_pll_enable(struct intel_encoder *encoder, + const struct intel_crtc_state *pipe_config, + const struct drm_connector_state *conn_state) { /* step2: enable IO power */ gen11_dsi_enable_io_power(encoder); /* step3: enable DSI PLL */ gen11_dsi_program_esc_clk_div(encoder); +} + +static void gen11_dsi_pre_enable(struct intel_encoder *encoder, + const struct intel_crtc_state *pipe_config, + const struct drm_connector_state *conn_state) +{ + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + + /* step3b */ + gen11_dsi_map_pll(encoder, pipe_config); /* step4: enable DSI port and DPHY */ - gen11_dsi_enable_port_and_phy(encoder); + gen11_dsi_enable_port_and_phy(encoder, pipe_config); + + /* step5: program and powerup panel */ + gen11_dsi_powerup_panel(encoder); + + /* step6c: configure transcoder timings */ + gen11_dsi_set_transcoder_timings(encoder, pipe_config); + + /* step6d: enable dsi transcoder */ + gen11_dsi_enable_transcoder(encoder); + + /* step7: enable backlight */ + intel_panel_enable_backlight(pipe_config, conn_state); + intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_BACKLIGHT_ON); +} + +static void gen11_dsi_disable_transcoder(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + enum port port; + enum transcoder dsi_trans; + u32 tmp; + + for_each_dsi_port(port, intel_dsi->ports) { + dsi_trans = dsi_port_to_transcoder(port); + + /* disable transcoder */ + tmp = I915_READ(PIPECONF(dsi_trans)); + tmp &= ~PIPECONF_ENABLE; + I915_WRITE(PIPECONF(dsi_trans), tmp); + + /* wait for transcoder to be disabled */ + if (intel_wait_for_register(dev_priv, PIPECONF(dsi_trans), + I965_PIPECONF_ACTIVE, 0, 50)) + DRM_ERROR("DSI trancoder not disabled\n"); + } +} + +static void gen11_dsi_powerdown_panel(struct intel_encoder *encoder) +{ + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + + intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_DISPLAY_OFF); + intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_ASSERT_RESET); + intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_POWER_OFF); + + /* ensure cmds dispatched to panel */ + wait_for_cmds_dispatched_to_panel(encoder); +} + +static void gen11_dsi_deconfigure_trancoder(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + enum port port; + enum transcoder dsi_trans; + u32 tmp; + + /* put dsi link in ULPS */ + for_each_dsi_port(port, intel_dsi->ports) { + dsi_trans = dsi_port_to_transcoder(port); + tmp = I915_READ(DSI_LP_MSG(dsi_trans)); + tmp |= LINK_ENTER_ULPS; + tmp &= ~LINK_ULPS_TYPE_LP11; + I915_WRITE(DSI_LP_MSG(dsi_trans), tmp); + + if (wait_for_us((I915_READ(DSI_LP_MSG(dsi_trans)) & + LINK_IN_ULPS), + 10)) + DRM_ERROR("DSI link not in ULPS\n"); + } + + /* disable ddi function */ + for_each_dsi_port(port, intel_dsi->ports) { + dsi_trans = dsi_port_to_transcoder(port); + tmp = I915_READ(TRANS_DDI_FUNC_CTL(dsi_trans)); + tmp &= ~TRANS_DDI_FUNC_ENABLE; + I915_WRITE(TRANS_DDI_FUNC_CTL(dsi_trans), tmp); + } + + /* disable port sync mode if dual link */ + if (intel_dsi->dual_link) { + for_each_dsi_port(port, intel_dsi->ports) { + dsi_trans = dsi_port_to_transcoder(port); + tmp = I915_READ(TRANS_DDI_FUNC_CTL2(dsi_trans)); + tmp &= ~PORT_SYNC_MODE_ENABLE; + I915_WRITE(TRANS_DDI_FUNC_CTL2(dsi_trans), tmp); + } + } +} + +static void gen11_dsi_disable_port(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + u32 tmp; + enum port port; + + gen11_dsi_ungate_clocks(encoder); + for_each_dsi_port(port, intel_dsi->ports) { + tmp = I915_READ(DDI_BUF_CTL(port)); + tmp &= ~DDI_BUF_CTL_ENABLE; + I915_WRITE(DDI_BUF_CTL(port), tmp); + + if (wait_for_us((I915_READ(DDI_BUF_CTL(port)) & + DDI_BUF_IS_IDLE), + 8)) + DRM_ERROR("DDI port:%c buffer not idle\n", + port_name(port)); + } + gen11_dsi_gate_clocks(encoder); +} + +static void gen11_dsi_disable_io_power(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + enum port port; + u32 tmp; + + for_each_dsi_port(port, intel_dsi->ports) { + intel_wakeref_t wakeref; + + wakeref = fetch_and_zero(&intel_dsi->io_wakeref[port]); + if (wakeref) { + intel_display_power_put(dev_priv, + port == PORT_A ? + POWER_DOMAIN_PORT_DDI_A_IO : + POWER_DOMAIN_PORT_DDI_B_IO, + wakeref); + } + } + + /* set mode to DDI */ + for_each_dsi_port(port, intel_dsi->ports) { + tmp = I915_READ(ICL_DSI_IO_MODECTL(port)); + tmp &= ~COMBO_PHY_MODE_DSI; + I915_WRITE(ICL_DSI_IO_MODECTL(port), tmp); + } +} + +static void gen11_dsi_disable(struct intel_encoder *encoder, + const struct intel_crtc_state *old_crtc_state, + const struct drm_connector_state *old_conn_state) +{ + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + + /* step1: turn off backlight */ + intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_BACKLIGHT_OFF); + intel_panel_disable_backlight(old_conn_state); + + /* step2d,e: disable transcoder and wait */ + gen11_dsi_disable_transcoder(encoder); + + /* step2f,g: powerdown panel */ + gen11_dsi_powerdown_panel(encoder); + + /* step2h,i,j: deconfig trancoder */ + gen11_dsi_deconfigure_trancoder(encoder); + + /* step3: disable port */ + gen11_dsi_disable_port(encoder); + + /* step4: disable IO power */ + gen11_dsi_disable_io_power(encoder); +} + +static void gen11_dsi_get_config(struct intel_encoder *encoder, + struct intel_crtc_state *pipe_config) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + u32 pll_id; + + /* FIXME: adapt icl_ddi_clock_get() for DSI and use that? */ + pll_id = intel_get_shared_dpll_id(dev_priv, pipe_config->shared_dpll); + pipe_config->port_clock = cnl_calc_wrpll_link(dev_priv, pll_id); + pipe_config->base.adjusted_mode.crtc_clock = intel_dsi->pclk; + pipe_config->output_types |= BIT(INTEL_OUTPUT_DSI); +} + +static int gen11_dsi_compute_config(struct intel_encoder *encoder, + struct intel_crtc_state *pipe_config, + struct drm_connector_state *conn_state) +{ + struct intel_dsi *intel_dsi = container_of(encoder, struct intel_dsi, + base); + struct intel_connector *intel_connector = intel_dsi->attached_connector; + struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc); + const struct drm_display_mode *fixed_mode = + intel_connector->panel.fixed_mode; + struct drm_display_mode *adjusted_mode = + &pipe_config->base.adjusted_mode; + + intel_fixed_panel_mode(fixed_mode, adjusted_mode); + intel_pch_panel_fitting(crtc, pipe_config, conn_state->scaling_mode); + + adjusted_mode->flags = 0; + + /* Dual link goes to trancoder DSI'0' */ + if (intel_dsi->ports == BIT(PORT_B)) + pipe_config->cpu_transcoder = TRANSCODER_DSI_1; + else + pipe_config->cpu_transcoder = TRANSCODER_DSI_0; + + pipe_config->clock_set = true; + pipe_config->port_clock = intel_dsi_bitrate(intel_dsi) / 5; + + return 0; +} + +static void gen11_dsi_get_power_domains(struct intel_encoder *encoder, + struct intel_crtc_state *crtc_state) +{ + get_dsi_io_power_domains(to_i915(encoder->base.dev), + enc_to_intel_dsi(&encoder->base)); +} + +static bool gen11_dsi_get_hw_state(struct intel_encoder *encoder, + enum pipe *pipe) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + enum transcoder dsi_trans; + intel_wakeref_t wakeref; + enum port port; + bool ret = false; + u32 tmp; + + wakeref = intel_display_power_get_if_enabled(dev_priv, + encoder->power_domain); + if (!wakeref) + return false; + + for_each_dsi_port(port, intel_dsi->ports) { + dsi_trans = dsi_port_to_transcoder(port); + tmp = I915_READ(TRANS_DDI_FUNC_CTL(dsi_trans)); + switch (tmp & TRANS_DDI_EDP_INPUT_MASK) { + case TRANS_DDI_EDP_INPUT_A_ON: + *pipe = PIPE_A; + break; + case TRANS_DDI_EDP_INPUT_B_ONOFF: + *pipe = PIPE_B; + break; + case TRANS_DDI_EDP_INPUT_C_ONOFF: + *pipe = PIPE_C; + break; + default: + DRM_ERROR("Invalid PIPE input\n"); + goto out; + } + + tmp = I915_READ(PIPECONF(dsi_trans)); + ret = tmp & PIPECONF_ENABLE; + } +out: + intel_display_power_put(dev_priv, encoder->power_domain, wakeref); + return ret; +} + +static void gen11_dsi_encoder_destroy(struct drm_encoder *encoder) +{ + intel_encoder_destroy(encoder); +} + +static const struct drm_encoder_funcs gen11_dsi_encoder_funcs = { + .destroy = gen11_dsi_encoder_destroy, +}; + +static const struct drm_connector_funcs gen11_dsi_connector_funcs = { + .late_register = intel_connector_register, + .early_unregister = intel_connector_unregister, + .destroy = intel_connector_destroy, + .fill_modes = drm_helper_probe_single_connector_modes, + .atomic_get_property = intel_digital_connector_atomic_get_property, + .atomic_set_property = intel_digital_connector_atomic_set_property, + .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, + .atomic_duplicate_state = intel_digital_connector_duplicate_state, +}; + +static const struct drm_connector_helper_funcs gen11_dsi_connector_helper_funcs = { + .get_modes = intel_dsi_get_modes, + .mode_valid = intel_dsi_mode_valid, + .atomic_check = intel_digital_connector_atomic_check, +}; + +static int gen11_dsi_host_attach(struct mipi_dsi_host *host, + struct mipi_dsi_device *dsi) +{ + return 0; +} + +static int gen11_dsi_host_detach(struct mipi_dsi_host *host, + struct mipi_dsi_device *dsi) +{ + return 0; +} + +static ssize_t gen11_dsi_host_transfer(struct mipi_dsi_host *host, + const struct mipi_dsi_msg *msg) +{ + struct intel_dsi_host *intel_dsi_host = to_intel_dsi_host(host); + struct mipi_dsi_packet dsi_pkt; + ssize_t ret; + bool enable_lpdt = false; + + ret = mipi_dsi_create_packet(&dsi_pkt, msg); + if (ret < 0) + return ret; + + if (msg->flags & MIPI_DSI_MSG_USE_LPM) + enable_lpdt = true; + + /* send packet header */ + ret = dsi_send_pkt_hdr(intel_dsi_host, dsi_pkt, enable_lpdt); + if (ret < 0) + return ret; + + /* only long packet contains payload */ + if (mipi_dsi_packet_format_is_long(msg->type)) { + ret = dsi_send_pkt_payld(intel_dsi_host, dsi_pkt); + if (ret < 0) + return ret; + } + + //TODO: add payload receive code if needed + + ret = sizeof(dsi_pkt.header) + dsi_pkt.payload_length; + + return ret; +} + +static const struct mipi_dsi_host_ops gen11_dsi_host_ops = { + .attach = gen11_dsi_host_attach, + .detach = gen11_dsi_host_detach, + .transfer = gen11_dsi_host_transfer, +}; + +void icl_dsi_init(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = &dev_priv->drm; + struct intel_dsi *intel_dsi; + struct intel_encoder *encoder; + struct intel_connector *intel_connector; + struct drm_connector *connector; + struct drm_display_mode *scan, *fixed_mode = NULL; + enum port port; + + if (!intel_bios_is_dsi_present(dev_priv, &port)) + return; + + intel_dsi = kzalloc(sizeof(*intel_dsi), GFP_KERNEL); + if (!intel_dsi) + return; + + intel_connector = intel_connector_alloc(); + if (!intel_connector) { + kfree(intel_dsi); + return; + } + + encoder = &intel_dsi->base; + intel_dsi->attached_connector = intel_connector; + connector = &intel_connector->base; + + /* register DSI encoder with DRM subsystem */ + drm_encoder_init(dev, &encoder->base, &gen11_dsi_encoder_funcs, + DRM_MODE_ENCODER_DSI, "DSI %c", port_name(port)); + + encoder->pre_pll_enable = gen11_dsi_pre_pll_enable; + encoder->pre_enable = gen11_dsi_pre_enable; + encoder->disable = gen11_dsi_disable; + encoder->port = port; + encoder->get_config = gen11_dsi_get_config; + encoder->update_pipe = intel_panel_update_backlight; + encoder->compute_config = gen11_dsi_compute_config; + encoder->get_hw_state = gen11_dsi_get_hw_state; + encoder->type = INTEL_OUTPUT_DSI; + encoder->cloneable = 0; + encoder->crtc_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C); + encoder->power_domain = POWER_DOMAIN_PORT_DSI; + encoder->get_power_domains = gen11_dsi_get_power_domains; + + /* register DSI connector with DRM subsystem */ + drm_connector_init(dev, connector, &gen11_dsi_connector_funcs, + DRM_MODE_CONNECTOR_DSI); + drm_connector_helper_add(connector, &gen11_dsi_connector_helper_funcs); + connector->display_info.subpixel_order = SubPixelHorizontalRGB; + connector->interlace_allowed = false; + connector->doublescan_allowed = false; + intel_connector->get_hw_state = intel_connector_get_hw_state; + + /* attach connector to encoder */ + intel_connector_attach_encoder(intel_connector, encoder); + + /* fill mode info from VBT */ + mutex_lock(&dev->mode_config.mutex); + intel_dsi_vbt_get_modes(intel_dsi); + list_for_each_entry(scan, &connector->probed_modes, head) { + if (scan->type & DRM_MODE_TYPE_PREFERRED) { + fixed_mode = drm_mode_duplicate(dev, scan); + break; + } + } + mutex_unlock(&dev->mode_config.mutex); + + if (!fixed_mode) { + DRM_ERROR("DSI fixed mode info missing\n"); + goto err; + } + + connector->display_info.width_mm = fixed_mode->width_mm; + connector->display_info.height_mm = fixed_mode->height_mm; + intel_panel_init(&intel_connector->panel, fixed_mode, NULL); + intel_panel_setup_backlight(connector, INVALID_PIPE); + + + if (dev_priv->vbt.dsi.config->dual_link) + intel_dsi->ports = BIT(PORT_A) | BIT(PORT_B); + else + intel_dsi->ports = BIT(port); + + intel_dsi->dcs_backlight_ports = dev_priv->vbt.dsi.bl_ports; + intel_dsi->dcs_cabc_ports = dev_priv->vbt.dsi.cabc_ports; + + for_each_dsi_port(port, intel_dsi->ports) { + struct intel_dsi_host *host; + + host = intel_dsi_host_init(intel_dsi, &gen11_dsi_host_ops, port); + if (!host) + goto err; + + intel_dsi->dsi_hosts[port] = host; + } + + if (!intel_dsi_vbt_init(intel_dsi, MIPI_DSI_GENERIC_PANEL_ID)) { + DRM_DEBUG_KMS("no device found\n"); + goto err; + } + + return; + +err: + drm_encoder_cleanup(&encoder->base); + kfree(intel_dsi); + kfree(intel_connector); } diff --git a/drivers/gpu/drm/i915/intel_acpi.c b/drivers/gpu/drm/i915/intel_acpi.c index 6ba478e57b9b..9d142d038a7d 100644 --- a/drivers/gpu/drm/i915/intel_acpi.c +++ b/drivers/gpu/drm/i915/intel_acpi.c @@ -6,7 +6,6 @@ */ #include <linux/pci.h> #include <linux/acpi.h> -#include <drm/drmP.h> #include "i915_drv.h" #define INTEL_DSM_REVISION_ID 1 /* For Calpella anyway... */ diff --git a/drivers/gpu/drm/i915/intel_atomic.c b/drivers/gpu/drm/i915/intel_atomic.c index b04952bacf77..7cf9290ea34a 100644 --- a/drivers/gpu/drm/i915/intel_atomic.c +++ b/drivers/gpu/drm/i915/intel_atomic.c @@ -29,10 +29,11 @@ * See intel_atomic_plane.c for the plane-specific atomic functionality. */ -#include <drm/drmP.h> #include <drm/drm_atomic.h> #include <drm/drm_atomic_helper.h> +#include <drm/drm_fourcc.h> #include <drm/drm_plane_helper.h> + #include "intel_drv.h" /** @@ -47,7 +48,7 @@ int intel_digital_connector_atomic_get_property(struct drm_connector *connector, const struct drm_connector_state *state, struct drm_property *property, - uint64_t *val) + u64 *val) { struct drm_device *dev = connector->dev; struct drm_i915_private *dev_priv = to_i915(dev); @@ -79,7 +80,7 @@ int intel_digital_connector_atomic_get_property(struct drm_connector *connector, int intel_digital_connector_atomic_set_property(struct drm_connector *connector, struct drm_connector_state *state, struct drm_property *property, - uint64_t val) + u64 val) { struct drm_device *dev = connector->dev; struct drm_i915_private *dev_priv = to_i915(dev); @@ -184,6 +185,7 @@ intel_crtc_duplicate_state(struct drm_crtc *crtc) crtc_state->fifo_changed = false; crtc_state->wm.need_postvbl_update = false; crtc_state->fb_bits = 0; + crtc_state->update_planes = 0; return &crtc_state->base; } @@ -203,6 +205,72 @@ intel_crtc_destroy_state(struct drm_crtc *crtc, drm_atomic_helper_crtc_destroy_state(crtc, state); } +static void intel_atomic_setup_scaler(struct intel_crtc_scaler_state *scaler_state, + int num_scalers_need, struct intel_crtc *intel_crtc, + const char *name, int idx, + struct intel_plane_state *plane_state, + int *scaler_id) +{ + struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev); + int j; + u32 mode; + + if (*scaler_id < 0) { + /* find a free scaler */ + for (j = 0; j < intel_crtc->num_scalers; j++) { + if (scaler_state->scalers[j].in_use) + continue; + + *scaler_id = j; + scaler_state->scalers[*scaler_id].in_use = 1; + break; + } + } + + if (WARN(*scaler_id < 0, "Cannot find scaler for %s:%d\n", name, idx)) + return; + + /* set scaler mode */ + if (plane_state && plane_state->base.fb && + plane_state->base.fb->format->is_yuv && + plane_state->base.fb->format->num_planes > 1) { + if (IS_GEN(dev_priv, 9) && + !IS_GEMINILAKE(dev_priv)) { + mode = SKL_PS_SCALER_MODE_NV12; + } else if (icl_is_hdr_plane(to_intel_plane(plane_state->base.plane))) { + /* + * On gen11+'s HDR planes we only use the scaler for + * scaling. They have a dedicated chroma upsampler, so + * we don't need the scaler to upsample the UV plane. + */ + mode = PS_SCALER_MODE_NORMAL; + } else { + mode = PS_SCALER_MODE_PLANAR; + + if (plane_state->linked_plane) + mode |= PS_PLANE_Y_SEL(plane_state->linked_plane->id); + } + } else if (INTEL_GEN(dev_priv) > 9 || IS_GEMINILAKE(dev_priv)) { + mode = PS_SCALER_MODE_NORMAL; + } else if (num_scalers_need == 1 && intel_crtc->num_scalers > 1) { + /* + * when only 1 scaler is in use on a pipe with 2 scalers + * scaler 0 operates in high quality (HQ) mode. + * In this case use scaler 0 to take advantage of HQ mode + */ + scaler_state->scalers[*scaler_id].in_use = 0; + *scaler_id = 0; + scaler_state->scalers[0].in_use = 1; + mode = SKL_PS_SCALER_MODE_HQ; + } else { + mode = SKL_PS_SCALER_MODE_DYN; + } + + DRM_DEBUG_KMS("Attached scaler id %u.%u to %s:%d\n", + intel_crtc->pipe, *scaler_id, name, idx); + scaler_state->scalers[*scaler_id].mode = mode; +} + /** * intel_atomic_setup_scalers() - setup scalers for crtc per staged requests * @dev_priv: i915 device @@ -232,7 +300,7 @@ int intel_atomic_setup_scalers(struct drm_i915_private *dev_priv, struct drm_atomic_state *drm_state = crtc_state->base.state; struct intel_atomic_state *intel_state = to_intel_atomic_state(drm_state); int num_scalers_need; - int i, j; + int i; num_scalers_need = hweight32(scaler_state->scaler_users); @@ -304,59 +372,17 @@ int intel_atomic_setup_scalers(struct drm_i915_private *dev_priv, idx = plane->base.id; /* plane on different crtc cannot be a scaler user of this crtc */ - if (WARN_ON(intel_plane->pipe != intel_crtc->pipe)) { + if (WARN_ON(intel_plane->pipe != intel_crtc->pipe)) continue; - } plane_state = intel_atomic_get_new_plane_state(intel_state, intel_plane); scaler_id = &plane_state->scaler_id; } - if (*scaler_id < 0) { - /* find a free scaler */ - for (j = 0; j < intel_crtc->num_scalers; j++) { - if (!scaler_state->scalers[j].in_use) { - scaler_state->scalers[j].in_use = 1; - *scaler_id = j; - DRM_DEBUG_KMS("Attached scaler id %u.%u to %s:%d\n", - intel_crtc->pipe, *scaler_id, name, idx); - break; - } - } - } - - if (WARN_ON(*scaler_id < 0)) { - DRM_DEBUG_KMS("Cannot find scaler for %s:%d\n", name, idx); - continue; - } - - /* set scaler mode */ - if ((INTEL_GEN(dev_priv) >= 9) && - plane_state && plane_state->base.fb && - plane_state->base.fb->format->format == - DRM_FORMAT_NV12) { - if (INTEL_GEN(dev_priv) == 9 && - !IS_GEMINILAKE(dev_priv) && - !IS_SKYLAKE(dev_priv)) - scaler_state->scalers[*scaler_id].mode = - SKL_PS_SCALER_MODE_NV12; - else - scaler_state->scalers[*scaler_id].mode = - PS_SCALER_MODE_PLANAR; - } else if (num_scalers_need == 1 && intel_crtc->pipe != PIPE_C) { - /* - * when only 1 scaler is in use on either pipe A or B, - * scaler 0 operates in high quality (HQ) mode. - * In this case use scaler 0 to take advantage of HQ mode - */ - *scaler_id = 0; - scaler_state->scalers[0].in_use = 1; - scaler_state->scalers[0].mode = PS_SCALER_MODE_HQ; - scaler_state->scalers[1].in_use = 0; - } else { - scaler_state->scalers[*scaler_id].mode = PS_SCALER_MODE_DYN; - } + intel_atomic_setup_scaler(scaler_state, num_scalers_need, + intel_crtc, name, idx, + plane_state, scaler_id); } return 0; diff --git a/drivers/gpu/drm/i915/intel_atomic_plane.c b/drivers/gpu/drm/i915/intel_atomic_plane.c index aabebe0d2e9b..db0965904439 100644 --- a/drivers/gpu/drm/i915/intel_atomic_plane.c +++ b/drivers/gpu/drm/i915/intel_atomic_plane.c @@ -31,33 +31,37 @@ * prepare/check/commit/cleanup steps. */ -#include <drm/drmP.h> #include <drm/drm_atomic_helper.h> +#include <drm/drm_fourcc.h> #include <drm/drm_plane_helper.h> + #include "intel_drv.h" -/** - * intel_create_plane_state - create plane state object - * @plane: drm plane - * - * Allocates a fresh plane state for the given plane and sets some of - * the state values to sensible initial values. - * - * Returns: A newly allocated plane state, or NULL on failure - */ -struct intel_plane_state * -intel_create_plane_state(struct drm_plane *plane) +struct intel_plane *intel_plane_alloc(void) { - struct intel_plane_state *state; + struct intel_plane_state *plane_state; + struct intel_plane *plane; - state = kzalloc(sizeof(*state), GFP_KERNEL); - if (!state) - return NULL; + plane = kzalloc(sizeof(*plane), GFP_KERNEL); + if (!plane) + return ERR_PTR(-ENOMEM); + + plane_state = kzalloc(sizeof(*plane_state), GFP_KERNEL); + if (!plane_state) { + kfree(plane); + return ERR_PTR(-ENOMEM); + } - state->base.plane = plane; - state->base.rotation = DRM_MODE_ROTATE_0; + __drm_atomic_helper_plane_reset(&plane->base, &plane_state->base); + plane_state->scaler_id = -1; - return state; + return plane; +} + +void intel_plane_free(struct intel_plane *plane) +{ + intel_plane_destroy_state(&plane->base, plane->base.state); + kfree(plane); } /** @@ -108,38 +112,39 @@ intel_plane_destroy_state(struct drm_plane *plane, } int intel_plane_atomic_check_with_state(const struct intel_crtc_state *old_crtc_state, - struct intel_crtc_state *crtc_state, + struct intel_crtc_state *new_crtc_state, const struct intel_plane_state *old_plane_state, - struct intel_plane_state *intel_state) + struct intel_plane_state *new_plane_state) { - struct drm_plane *plane = intel_state->base.plane; - struct drm_plane_state *state = &intel_state->base; - struct intel_plane *intel_plane = to_intel_plane(plane); + struct intel_plane *plane = to_intel_plane(new_plane_state->base.plane); int ret; - if (!intel_state->base.crtc && !old_plane_state->base.crtc) + new_crtc_state->active_planes &= ~BIT(plane->id); + new_crtc_state->nv12_planes &= ~BIT(plane->id); + new_plane_state->base.visible = false; + + if (!new_plane_state->base.crtc && !old_plane_state->base.crtc) return 0; - intel_state->base.visible = false; - ret = intel_plane->check_plane(crtc_state, intel_state); + ret = plane->check_plane(new_crtc_state, new_plane_state); if (ret) return ret; /* FIXME pre-g4x don't work like this */ - if (state->visible) - crtc_state->active_planes |= BIT(intel_plane->id); - else - crtc_state->active_planes &= ~BIT(intel_plane->id); + if (new_plane_state->base.visible) + new_crtc_state->active_planes |= BIT(plane->id); - if (state->visible && state->fb->format->format == DRM_FORMAT_NV12) - crtc_state->nv12_planes |= BIT(intel_plane->id); - else - crtc_state->nv12_planes &= ~BIT(intel_plane->id); + if (new_plane_state->base.visible && + new_plane_state->base.fb->format->format == DRM_FORMAT_NV12) + new_crtc_state->nv12_planes |= BIT(plane->id); + + if (new_plane_state->base.visible || old_plane_state->base.visible) + new_crtc_state->update_planes |= BIT(plane->id); return intel_plane_atomic_calc_changes(old_crtc_state, - &crtc_state->base, + &new_crtc_state->base, old_plane_state, - state); + &new_plane_state->base); } static int intel_plane_atomic_check(struct drm_plane *plane, @@ -152,6 +157,7 @@ static int intel_plane_atomic_check(struct drm_plane *plane, const struct drm_crtc_state *old_crtc_state; struct drm_crtc_state *new_crtc_state; + new_plane_state->visible = false; if (!crtc) return 0; @@ -164,29 +170,123 @@ static int intel_plane_atomic_check(struct drm_plane *plane, to_intel_plane_state(new_plane_state)); } -static void intel_plane_atomic_update(struct drm_plane *plane, - struct drm_plane_state *old_state) +static struct intel_plane * +skl_next_plane_to_commit(struct intel_atomic_state *state, + struct intel_crtc *crtc, + struct skl_ddb_entry entries_y[I915_MAX_PLANES], + struct skl_ddb_entry entries_uv[I915_MAX_PLANES], + unsigned int *update_mask) +{ + struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + struct intel_plane_state *plane_state; + struct intel_plane *plane; + int i; + + if (*update_mask == 0) + return NULL; + + for_each_new_intel_plane_in_state(state, plane, plane_state, i) { + enum plane_id plane_id = plane->id; + + if (crtc->pipe != plane->pipe || + !(*update_mask & BIT(plane_id))) + continue; + + if (skl_ddb_allocation_overlaps(&crtc_state->wm.skl.plane_ddb_y[plane_id], + entries_y, + I915_MAX_PLANES, plane_id) || + skl_ddb_allocation_overlaps(&crtc_state->wm.skl.plane_ddb_uv[plane_id], + entries_uv, + I915_MAX_PLANES, plane_id)) + continue; + + *update_mask &= ~BIT(plane_id); + entries_y[plane_id] = crtc_state->wm.skl.plane_ddb_y[plane_id]; + entries_uv[plane_id] = crtc_state->wm.skl.plane_ddb_uv[plane_id]; + + return plane; + } + + /* should never happen */ + WARN_ON(1); + + return NULL; +} + +void skl_update_planes_on_crtc(struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct intel_crtc_state *old_crtc_state = + intel_atomic_get_old_crtc_state(state, crtc); + struct intel_crtc_state *new_crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + struct skl_ddb_entry entries_y[I915_MAX_PLANES]; + struct skl_ddb_entry entries_uv[I915_MAX_PLANES]; + u32 update_mask = new_crtc_state->update_planes; + struct intel_plane *plane; + + memcpy(entries_y, old_crtc_state->wm.skl.plane_ddb_y, + sizeof(old_crtc_state->wm.skl.plane_ddb_y)); + memcpy(entries_uv, old_crtc_state->wm.skl.plane_ddb_uv, + sizeof(old_crtc_state->wm.skl.plane_ddb_uv)); + + while ((plane = skl_next_plane_to_commit(state, crtc, + entries_y, entries_uv, + &update_mask))) { + struct intel_plane_state *new_plane_state = + intel_atomic_get_new_plane_state(state, plane); + + if (new_plane_state->base.visible) { + trace_intel_update_plane(&plane->base, crtc); + plane->update_plane(plane, new_crtc_state, new_plane_state); + } else if (new_plane_state->slave) { + struct intel_plane *master = + new_plane_state->linked_plane; + + /* + * We update the slave plane from this function because + * programming it from the master plane's update_plane + * callback runs into issues when the Y plane is + * reassigned, disabled or used by a different plane. + * + * The slave plane is updated with the master plane's + * plane_state. + */ + new_plane_state = + intel_atomic_get_new_plane_state(state, master); + + trace_intel_update_plane(&plane->base, crtc); + plane->update_slave(plane, new_crtc_state, new_plane_state); + } else { + trace_intel_disable_plane(&plane->base, crtc); + plane->disable_plane(plane, new_crtc_state); + } + } +} + +void i9xx_update_planes_on_crtc(struct intel_atomic_state *state, + struct intel_crtc *crtc) { - struct intel_atomic_state *state = to_intel_atomic_state(old_state->state); - struct intel_plane *intel_plane = to_intel_plane(plane); - const struct intel_plane_state *new_plane_state = - intel_atomic_get_new_plane_state(state, intel_plane); - struct drm_crtc *crtc = new_plane_state->base.crtc ?: old_state->crtc; - - if (new_plane_state->base.visible) { - const struct intel_crtc_state *new_crtc_state = - intel_atomic_get_new_crtc_state(state, to_intel_crtc(crtc)); - - trace_intel_update_plane(plane, - to_intel_crtc(crtc)); - - intel_plane->update_plane(intel_plane, - new_crtc_state, new_plane_state); - } else { - trace_intel_disable_plane(plane, - to_intel_crtc(crtc)); - - intel_plane->disable_plane(intel_plane, to_intel_crtc(crtc)); + struct intel_crtc_state *new_crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + u32 update_mask = new_crtc_state->update_planes; + struct intel_plane_state *new_plane_state; + struct intel_plane *plane; + int i; + + for_each_new_intel_plane_in_state(state, plane, new_plane_state, i) { + if (crtc->pipe != plane->pipe || + !(update_mask & BIT(plane->id))) + continue; + + if (new_plane_state->base.visible) { + trace_intel_update_plane(&plane->base, crtc); + plane->update_plane(plane, new_crtc_state, new_plane_state); + } else { + trace_intel_disable_plane(&plane->base, crtc); + plane->disable_plane(plane, new_crtc_state); + } } } @@ -194,7 +294,6 @@ const struct drm_plane_helper_funcs intel_plane_helper_funcs = { .prepare_fb = intel_prepare_plane_fb, .cleanup_fb = intel_cleanup_plane_fb, .atomic_check = intel_plane_atomic_check, - .atomic_update = intel_plane_atomic_update, }; /** @@ -212,7 +311,7 @@ int intel_plane_atomic_get_property(struct drm_plane *plane, const struct drm_plane_state *state, struct drm_property *property, - uint64_t *val) + u64 *val) { DRM_DEBUG_KMS("Unknown property [PROP:%d:%s]\n", property->base.id, property->name); @@ -235,7 +334,7 @@ int intel_plane_atomic_set_property(struct drm_plane *plane, struct drm_plane_state *state, struct drm_property *property, - uint64_t val) + u64 val) { DRM_DEBUG_KMS("Unknown property [PROP:%d:%s]\n", property->base.id, property->name); diff --git a/drivers/gpu/drm/i915/intel_audio.c b/drivers/gpu/drm/i915/intel_audio.c index ee3ca2de983b..5104c6bbd66f 100644 --- a/drivers/gpu/drm/i915/intel_audio.c +++ b/drivers/gpu/drm/i915/intel_audio.c @@ -27,7 +27,6 @@ #include <drm/intel_lpe_audio.h> #include "intel_drv.h" -#include <drm/drmP.h> #include <drm/drm_edid.h> #include "i915_drv.h" @@ -153,32 +152,32 @@ static const struct { int n; int cts; } hdmi_aud_ncts[] = { - { 44100, TMDS_296M, 4459, 234375 }, - { 44100, TMDS_297M, 4704, 247500 }, - { 48000, TMDS_296M, 5824, 281250 }, - { 48000, TMDS_297M, 5120, 247500 }, { 32000, TMDS_296M, 5824, 421875 }, { 32000, TMDS_297M, 3072, 222750 }, + { 32000, TMDS_593M, 5824, 843750 }, + { 32000, TMDS_594M, 3072, 445500 }, + { 44100, TMDS_296M, 4459, 234375 }, + { 44100, TMDS_297M, 4704, 247500 }, + { 44100, TMDS_593M, 8918, 937500 }, + { 44100, TMDS_594M, 9408, 990000 }, { 88200, TMDS_296M, 8918, 234375 }, { 88200, TMDS_297M, 9408, 247500 }, - { 96000, TMDS_296M, 11648, 281250 }, - { 96000, TMDS_297M, 10240, 247500 }, + { 88200, TMDS_593M, 17836, 937500 }, + { 88200, TMDS_594M, 18816, 990000 }, { 176400, TMDS_296M, 17836, 234375 }, { 176400, TMDS_297M, 18816, 247500 }, - { 192000, TMDS_296M, 23296, 281250 }, - { 192000, TMDS_297M, 20480, 247500 }, - { 44100, TMDS_593M, 8918, 937500 }, - { 44100, TMDS_594M, 9408, 990000 }, + { 176400, TMDS_593M, 35672, 937500 }, + { 176400, TMDS_594M, 37632, 990000 }, + { 48000, TMDS_296M, 5824, 281250 }, + { 48000, TMDS_297M, 5120, 247500 }, { 48000, TMDS_593M, 5824, 562500 }, { 48000, TMDS_594M, 6144, 594000 }, - { 32000, TMDS_593M, 5824, 843750 }, - { 32000, TMDS_594M, 3072, 445500 }, - { 88200, TMDS_593M, 17836, 937500 }, - { 88200, TMDS_594M, 18816, 990000 }, + { 96000, TMDS_296M, 11648, 281250 }, + { 96000, TMDS_297M, 10240, 247500 }, { 96000, TMDS_593M, 11648, 562500 }, { 96000, TMDS_594M, 12288, 594000 }, - { 176400, TMDS_593M, 35672, 937500 }, - { 176400, TMDS_594M, 37632, 990000 }, + { 192000, TMDS_296M, 23296, 281250 }, + { 192000, TMDS_297M, 20480, 247500 }, { 192000, TMDS_593M, 23296, 562500 }, { 192000, TMDS_594M, 24576, 594000 }, }; @@ -749,7 +748,8 @@ static void i915_audio_component_get_power(struct device *kdev) static void i915_audio_component_put_power(struct device *kdev) { - intel_display_power_put(kdev_to_i915(kdev), POWER_DOMAIN_AUDIO); + intel_display_power_put_unchecked(kdev_to_i915(kdev), + POWER_DOMAIN_AUDIO); } static void i915_audio_component_codec_wake_override(struct device *kdev, @@ -758,7 +758,7 @@ static void i915_audio_component_codec_wake_override(struct device *kdev, struct drm_i915_private *dev_priv = kdev_to_i915(kdev); u32 tmp; - if (!IS_GEN9(dev_priv)) + if (!IS_GEN(dev_priv, 9)) return; i915_audio_component_get_power(kdev); @@ -929,6 +929,9 @@ static int i915_audio_component_bind(struct device *i915_kdev, if (WARN_ON(acomp->base.ops || acomp->base.dev)) return -EEXIST; + if (WARN_ON(!device_link_add(hda_kdev, i915_kdev, DL_FLAG_STATELESS))) + return -ENOMEM; + drm_modeset_lock_all(&dev_priv->drm); acomp->base.ops = &i915_audio_component_ops; acomp->base.dev = i915_kdev; @@ -952,6 +955,8 @@ static void i915_audio_component_unbind(struct device *i915_kdev, acomp->base.dev = NULL; dev_priv->audio_component = NULL; drm_modeset_unlock_all(&dev_priv->drm); + + device_link_remove(hda_kdev, i915_kdev); } static const struct component_ops i915_audio_component_bind_ops = { @@ -979,7 +984,9 @@ void i915_audio_component_init(struct drm_i915_private *dev_priv) { int ret; - ret = component_add(dev_priv->drm.dev, &i915_audio_component_bind_ops); + ret = component_add_typed(dev_priv->drm.dev, + &i915_audio_component_bind_ops, + I915_COMPONENT_AUDIO); if (ret < 0) { DRM_ERROR("failed to add audio component (%d)\n", ret); /* continue with reduced functionality */ diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c index 1faa494e2bc9..4364f42cac6b 100644 --- a/drivers/gpu/drm/i915/intel_bios.c +++ b/drivers/gpu/drm/i915/intel_bios.c @@ -26,7 +26,6 @@ */ #include <drm/drm_dp_helper.h> -#include <drm/drmP.h> #include <drm/i915_drm.h> #include "i915_drv.h" @@ -420,6 +419,13 @@ parse_general_features(struct drm_i915_private *dev_priv, intel_bios_ssc_frequency(dev_priv, general->ssc_freq); dev_priv->vbt.display_clock_mode = general->display_clock_mode; dev_priv->vbt.fdi_rx_polarity_inverted = general->fdi_rx_polarity_inverted; + if (bdb->version >= 181) { + dev_priv->vbt.orientation = general->rotate_180 ? + DRM_MODE_PANEL_ORIENTATION_BOTTOM_UP : + DRM_MODE_PANEL_ORIENTATION_NORMAL; + } else { + dev_priv->vbt.orientation = DRM_MODE_PANEL_ORIENTATION_UNKNOWN; + } DRM_DEBUG_KMS("BDB_GENERAL_FEATURES int_tv_support %d int_crt_support %d lvds_use_ssc %d lvds_ssc_freq %d display_clock_mode %d fdi_rx_polarity_inverted %d\n", dev_priv->vbt.int_tv_support, dev_priv->vbt.int_crt_support, @@ -446,7 +452,7 @@ parse_sdvo_device_mapping(struct drm_i915_private *dev_priv, u8 bdb_version) * Only parse SDVO mappings on gens that could have SDVO. This isn't * accurate and doesn't have to be, as long as it's not too strict. */ - if (!IS_GEN(dev_priv, 3, 7)) { + if (!IS_GEN_RANGE(dev_priv, 3, 7)) { DRM_DEBUG_KMS("Skipping SDVO device mapping\n"); return; } @@ -852,6 +858,30 @@ parse_mipi_config(struct drm_i915_private *dev_priv, parse_dsi_backlight_ports(dev_priv, bdb->version, port); + /* FIXME is the 90 vs. 270 correct? */ + switch (config->rotation) { + case ENABLE_ROTATION_0: + /* + * Most (all?) VBTs claim 0 degrees despite having + * an upside down panel, thus we do not trust this. + */ + dev_priv->vbt.dsi.orientation = + DRM_MODE_PANEL_ORIENTATION_UNKNOWN; + break; + case ENABLE_ROTATION_90: + dev_priv->vbt.dsi.orientation = + DRM_MODE_PANEL_ORIENTATION_RIGHT_UP; + break; + case ENABLE_ROTATION_180: + dev_priv->vbt.dsi.orientation = + DRM_MODE_PANEL_ORIENTATION_BOTTOM_UP; + break; + case ENABLE_ROTATION_270: + dev_priv->vbt.dsi.orientation = + DRM_MODE_PANEL_ORIENTATION_LEFT_UP; + break; + } + /* We have mandatory mipi config blocks. Initialize as generic panel */ dev_priv->vbt.dsi.panel_id = MIPI_DSI_GENERIC_PANEL_ID; } @@ -1355,8 +1385,15 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port, info->supports_dp = is_dp; info->supports_edp = is_edp; - DRM_DEBUG_KMS("Port %c VBT info: DP:%d HDMI:%d DVI:%d EDP:%d CRT:%d\n", - port_name(port), is_dp, is_hdmi, is_dvi, is_edp, is_crt); + if (bdb_version >= 195) + info->supports_typec_usb = child->dp_usb_type_c; + + if (bdb_version >= 209) + info->supports_tbt = child->tbt; + + DRM_DEBUG_KMS("Port %c VBT info: DP:%d HDMI:%d DVI:%d EDP:%d CRT:%d TCUSB:%d TBT:%d\n", + port_name(port), is_dp, is_hdmi, is_dvi, is_edp, is_crt, + info->supports_typec_usb, info->supports_tbt); if (is_edp && is_dvi) DRM_DEBUG_KMS("Internal DP port %c is TMDS compatible\n", @@ -1626,9 +1663,17 @@ init_vbt_missing_defaults(struct drm_i915_private *dev_priv) struct ddi_vbt_port_info *info = &dev_priv->vbt.ddi_port_info[port]; + /* + * VBT has the TypeC mode (native,TBT/USB) and we don't want + * to detect it. + */ + if (intel_port_is_tc(dev_priv, port)) + continue; + info->supports_dvi = (port != PORT_A && port != PORT_E); info->supports_hdmi = info->supports_dvi; info->supports_dp = (port != PORT_E); + info->supports_edp = (port == PORT_A); } } @@ -1721,7 +1766,7 @@ void intel_bios_init(struct drm_i915_private *dev_priv) const struct bdb_header *bdb; u8 __iomem *bios = NULL; - if (INTEL_INFO(dev_priv)->num_pipes == 0) { + if (!HAS_DISPLAY(dev_priv)) { DRM_DEBUG_KMS("Skipping VBT init due to disabled display.\n"); return; } @@ -1909,6 +1954,15 @@ bool intel_bios_is_port_present(struct drm_i915_private *dev_priv, enum port por }; int i; + if (HAS_DDI(dev_priv)) { + const struct ddi_vbt_port_info *port_info = + &dev_priv->vbt.ddi_port_info[port]; + + return port_info->supports_dp || + port_info->supports_dvi || + port_info->supports_hdmi; + } + /* FIXME maybe deal with port A as well? */ if (WARN_ON(port == PORT_A) || port >= ARRAY_SIZE(port_mapping)) return false; @@ -2039,17 +2093,17 @@ bool intel_bios_is_dsi_present(struct drm_i915_private *dev_priv, dvo_port = child->dvo_port; - switch (dvo_port) { - case DVO_PORT_MIPIA: - case DVO_PORT_MIPIC: + if (dvo_port == DVO_PORT_MIPIA || + (dvo_port == DVO_PORT_MIPIB && IS_ICELAKE(dev_priv)) || + (dvo_port == DVO_PORT_MIPIC && !IS_ICELAKE(dev_priv))) { if (port) *port = dvo_port - DVO_PORT_MIPIA; return true; - case DVO_PORT_MIPIB: - case DVO_PORT_MIPID: + } else if (dvo_port == DVO_PORT_MIPIB || + dvo_port == DVO_PORT_MIPIC || + dvo_port == DVO_PORT_MIPID) { DRM_DEBUG_KMS("VBT has unsupported DSI port %c\n", port_name(dvo_port - DVO_PORT_MIPIA)); - break; } } @@ -2159,3 +2213,49 @@ intel_bios_is_lspcon_present(struct drm_i915_private *dev_priv, return false; } + +enum aux_ch intel_bios_port_aux_ch(struct drm_i915_private *dev_priv, + enum port port) +{ + const struct ddi_vbt_port_info *info = + &dev_priv->vbt.ddi_port_info[port]; + enum aux_ch aux_ch; + + if (!info->alternate_aux_channel) { + aux_ch = (enum aux_ch)port; + + DRM_DEBUG_KMS("using AUX %c for port %c (platform default)\n", + aux_ch_name(aux_ch), port_name(port)); + return aux_ch; + } + + switch (info->alternate_aux_channel) { + case DP_AUX_A: + aux_ch = AUX_CH_A; + break; + case DP_AUX_B: + aux_ch = AUX_CH_B; + break; + case DP_AUX_C: + aux_ch = AUX_CH_C; + break; + case DP_AUX_D: + aux_ch = AUX_CH_D; + break; + case DP_AUX_E: + aux_ch = AUX_CH_E; + break; + case DP_AUX_F: + aux_ch = AUX_CH_F; + break; + default: + MISSING_CASE(info->alternate_aux_channel); + aux_ch = AUX_CH_A; + break; + } + + DRM_DEBUG_KMS("using AUX %c for port %c (VBT)\n", + aux_ch_name(aux_ch), port_name(port)); + + return aux_ch; +} diff --git a/drivers/gpu/drm/i915/intel_breadcrumbs.c b/drivers/gpu/drm/i915/intel_breadcrumbs.c index 84bf8d827136..09ed90c0ba00 100644 --- a/drivers/gpu/drm/i915/intel_breadcrumbs.c +++ b/drivers/gpu/drm/i915/intel_breadcrumbs.c @@ -27,186 +27,146 @@ #include "i915_drv.h" -#ifdef CONFIG_SMP -#define task_asleep(tsk) ((tsk)->state & TASK_NORMAL && !(tsk)->on_cpu) -#else -#define task_asleep(tsk) ((tsk)->state & TASK_NORMAL) -#endif +#define task_asleep(tsk) ((tsk)->state & TASK_NORMAL && !(tsk)->on_rq) -static unsigned int __intel_breadcrumbs_wakeup(struct intel_breadcrumbs *b) +static void irq_enable(struct intel_engine_cs *engine) { - struct intel_wait *wait; - unsigned int result = 0; - - lockdep_assert_held(&b->irq_lock); - - wait = b->irq_wait; - if (wait) { - /* - * N.B. Since task_asleep() and ttwu are not atomic, the - * waiter may actually go to sleep after the check, causing - * us to suppress a valid wakeup. We prefer to reduce the - * number of false positive missed_breadcrumb() warnings - * at the expense of a few false negatives, as it it easy - * to trigger a false positive under heavy load. Enough - * signal should remain from genuine missed_breadcrumb() - * for us to detect in CI. - */ - bool was_asleep = task_asleep(wait->tsk); - - result = ENGINE_WAKEUP_WAITER; - if (wake_up_process(wait->tsk) && was_asleep) - result |= ENGINE_WAKEUP_ASLEEP; - } + if (!engine->irq_enable) + return; - return result; + /* Caller disables interrupts */ + spin_lock(&engine->i915->irq_lock); + engine->irq_enable(engine); + spin_unlock(&engine->i915->irq_lock); } -unsigned int intel_engine_wakeup(struct intel_engine_cs *engine) +static void irq_disable(struct intel_engine_cs *engine) { - struct intel_breadcrumbs *b = &engine->breadcrumbs; - unsigned long flags; - unsigned int result; - - spin_lock_irqsave(&b->irq_lock, flags); - result = __intel_breadcrumbs_wakeup(b); - spin_unlock_irqrestore(&b->irq_lock, flags); - - return result; -} + if (!engine->irq_disable) + return; -static unsigned long wait_timeout(void) -{ - return round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES); + /* Caller disables interrupts */ + spin_lock(&engine->i915->irq_lock); + engine->irq_disable(engine); + spin_unlock(&engine->i915->irq_lock); } -static noinline void missed_breadcrumb(struct intel_engine_cs *engine) +static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b) { - if (GEM_SHOW_DEBUG()) { - struct drm_printer p = drm_debug_printer(__func__); + lockdep_assert_held(&b->irq_lock); - intel_engine_dump(engine, &p, - "%s missed breadcrumb at %pS\n", - engine->name, __builtin_return_address(0)); - } + GEM_BUG_ON(!b->irq_enabled); + if (!--b->irq_enabled) + irq_disable(container_of(b, + struct intel_engine_cs, + breadcrumbs)); - set_bit(engine->id, &engine->i915->gpu_error.missed_irq_rings); + b->irq_armed = false; } -static void intel_breadcrumbs_hangcheck(struct timer_list *t) +void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine) { - struct intel_engine_cs *engine = - from_timer(engine, t, breadcrumbs.hangcheck); struct intel_breadcrumbs *b = &engine->breadcrumbs; - unsigned int irq_count; if (!b->irq_armed) return; - irq_count = READ_ONCE(b->irq_count); - if (b->hangcheck_interrupts != irq_count) { - b->hangcheck_interrupts = irq_count; - mod_timer(&b->hangcheck, wait_timeout()); - return; - } + spin_lock_irq(&b->irq_lock); + if (b->irq_armed) + __intel_breadcrumbs_disarm_irq(b); + spin_unlock_irq(&b->irq_lock); +} - /* We keep the hangcheck timer alive until we disarm the irq, even - * if there are no waiters at present. - * - * If the waiter was currently running, assume it hasn't had a chance - * to process the pending interrupt (e.g, low priority task on a loaded - * system) and wait until it sleeps before declaring a missed interrupt. - * - * If the waiter was asleep (and not even pending a wakeup), then we - * must have missed an interrupt as the GPU has stopped advancing - * but we still have a waiter. Assuming all batches complete within - * DRM_I915_HANGCHECK_JIFFIES [1.5s]! - */ - if (intel_engine_wakeup(engine) & ENGINE_WAKEUP_ASLEEP) { - missed_breadcrumb(engine); - mod_timer(&b->fake_irq, jiffies + 1); - } else { - mod_timer(&b->hangcheck, wait_timeout()); - } +static inline bool __request_completed(const struct i915_request *rq) +{ + return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno); } -static void intel_breadcrumbs_fake_irq(struct timer_list *t) +bool intel_engine_breadcrumbs_irq(struct intel_engine_cs *engine) { - struct intel_engine_cs *engine = - from_timer(engine, t, breadcrumbs.fake_irq); struct intel_breadcrumbs *b = &engine->breadcrumbs; + struct intel_context *ce, *cn; + struct list_head *pos, *next; + LIST_HEAD(signal); - /* - * The timer persists in case we cannot enable interrupts, - * or if we have previously seen seqno/interrupt incoherency - * ("missed interrupt" syndrome, better known as a "missed breadcrumb"). - * Here the worker will wake up every jiffie in order to kick the - * oldest waiter to do the coherent seqno check. - */ + spin_lock(&b->irq_lock); - spin_lock_irq(&b->irq_lock); - if (b->irq_armed && !__intel_breadcrumbs_wakeup(b)) - __intel_engine_disarm_breadcrumbs(engine); - spin_unlock_irq(&b->irq_lock); - if (!b->irq_armed) - return; + if (b->irq_armed && list_empty(&b->signalers)) + __intel_breadcrumbs_disarm_irq(b); - /* If the user has disabled the fake-irq, restore the hangchecking */ - if (!test_bit(engine->id, &engine->i915->gpu_error.missed_irq_rings)) { - mod_timer(&b->hangcheck, wait_timeout()); - return; - } + list_for_each_entry_safe(ce, cn, &b->signalers, signal_link) { + GEM_BUG_ON(list_empty(&ce->signals)); - mod_timer(&b->fake_irq, jiffies + 1); -} + list_for_each_safe(pos, next, &ce->signals) { + struct i915_request *rq = + list_entry(pos, typeof(*rq), signal_link); -static void irq_enable(struct intel_engine_cs *engine) -{ - /* - * FIXME: Ideally we want this on the API boundary, but for the - * sake of testing with mock breadcrumbs (no HW so unable to - * enable irqs) we place it deep within the bowels, at the point - * of no return. - */ - GEM_BUG_ON(!intel_irqs_enabled(engine->i915)); + if (!__request_completed(rq)) + break; - /* Enabling the IRQ may miss the generation of the interrupt, but - * we still need to force the barrier before reading the seqno, - * just in case. - */ - set_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted); + GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_SIGNAL, + &rq->fence.flags)); - /* Caller disables interrupts */ - if (engine->irq_enable) { - spin_lock(&engine->i915->irq_lock); - engine->irq_enable(engine); - spin_unlock(&engine->i915->irq_lock); + /* + * Queue for execution after dropping the signaling + * spinlock as the callback chain may end up adding + * more signalers to the same context or engine. + */ + i915_request_get(rq); + + /* + * We may race with direct invocation of + * dma_fence_signal(), e.g. i915_request_retire(), + * so we need to acquire our reference to the request + * before we cancel the breadcrumb. + */ + clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); + list_add_tail(&rq->signal_link, &signal); + } + + /* + * We process the list deletion in bulk, only using a list_add + * (not list_move) above but keeping the status of + * rq->signal_link known with the I915_FENCE_FLAG_SIGNAL bit. + */ + if (!list_is_first(pos, &ce->signals)) { + /* Advance the list to the first incomplete request */ + __list_del_many(&ce->signals, pos); + if (&ce->signals == pos) /* now empty */ + list_del_init(&ce->signal_link); + } } -} -static void irq_disable(struct intel_engine_cs *engine) -{ - /* Caller disables interrupts */ - if (engine->irq_disable) { - spin_lock(&engine->i915->irq_lock); - engine->irq_disable(engine); - spin_unlock(&engine->i915->irq_lock); + spin_unlock(&b->irq_lock); + + list_for_each_safe(pos, next, &signal) { + struct i915_request *rq = + list_entry(pos, typeof(*rq), signal_link); + + dma_fence_signal(&rq->fence); + i915_request_put(rq); } + + return !list_empty(&signal); } -void __intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine) +bool intel_engine_signal_breadcrumbs(struct intel_engine_cs *engine) { - struct intel_breadcrumbs *b = &engine->breadcrumbs; + bool result; - lockdep_assert_held(&b->irq_lock); - GEM_BUG_ON(b->irq_wait); - GEM_BUG_ON(!b->irq_armed); + local_irq_disable(); + result = intel_engine_breadcrumbs_irq(engine); + local_irq_enable(); - GEM_BUG_ON(!b->irq_enabled); - if (!--b->irq_enabled) - irq_disable(engine); + return result; +} - b->irq_armed = false; +static void signal_irq_work(struct irq_work *work) +{ + struct intel_engine_cs *engine = + container_of(work, typeof(*engine), breadcrumbs.irq_work); + + intel_engine_breadcrumbs_irq(engine); } void intel_engine_pin_breadcrumbs_irq(struct intel_engine_cs *engine) @@ -231,666 +191,155 @@ void intel_engine_unpin_breadcrumbs_irq(struct intel_engine_cs *engine) spin_unlock_irq(&b->irq_lock); } -void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine) -{ - struct intel_breadcrumbs *b = &engine->breadcrumbs; - struct intel_wait *wait, *n; - - if (!b->irq_armed) - return; - - /* - * We only disarm the irq when we are idle (all requests completed), - * so if the bottom-half remains asleep, it missed the request - * completion. - */ - if (intel_engine_wakeup(engine) & ENGINE_WAKEUP_ASLEEP) - missed_breadcrumb(engine); - - spin_lock_irq(&b->rb_lock); - - spin_lock(&b->irq_lock); - b->irq_wait = NULL; - if (b->irq_armed) - __intel_engine_disarm_breadcrumbs(engine); - spin_unlock(&b->irq_lock); - - rbtree_postorder_for_each_entry_safe(wait, n, &b->waiters, node) { - GEM_BUG_ON(!intel_engine_signaled(engine, wait->seqno)); - RB_CLEAR_NODE(&wait->node); - wake_up_process(wait->tsk); - } - b->waiters = RB_ROOT; - - spin_unlock_irq(&b->rb_lock); -} - -static bool use_fake_irq(const struct intel_breadcrumbs *b) -{ - const struct intel_engine_cs *engine = - container_of(b, struct intel_engine_cs, breadcrumbs); - - if (!test_bit(engine->id, &engine->i915->gpu_error.missed_irq_rings)) - return false; - - /* - * Only start with the heavy weight fake irq timer if we have not - * seen any interrupts since enabling it the first time. If the - * interrupts are still arriving, it means we made a mistake in our - * engine->seqno_barrier(), a timing error that should be transient - * and unlikely to reoccur. - */ - return READ_ONCE(b->irq_count) == b->hangcheck_interrupts; -} - -static void enable_fake_irq(struct intel_breadcrumbs *b) -{ - /* Ensure we never sleep indefinitely */ - if (!b->irq_enabled || use_fake_irq(b)) - mod_timer(&b->fake_irq, jiffies + 1); - else - mod_timer(&b->hangcheck, wait_timeout()); -} - -static bool __intel_breadcrumbs_enable_irq(struct intel_breadcrumbs *b) +static void __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b) { struct intel_engine_cs *engine = container_of(b, struct intel_engine_cs, breadcrumbs); - struct drm_i915_private *i915 = engine->i915; - bool enabled; lockdep_assert_held(&b->irq_lock); if (b->irq_armed) - return false; + return; - /* The breadcrumb irq will be disarmed on the interrupt after the + /* + * The breadcrumb irq will be disarmed on the interrupt after the * waiters are signaled. This gives us a single interrupt window in * which we can add a new waiter and avoid the cost of re-enabling * the irq. */ b->irq_armed = true; - if (I915_SELFTEST_ONLY(b->mock)) { - /* For our mock objects we want to avoid interaction - * with the real hardware (which is not set up). So - * we simply pretend we have enabled the powerwell - * and the irq, and leave it up to the mock - * implementation to call intel_engine_wakeup() - * itself when it wants to simulate a user interrupt, - */ - return true; - } - - /* Since we are waiting on a request, the GPU should be busy + /* + * Since we are waiting on a request, the GPU should be busy * and should have its own rpm reference. This is tracked * by i915->gt.awake, we can forgo holding our own wakref * for the interrupt as before i915->gt.awake is released (when * the driver is idle) we disarm the breadcrumbs. */ - /* No interrupts? Kick the waiter every jiffie! */ - enabled = false; - if (!b->irq_enabled++ && - !test_bit(engine->id, &i915->gpu_error.test_irq_rings)) { + if (!b->irq_enabled++) irq_enable(engine); - enabled = true; - } - - enable_fake_irq(b); - return enabled; -} - -static inline struct intel_wait *to_wait(struct rb_node *node) -{ - return rb_entry(node, struct intel_wait, node); -} - -static inline void __intel_breadcrumbs_finish(struct intel_breadcrumbs *b, - struct intel_wait *wait) -{ - lockdep_assert_held(&b->rb_lock); - GEM_BUG_ON(b->irq_wait == wait); - - /* - * This request is completed, so remove it from the tree, mark it as - * complete, and *then* wake up the associated task. N.B. when the - * task wakes up, it will find the empty rb_node, discern that it - * has already been removed from the tree and skip the serialisation - * of the b->rb_lock and b->irq_lock. This means that the destruction - * of the intel_wait is not serialised with the interrupt handler - * by the waiter - it must instead be serialised by the caller. - */ - rb_erase(&wait->node, &b->waiters); - RB_CLEAR_NODE(&wait->node); - - if (wait->tsk->state != TASK_RUNNING) - wake_up_process(wait->tsk); /* implicit smp_wmb() */ -} - -static inline void __intel_breadcrumbs_next(struct intel_engine_cs *engine, - struct rb_node *next) -{ - struct intel_breadcrumbs *b = &engine->breadcrumbs; - - spin_lock(&b->irq_lock); - GEM_BUG_ON(!b->irq_armed); - GEM_BUG_ON(!b->irq_wait); - b->irq_wait = to_wait(next); - spin_unlock(&b->irq_lock); - - /* We always wake up the next waiter that takes over as the bottom-half - * as we may delegate not only the irq-seqno barrier to the next waiter - * but also the task of waking up concurrent waiters. - */ - if (next) - wake_up_process(to_wait(next)->tsk); } -static bool __intel_engine_add_wait(struct intel_engine_cs *engine, - struct intel_wait *wait) +void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine) { struct intel_breadcrumbs *b = &engine->breadcrumbs; - struct rb_node **p, *parent, *completed; - bool first, armed; - u32 seqno; - - GEM_BUG_ON(!wait->seqno); - - /* Insert the request into the retirement ordered list - * of waiters by walking the rbtree. If we are the oldest - * seqno in the tree (the first to be retired), then - * set ourselves as the bottom-half. - * - * As we descend the tree, prune completed branches since we hold the - * spinlock we know that the first_waiter must be delayed and can - * reduce some of the sequential wake up latency if we take action - * ourselves and wake up the completed tasks in parallel. Also, by - * removing stale elements in the tree, we may be able to reduce the - * ping-pong between the old bottom-half and ourselves as first-waiter. - */ - armed = false; - first = true; - parent = NULL; - completed = NULL; - seqno = intel_engine_get_seqno(engine); - - /* If the request completed before we managed to grab the spinlock, - * return now before adding ourselves to the rbtree. We let the - * current bottom-half handle any pending wakeups and instead - * try and get out of the way quickly. - */ - if (i915_seqno_passed(seqno, wait->seqno)) { - RB_CLEAR_NODE(&wait->node); - return first; - } - - p = &b->waiters.rb_node; - while (*p) { - parent = *p; - if (wait->seqno == to_wait(parent)->seqno) { - /* We have multiple waiters on the same seqno, select - * the highest priority task (that with the smallest - * task->prio) to serve as the bottom-half for this - * group. - */ - if (wait->tsk->prio > to_wait(parent)->tsk->prio) { - p = &parent->rb_right; - first = false; - } else { - p = &parent->rb_left; - } - } else if (i915_seqno_passed(wait->seqno, - to_wait(parent)->seqno)) { - p = &parent->rb_right; - if (i915_seqno_passed(seqno, to_wait(parent)->seqno)) - completed = parent; - else - first = false; - } else { - p = &parent->rb_left; - } - } - rb_link_node(&wait->node, parent, p); - rb_insert_color(&wait->node, &b->waiters); - - if (first) { - spin_lock(&b->irq_lock); - b->irq_wait = wait; - /* After assigning ourselves as the new bottom-half, we must - * perform a cursory check to prevent a missed interrupt. - * Either we miss the interrupt whilst programming the hardware, - * or if there was a previous waiter (for a later seqno) they - * may be woken instead of us (due to the inherent race - * in the unlocked read of b->irq_seqno_bh in the irq handler) - * and so we miss the wake up. - */ - armed = __intel_breadcrumbs_enable_irq(b); - spin_unlock(&b->irq_lock); - } - if (completed) { - /* Advance the bottom-half (b->irq_wait) before we wake up - * the waiters who may scribble over their intel_wait - * just as the interrupt handler is dereferencing it via - * b->irq_wait. - */ - if (!first) { - struct rb_node *next = rb_next(completed); - GEM_BUG_ON(next == &wait->node); - __intel_breadcrumbs_next(engine, next); - } - - do { - struct intel_wait *crumb = to_wait(completed); - completed = rb_prev(completed); - __intel_breadcrumbs_finish(b, crumb); - } while (completed); - } - - GEM_BUG_ON(!b->irq_wait); - GEM_BUG_ON(!b->irq_armed); - GEM_BUG_ON(rb_first(&b->waiters) != &b->irq_wait->node); + spin_lock_init(&b->irq_lock); + INIT_LIST_HEAD(&b->signalers); - return armed; + init_irq_work(&b->irq_work, signal_irq_work); } -bool intel_engine_add_wait(struct intel_engine_cs *engine, - struct intel_wait *wait) +void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine) { struct intel_breadcrumbs *b = &engine->breadcrumbs; - bool armed; - - spin_lock_irq(&b->rb_lock); - armed = __intel_engine_add_wait(engine, wait); - spin_unlock_irq(&b->rb_lock); - if (armed) - return armed; - - /* Make the caller recheck if its request has already started. */ - return intel_engine_has_started(engine, wait->seqno); -} + unsigned long flags; -static inline bool chain_wakeup(struct rb_node *rb, int priority) -{ - return rb && to_wait(rb)->tsk->prio <= priority; -} + spin_lock_irqsave(&b->irq_lock, flags); -static inline int wakeup_priority(struct intel_breadcrumbs *b, - struct task_struct *tsk) -{ - if (tsk == b->signaler) - return INT_MIN; + if (b->irq_enabled) + irq_enable(engine); else - return tsk->prio; -} - -static void __intel_engine_remove_wait(struct intel_engine_cs *engine, - struct intel_wait *wait) -{ - struct intel_breadcrumbs *b = &engine->breadcrumbs; - - lockdep_assert_held(&b->rb_lock); - - if (RB_EMPTY_NODE(&wait->node)) - goto out; - - if (b->irq_wait == wait) { - const int priority = wakeup_priority(b, wait->tsk); - struct rb_node *next; - - /* We are the current bottom-half. Find the next candidate, - * the first waiter in the queue on the remaining oldest - * request. As multiple seqnos may complete in the time it - * takes us to wake up and find the next waiter, we have to - * wake up that waiter for it to perform its own coherent - * completion check. - */ - next = rb_next(&wait->node); - if (chain_wakeup(next, priority)) { - /* If the next waiter is already complete, - * wake it up and continue onto the next waiter. So - * if have a small herd, they will wake up in parallel - * rather than sequentially, which should reduce - * the overall latency in waking all the completed - * clients. - * - * However, waking up a chain adds extra latency to - * the first_waiter. This is undesirable if that - * waiter is a high priority task. - */ - u32 seqno = intel_engine_get_seqno(engine); - - while (i915_seqno_passed(seqno, to_wait(next)->seqno)) { - struct rb_node *n = rb_next(next); - - __intel_breadcrumbs_finish(b, to_wait(next)); - next = n; - if (!chain_wakeup(next, priority)) - break; - } - } - - __intel_breadcrumbs_next(engine, next); - } else { - GEM_BUG_ON(rb_first(&b->waiters) == &wait->node); - } - - GEM_BUG_ON(RB_EMPTY_NODE(&wait->node)); - rb_erase(&wait->node, &b->waiters); - RB_CLEAR_NODE(&wait->node); + irq_disable(engine); -out: - GEM_BUG_ON(b->irq_wait == wait); - GEM_BUG_ON(rb_first(&b->waiters) != - (b->irq_wait ? &b->irq_wait->node : NULL)); + spin_unlock_irqrestore(&b->irq_lock, flags); } -void intel_engine_remove_wait(struct intel_engine_cs *engine, - struct intel_wait *wait) +void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine) { - struct intel_breadcrumbs *b = &engine->breadcrumbs; - - /* Quick check to see if this waiter was already decoupled from - * the tree by the bottom-half to avoid contention on the spinlock - * by the herd. - */ - if (RB_EMPTY_NODE(&wait->node)) { - GEM_BUG_ON(READ_ONCE(b->irq_wait) == wait); - return; - } - - spin_lock_irq(&b->rb_lock); - __intel_engine_remove_wait(engine, wait); - spin_unlock_irq(&b->rb_lock); } -static void signaler_set_rtpriority(void) +bool i915_request_enable_breadcrumb(struct i915_request *rq) { - struct sched_param param = { .sched_priority = 1 }; - - sched_setscheduler_nocheck(current, SCHED_FIFO, ¶m); -} + struct intel_breadcrumbs *b = &rq->engine->breadcrumbs; -static int intel_breadcrumbs_signaler(void *arg) -{ - struct intel_engine_cs *engine = arg; - struct intel_breadcrumbs *b = &engine->breadcrumbs; - struct i915_request *rq, *n; + GEM_BUG_ON(test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)); - /* Install ourselves with high priority to reduce signalling latency */ - signaler_set_rtpriority(); + if (!test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) + return true; - do { - bool do_schedule = true; - LIST_HEAD(list); - u32 seqno; + spin_lock(&b->irq_lock); + if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags) && + !__request_completed(rq)) { + struct intel_context *ce = rq->hw_context; + struct list_head *pos; - set_current_state(TASK_INTERRUPTIBLE); - if (list_empty(&b->signals)) - goto sleep; + __intel_breadcrumbs_arm_irq(b); /* - * We are either woken up by the interrupt bottom-half, - * or by a client adding a new signaller. In both cases, - * the GPU seqno may have advanced beyond our oldest signal. - * If it has, propagate the signal, remove the waiter and - * check again with the next oldest signal. Otherwise we - * need to wait for a new interrupt from the GPU or for - * a new client. + * We keep the seqno in retirement order, so we can break + * inside intel_engine_breadcrumbs_irq as soon as we've passed + * the last completed request (or seen a request that hasn't + * event started). We could iterate the timeline->requests list, + * but keeping a separate signalers_list has the advantage of + * hopefully being much smaller than the full list and so + * provides faster iteration and detection when there are no + * more interrupts required for this context. + * + * We typically expect to add new signalers in order, so we + * start looking for our insertion point from the tail of + * the list. */ - seqno = intel_engine_get_seqno(engine); - - spin_lock_irq(&b->rb_lock); - list_for_each_entry_safe(rq, n, &b->signals, signaling.link) { - u32 this = rq->signaling.wait.seqno; - - GEM_BUG_ON(!rq->signaling.wait.seqno); - - if (!i915_seqno_passed(seqno, this)) - break; - - if (likely(this == i915_request_global_seqno(rq))) { - __intel_engine_remove_wait(engine, - &rq->signaling.wait); + list_for_each_prev(pos, &ce->signals) { + struct i915_request *it = + list_entry(pos, typeof(*it), signal_link); - rq->signaling.wait.seqno = 0; - __list_del_entry(&rq->signaling.link); - - if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, - &rq->fence.flags)) { - list_add_tail(&rq->signaling.link, - &list); - i915_request_get(rq); - } - } - } - spin_unlock_irq(&b->rb_lock); - - if (!list_empty(&list)) { - local_bh_disable(); - list_for_each_entry_safe(rq, n, &list, signaling.link) { - dma_fence_signal(&rq->fence); - GEM_BUG_ON(!i915_request_completed(rq)); - i915_request_put(rq); - } - local_bh_enable(); /* kick start the tasklets */ - - /* - * If the engine is saturated we may be continually - * processing completed requests. This angers the - * NMI watchdog if we never let anything else - * have access to the CPU. Let's pretend to be nice - * and relinquish the CPU if we burn through the - * entire RT timeslice! - */ - do_schedule = need_resched(); - } - - if (unlikely(do_schedule)) { - /* Before we sleep, check for a missed seqno */ - if (current->state & TASK_NORMAL && - !list_empty(&b->signals) && - engine->irq_seqno_barrier && - test_and_clear_bit(ENGINE_IRQ_BREADCRUMB, - &engine->irq_posted)) { - engine->irq_seqno_barrier(engine); - intel_engine_wakeup(engine); - } - -sleep: - if (kthread_should_park()) - kthread_parkme(); - - if (unlikely(kthread_should_stop())) + if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno)) break; - - schedule(); } - } while (1); - __set_current_state(TASK_RUNNING); + list_add(&rq->signal_link, pos); + if (pos == &ce->signals) /* catch transitions from empty list */ + list_move_tail(&ce->signal_link, &b->signalers); - return 0; -} - -static void insert_signal(struct intel_breadcrumbs *b, - struct i915_request *request, - const u32 seqno) -{ - struct i915_request *iter; - - lockdep_assert_held(&b->rb_lock); - - /* - * A reasonable assumption is that we are called to add signals - * in sequence, as the requests are submitted for execution and - * assigned a global_seqno. This will be the case for the majority - * of internally generated signals (inter-engine signaling). - * - * Out of order waiters triggering random signaling enabling will - * be more problematic, but hopefully rare enough and the list - * small enough that the O(N) insertion sort is not an issue. - */ - - list_for_each_entry_reverse(iter, &b->signals, signaling.link) - if (i915_seqno_passed(seqno, iter->signaling.wait.seqno)) - break; - - list_add(&request->signaling.link, &iter->signaling.link); -} - -bool intel_engine_enable_signaling(struct i915_request *request, bool wakeup) -{ - struct intel_engine_cs *engine = request->engine; - struct intel_breadcrumbs *b = &engine->breadcrumbs; - struct intel_wait *wait = &request->signaling.wait; - u32 seqno; - - /* - * Note that we may be called from an interrupt handler on another - * device (e.g. nouveau signaling a fence completion causing us - * to submit a request, and so enable signaling). As such, - * we need to make sure that all other users of b->rb_lock protect - * against interrupts, i.e. use spin_lock_irqsave. - */ - - /* locked by dma_fence_enable_sw_signaling() (irqsafe fence->lock) */ - GEM_BUG_ON(!irqs_disabled()); - lockdep_assert_held(&request->lock); - - seqno = i915_request_global_seqno(request); - if (!seqno) /* will be enabled later upon execution */ - return true; - - GEM_BUG_ON(wait->seqno); - wait->tsk = b->signaler; - wait->request = request; - wait->seqno = seqno; - - /* - * Add ourselves into the list of waiters, but registering our - * bottom-half as the signaller thread. As per usual, only the oldest - * waiter (not just signaller) is tasked as the bottom-half waking - * up all completed waiters after the user interrupt. - * - * If we are the oldest waiter, enable the irq (after which we - * must double check that the seqno did not complete). - */ - spin_lock(&b->rb_lock); - insert_signal(b, request, seqno); - wakeup &= __intel_engine_add_wait(engine, wait); - spin_unlock(&b->rb_lock); - - if (wakeup) { - wake_up_process(b->signaler); - return !intel_wait_complete(wait); + set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); } + spin_unlock(&b->irq_lock); - return true; + return !__request_completed(rq); } -void intel_engine_cancel_signaling(struct i915_request *request) +void i915_request_cancel_breadcrumb(struct i915_request *rq) { - struct intel_engine_cs *engine = request->engine; - struct intel_breadcrumbs *b = &engine->breadcrumbs; + struct intel_breadcrumbs *b = &rq->engine->breadcrumbs; - GEM_BUG_ON(!irqs_disabled()); - lockdep_assert_held(&request->lock); - - if (!READ_ONCE(request->signaling.wait.seqno)) + if (!test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) return; - spin_lock(&b->rb_lock); - __intel_engine_remove_wait(engine, &request->signaling.wait); - if (fetch_and_zero(&request->signaling.wait.seqno)) - __list_del_entry(&request->signaling.link); - spin_unlock(&b->rb_lock); -} - -int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine) -{ - struct intel_breadcrumbs *b = &engine->breadcrumbs; - struct task_struct *tsk; - - spin_lock_init(&b->rb_lock); - spin_lock_init(&b->irq_lock); - - timer_setup(&b->fake_irq, intel_breadcrumbs_fake_irq, 0); - timer_setup(&b->hangcheck, intel_breadcrumbs_hangcheck, 0); - - INIT_LIST_HEAD(&b->signals); - - /* Spawn a thread to provide a common bottom-half for all signals. - * As this is an asynchronous interface we cannot steal the current - * task for handling the bottom-half to the user interrupt, therefore - * we create a thread to do the coherent seqno dance after the - * interrupt and then signal the waitqueue (via the dma-buf/fence). - */ - tsk = kthread_run(intel_breadcrumbs_signaler, engine, - "i915/signal:%d", engine->id); - if (IS_ERR(tsk)) - return PTR_ERR(tsk); - - b->signaler = tsk; - - return 0; -} - -static void cancel_fake_irq(struct intel_engine_cs *engine) -{ - struct intel_breadcrumbs *b = &engine->breadcrumbs; - - del_timer_sync(&b->fake_irq); /* may queue b->hangcheck */ - del_timer_sync(&b->hangcheck); - clear_bit(engine->id, &engine->i915->gpu_error.missed_irq_rings); -} - -void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine) -{ - struct intel_breadcrumbs *b = &engine->breadcrumbs; - unsigned long flags; - - spin_lock_irqsave(&b->irq_lock, flags); - - /* - * Leave the fake_irq timer enabled (if it is running), but clear the - * bit so that it turns itself off on its next wake up and goes back - * to the long hangcheck interval if still required. - */ - clear_bit(engine->id, &engine->i915->gpu_error.missed_irq_rings); - - if (b->irq_enabled) - irq_enable(engine); - else - irq_disable(engine); + spin_lock(&b->irq_lock); + if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) { + struct intel_context *ce = rq->hw_context; - /* - * We set the IRQ_BREADCRUMB bit when we enable the irq presuming the - * GPU is active and may have already executed the MI_USER_INTERRUPT - * before the CPU is ready to receive. However, the engine is currently - * idle (we haven't started it yet), there is no possibility for a - * missed interrupt as we enabled the irq and so we can clear the - * immediate wakeup (until a real interrupt arrives for the waiter). - */ - clear_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted); + list_del(&rq->signal_link); + if (list_empty(&ce->signals)) + list_del_init(&ce->signal_link); - spin_unlock_irqrestore(&b->irq_lock, flags); + clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); + } + spin_unlock(&b->irq_lock); } -void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine) +void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine, + struct drm_printer *p) { struct intel_breadcrumbs *b = &engine->breadcrumbs; + struct intel_context *ce; + struct i915_request *rq; - /* The engines should be idle and all requests accounted for! */ - WARN_ON(READ_ONCE(b->irq_wait)); - WARN_ON(!RB_EMPTY_ROOT(&b->waiters)); - WARN_ON(!list_empty(&b->signals)); + if (list_empty(&b->signalers)) + return; - if (!IS_ERR_OR_NULL(b->signaler)) - kthread_stop(b->signaler); + drm_printf(p, "Signals:\n"); - cancel_fake_irq(engine); + spin_lock_irq(&b->irq_lock); + list_for_each_entry(ce, &b->signalers, signal_link) { + list_for_each_entry(rq, &ce->signals, signal_link) { + drm_printf(p, "\t[%llx:%llx%s] @ %dms\n", + rq->fence.context, rq->fence.seqno, + i915_request_completed(rq) ? "!" : + i915_request_started(rq) ? "*" : + "", + jiffies_to_msecs(jiffies - rq->emitted_jiffies)); + } + } + spin_unlock_irq(&b->irq_lock); } - -#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) -#include "selftests/intel_breadcrumbs.c" -#endif diff --git a/drivers/gpu/drm/i915/intel_cdclk.c b/drivers/gpu/drm/i915/intel_cdclk.c index 8d74276029e6..15ba950dee00 100644 --- a/drivers/gpu/drm/i915/intel_cdclk.c +++ b/drivers/gpu/drm/i915/intel_cdclk.c @@ -218,7 +218,7 @@ static unsigned int intel_hpll_vco(struct drm_i915_private *dev_priv) }; const unsigned int *vco_table; unsigned int vco; - uint8_t tmp = 0; + u8 tmp = 0; /* FIXME other chipsets? */ if (IS_GM45(dev_priv)) @@ -249,13 +249,13 @@ static void g33_get_cdclk(struct drm_i915_private *dev_priv, struct intel_cdclk_state *cdclk_state) { struct pci_dev *pdev = dev_priv->drm.pdev; - static const uint8_t div_3200[] = { 12, 10, 8, 7, 5, 16 }; - static const uint8_t div_4000[] = { 14, 12, 10, 8, 6, 20 }; - static const uint8_t div_4800[] = { 20, 14, 12, 10, 8, 24 }; - static const uint8_t div_5333[] = { 20, 16, 12, 12, 8, 28 }; - const uint8_t *div_table; + static const u8 div_3200[] = { 12, 10, 8, 7, 5, 16 }; + static const u8 div_4000[] = { 14, 12, 10, 8, 6, 20 }; + static const u8 div_4800[] = { 20, 14, 12, 10, 8, 24 }; + static const u8 div_5333[] = { 20, 16, 12, 12, 8, 28 }; + const u8 *div_table; unsigned int cdclk_sel; - uint16_t tmp = 0; + u16 tmp = 0; cdclk_state->vco = intel_hpll_vco(dev_priv); @@ -330,12 +330,12 @@ static void i965gm_get_cdclk(struct drm_i915_private *dev_priv, struct intel_cdclk_state *cdclk_state) { struct pci_dev *pdev = dev_priv->drm.pdev; - static const uint8_t div_3200[] = { 16, 10, 8 }; - static const uint8_t div_4000[] = { 20, 12, 10 }; - static const uint8_t div_5333[] = { 24, 16, 14 }; - const uint8_t *div_table; + static const u8 div_3200[] = { 16, 10, 8 }; + static const u8 div_4000[] = { 20, 12, 10 }; + static const u8 div_5333[] = { 24, 16, 14 }; + const u8 *div_table; unsigned int cdclk_sel; - uint16_t tmp = 0; + u16 tmp = 0; cdclk_state->vco = intel_hpll_vco(dev_priv); @@ -375,7 +375,7 @@ static void gm45_get_cdclk(struct drm_i915_private *dev_priv, { struct pci_dev *pdev = dev_priv->drm.pdev; unsigned int cdclk_sel; - uint16_t tmp = 0; + u16 tmp = 0; cdclk_state->vco = intel_hpll_vco(dev_priv); @@ -403,8 +403,8 @@ static void gm45_get_cdclk(struct drm_i915_private *dev_priv, static void hsw_get_cdclk(struct drm_i915_private *dev_priv, struct intel_cdclk_state *cdclk_state) { - uint32_t lcpll = I915_READ(LCPLL_CTL); - uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK; + u32 lcpll = I915_READ(LCPLL_CTL); + u32 freq = lcpll & LCPLL_CLK_FREQ_MASK; if (lcpll & LCPLL_CD_SOURCE_FCLK) cdclk_state->cdclk = 800000; @@ -520,6 +520,7 @@ static void vlv_set_cdclk(struct drm_i915_private *dev_priv, { int cdclk = cdclk_state->cdclk; u32 val, cmd = cdclk_state->voltage_level; + intel_wakeref_t wakeref; switch (cdclk) { case 400000: @@ -539,7 +540,7 @@ static void vlv_set_cdclk(struct drm_i915_private *dev_priv, * a system suspend. So grab the PIPE-A domain, which covers * the HW blocks needed for the following programming. */ - intel_display_power_get(dev_priv, POWER_DOMAIN_PIPE_A); + wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_PIPE_A); mutex_lock(&dev_priv->pcu_lock); val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ); @@ -593,7 +594,7 @@ static void vlv_set_cdclk(struct drm_i915_private *dev_priv, vlv_program_pfi_credits(dev_priv); - intel_display_power_put(dev_priv, POWER_DOMAIN_PIPE_A); + intel_display_power_put(dev_priv, POWER_DOMAIN_PIPE_A, wakeref); } static void chv_set_cdclk(struct drm_i915_private *dev_priv, @@ -601,6 +602,7 @@ static void chv_set_cdclk(struct drm_i915_private *dev_priv, { int cdclk = cdclk_state->cdclk; u32 val, cmd = cdclk_state->voltage_level; + intel_wakeref_t wakeref; switch (cdclk) { case 333333: @@ -619,7 +621,7 @@ static void chv_set_cdclk(struct drm_i915_private *dev_priv, * a system suspend. So grab the PIPE-A domain, which covers * the HW blocks needed for the following programming. */ - intel_display_power_get(dev_priv, POWER_DOMAIN_PIPE_A); + wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_PIPE_A); mutex_lock(&dev_priv->pcu_lock); val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ); @@ -637,7 +639,7 @@ static void chv_set_cdclk(struct drm_i915_private *dev_priv, vlv_program_pfi_credits(dev_priv); - intel_display_power_put(dev_priv, POWER_DOMAIN_PIPE_A); + intel_display_power_put(dev_priv, POWER_DOMAIN_PIPE_A, wakeref); } static int bdw_calc_cdclk(int min_cdclk) @@ -670,8 +672,8 @@ static u8 bdw_calc_voltage_level(int cdclk) static void bdw_get_cdclk(struct drm_i915_private *dev_priv, struct intel_cdclk_state *cdclk_state) { - uint32_t lcpll = I915_READ(LCPLL_CTL); - uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK; + u32 lcpll = I915_READ(LCPLL_CTL); + u32 freq = lcpll & LCPLL_CLK_FREQ_MASK; if (lcpll & LCPLL_CD_SOURCE_FCLK) cdclk_state->cdclk = 800000; @@ -698,7 +700,7 @@ static void bdw_set_cdclk(struct drm_i915_private *dev_priv, const struct intel_cdclk_state *cdclk_state) { int cdclk = cdclk_state->cdclk; - uint32_t val; + u32 val; int ret; if (WARN((I915_READ(LCPLL_CTL) & @@ -1081,7 +1083,7 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv, static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv) { - uint32_t cdctl, expected; + u32 cdctl, expected; /* * check if the pre-os initialized the display @@ -2140,7 +2142,7 @@ static int intel_pixel_rate_to_cdclk(struct drm_i915_private *dev_priv, { if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) return DIV_ROUND_UP(pixel_rate, 2); - else if (IS_GEN9(dev_priv) || + else if (IS_GEN(dev_priv, 9) || IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) return pixel_rate; else if (IS_CHERRYVIEW(dev_priv)) @@ -2176,7 +2178,7 @@ int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state) if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv)) { /* Display WA #1145: glk,cnl */ min_cdclk = max(316800, min_cdclk); - } else if (IS_GEN9(dev_priv) || IS_BROADWELL(dev_priv)) { + } else if (IS_GEN(dev_priv, 9) || IS_BROADWELL(dev_priv)) { /* Display WA #1144: skl,bxt */ min_cdclk = max(432000, min_cdclk); } @@ -2537,7 +2539,7 @@ static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv) if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) return 2 * max_cdclk_freq; - else if (IS_GEN9(dev_priv) || + else if (IS_GEN(dev_priv, 9) || IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) return max_cdclk_freq; else if (IS_CHERRYVIEW(dev_priv)) @@ -2660,37 +2662,18 @@ static int cnp_rawclk(struct drm_i915_private *dev_priv) fraction = 200; } - rawclk = CNP_RAWCLK_DIV((divider / 1000) - 1); - if (fraction) - rawclk |= CNP_RAWCLK_FRAC(DIV_ROUND_CLOSEST(1000, - fraction) - 1); + rawclk = CNP_RAWCLK_DIV(divider / 1000); + if (fraction) { + int numerator = 1; - I915_WRITE(PCH_RAWCLK_FREQ, rawclk); - return divider + fraction; -} - -static int icp_rawclk(struct drm_i915_private *dev_priv) -{ - u32 rawclk; - int divider, numerator, denominator, frequency; - - if (I915_READ(SFUSE_STRAP) & SFUSE_STRAP_RAW_FREQUENCY) { - frequency = 24000; - divider = 23; - numerator = 0; - denominator = 0; - } else { - frequency = 19200; - divider = 18; - numerator = 1; - denominator = 4; + rawclk |= CNP_RAWCLK_DEN(DIV_ROUND_CLOSEST(numerator * 1000, + fraction) - 1); + if (HAS_PCH_ICP(dev_priv)) + rawclk |= ICP_RAWCLK_NUM(numerator); } - rawclk = CNP_RAWCLK_DIV(divider) | ICP_RAWCLK_NUM(numerator) | - ICP_RAWCLK_DEN(denominator); - I915_WRITE(PCH_RAWCLK_FREQ, rawclk); - return frequency; + return divider + fraction; } static int pch_rawclk(struct drm_i915_private *dev_priv) @@ -2707,7 +2690,7 @@ static int vlv_hrawclk(struct drm_i915_private *dev_priv) static int g4x_hrawclk(struct drm_i915_private *dev_priv) { - uint32_t clkcfg; + u32 clkcfg; /* hrawclock is 1/4 the FSB frequency */ clkcfg = I915_READ(CLKCFG); @@ -2740,9 +2723,7 @@ static int g4x_hrawclk(struct drm_i915_private *dev_priv) */ void intel_update_rawclk(struct drm_i915_private *dev_priv) { - if (HAS_PCH_ICP(dev_priv)) - dev_priv->rawclk_freq = icp_rawclk(dev_priv); - else if (HAS_PCH_CNP(dev_priv)) + if (HAS_PCH_CNP(dev_priv) || HAS_PCH_ICP(dev_priv)) dev_priv->rawclk_freq = cnp_rawclk(dev_priv); else if (HAS_PCH_SPLIT(dev_priv)) dev_priv->rawclk_freq = pch_rawclk(dev_priv); @@ -2806,9 +2787,9 @@ void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv) dev_priv->display.get_cdclk = hsw_get_cdclk; else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) dev_priv->display.get_cdclk = vlv_get_cdclk; - else if (IS_GEN6(dev_priv) || IS_IVYBRIDGE(dev_priv)) + else if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv)) dev_priv->display.get_cdclk = fixed_400mhz_get_cdclk; - else if (IS_GEN5(dev_priv)) + else if (IS_GEN(dev_priv, 5)) dev_priv->display.get_cdclk = fixed_450mhz_get_cdclk; else if (IS_GM45(dev_priv)) dev_priv->display.get_cdclk = gm45_get_cdclk; diff --git a/drivers/gpu/drm/i915/intel_color.c b/drivers/gpu/drm/i915/intel_color.c index c6a7beabd58d..71a1f12c6b2a 100644 --- a/drivers/gpu/drm/i915/intel_color.c +++ b/drivers/gpu/drm/i915/intel_color.c @@ -74,12 +74,17 @@ #define ILK_CSC_COEFF_1_0 \ ((7 << 12) | ILK_CSC_COEFF_FP(CTM_COEFF_1_0, 8)) -static bool crtc_state_is_legacy_gamma(struct drm_crtc_state *state) +static bool lut_is_legacy(const struct drm_property_blob *lut) { - return !state->degamma_lut && - !state->ctm && - state->gamma_lut && - drm_color_lut_size(state->gamma_lut) == LEGACY_LUT_LENGTH; + return drm_color_lut_size(lut) == LEGACY_LUT_LENGTH; +} + +static bool crtc_state_is_legacy_gamma(const struct intel_crtc_state *crtc_state) +{ + return !crtc_state->base.degamma_lut && + !crtc_state->base.ctm && + crtc_state->base.gamma_lut && + lut_is_legacy(crtc_state->base.gamma_lut); } /* @@ -108,10 +113,10 @@ static u64 *ctm_mult_by_limited(u64 *result, const u64 *input) return result; } -static void ilk_load_ycbcr_conversion_matrix(struct intel_crtc *intel_crtc) +static void ilk_load_ycbcr_conversion_matrix(struct intel_crtc *crtc) { - int pipe = intel_crtc->pipe; - struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; I915_WRITE(PIPE_CSC_PREOFF_HI(pipe), 0); I915_WRITE(PIPE_CSC_PREOFF_ME(pipe), 0); @@ -132,28 +137,28 @@ static void ilk_load_ycbcr_conversion_matrix(struct intel_crtc *intel_crtc) I915_WRITE(PIPE_CSC_MODE(pipe), 0); } -static void ilk_load_csc_matrix(struct drm_crtc_state *crtc_state) +static void ilk_load_csc_matrix(const struct intel_crtc_state *crtc_state) { - struct drm_crtc *crtc = crtc_state->crtc; - struct drm_i915_private *dev_priv = to_i915(crtc->dev); - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - int i, pipe = intel_crtc->pipe; - uint16_t coeffs[9] = { 0, }; - struct intel_crtc_state *intel_crtc_state = to_intel_crtc_state(crtc_state); + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); bool limited_color_range = false; + enum pipe pipe = crtc->pipe; + u16 coeffs[9] = {}; + int i; /* * FIXME if there's a gamma LUT after the CSC, we should * do the range compression using the gamma LUT instead. */ if (INTEL_GEN(dev_priv) >= 8 || IS_HASWELL(dev_priv)) - limited_color_range = intel_crtc_state->limited_color_range; + limited_color_range = crtc_state->limited_color_range; - if (intel_crtc_state->ycbcr420) { - ilk_load_ycbcr_conversion_matrix(intel_crtc); + if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 || + crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444) { + ilk_load_ycbcr_conversion_matrix(crtc); return; - } else if (crtc_state->ctm) { - struct drm_color_ctm *ctm = crtc_state->ctm->data; + } else if (crtc_state->base.ctm) { + struct drm_color_ctm *ctm = crtc_state->base.ctm->data; const u64 *input; u64 temp[9]; @@ -167,7 +172,7 @@ static void ilk_load_csc_matrix(struct drm_crtc_state *crtc_state) * hardware. */ for (i = 0; i < ARRAY_SIZE(coeffs); i++) { - uint64_t abs_coeff = ((1ULL << 63) - 1) & input[i]; + u64 abs_coeff = ((1ULL << 63) - 1) & input[i]; /* * Clamp input value to min/max supported by @@ -229,7 +234,7 @@ static void ilk_load_csc_matrix(struct drm_crtc_state *crtc_state) I915_WRITE(PIPE_CSC_PREOFF_LO(pipe), 0); if (INTEL_GEN(dev_priv) > 6) { - uint16_t postoff = 0; + u16 postoff = 0; if (limited_color_range) postoff = (16 * (1 << 12) / 255) & 0x1fff; @@ -240,7 +245,7 @@ static void ilk_load_csc_matrix(struct drm_crtc_state *crtc_state) I915_WRITE(PIPE_CSC_MODE(pipe), 0); } else { - uint32_t mode = CSC_MODE_YUV_TO_RGB; + u32 mode = CSC_MODE_YUV_TO_RGB; if (limited_color_range) mode |= CSC_BLACK_SCREEN_OFFSET; @@ -252,21 +257,20 @@ static void ilk_load_csc_matrix(struct drm_crtc_state *crtc_state) /* * Set up the pipe CSC unit on CherryView. */ -static void cherryview_load_csc_matrix(struct drm_crtc_state *state) +static void cherryview_load_csc_matrix(const struct intel_crtc_state *crtc_state) { - struct drm_crtc *crtc = state->crtc; - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); - int pipe = to_intel_crtc(crtc)->pipe; - uint32_t mode; - - if (state->ctm) { - struct drm_color_ctm *ctm = state->ctm->data; - uint16_t coeffs[9] = { 0, }; + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; + u32 mode; + + if (crtc_state->base.ctm) { + const struct drm_color_ctm *ctm = crtc_state->base.ctm->data; + u16 coeffs[9] = {}; int i; for (i = 0; i < ARRAY_SIZE(coeffs); i++) { - uint64_t abs_coeff = + u64 abs_coeff = ((1ULL << 63) - 1) & ctm->matrix[i]; /* Round coefficient. */ @@ -292,35 +296,24 @@ static void cherryview_load_csc_matrix(struct drm_crtc_state *state) I915_WRITE(CGM_PIPE_CSC_COEFF8(pipe), coeffs[8]); } - mode = (state->ctm ? CGM_PIPE_MODE_CSC : 0); - if (!crtc_state_is_legacy_gamma(state)) { - mode |= (state->degamma_lut ? CGM_PIPE_MODE_DEGAMMA : 0) | - (state->gamma_lut ? CGM_PIPE_MODE_GAMMA : 0); + mode = (crtc_state->base.ctm ? CGM_PIPE_MODE_CSC : 0); + if (!crtc_state_is_legacy_gamma(crtc_state)) { + mode |= (crtc_state->base.degamma_lut ? CGM_PIPE_MODE_DEGAMMA : 0) | + (crtc_state->base.gamma_lut ? CGM_PIPE_MODE_GAMMA : 0); } I915_WRITE(CGM_PIPE_MODE(pipe), mode); } -void intel_color_set_csc(struct drm_crtc_state *crtc_state) -{ - struct drm_device *dev = crtc_state->crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); - - if (dev_priv->display.load_csc_matrix) - dev_priv->display.load_csc_matrix(crtc_state); -} - /* Loads the legacy palette/gamma unit for the CRTC. */ -static void i9xx_load_luts_internal(struct drm_crtc *crtc, - struct drm_property_blob *blob, - struct intel_crtc_state *crtc_state) +static void i9xx_load_luts_internal(const struct intel_crtc_state *crtc_state, + const struct drm_property_blob *blob) { - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - enum pipe pipe = intel_crtc->pipe; + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; int i; - if (HAS_GMCH_DISPLAY(dev_priv)) { + if (HAS_GMCH(dev_priv)) { if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI)) assert_dsi_pll_enabled(dev_priv); else @@ -328,23 +321,24 @@ static void i9xx_load_luts_internal(struct drm_crtc *crtc, } if (blob) { - struct drm_color_lut *lut = blob->data; + const struct drm_color_lut *lut = blob->data; + for (i = 0; i < 256; i++) { - uint32_t word = + u32 word = (drm_color_lut_extract(lut[i].red, 8) << 16) | (drm_color_lut_extract(lut[i].green, 8) << 8) | drm_color_lut_extract(lut[i].blue, 8); - if (HAS_GMCH_DISPLAY(dev_priv)) + if (HAS_GMCH(dev_priv)) I915_WRITE(PALETTE(pipe, i), word); else I915_WRITE(LGC_PALETTE(pipe, i), word); } } else { for (i = 0; i < 256; i++) { - uint32_t word = (i << 16) | (i << 8) | i; + u32 word = (i << 16) | (i << 8) | i; - if (HAS_GMCH_DISPLAY(dev_priv)) + if (HAS_GMCH(dev_priv)) I915_WRITE(PALETTE(pipe, i), word); else I915_WRITE(LGC_PALETTE(pipe, i), word); @@ -352,56 +346,37 @@ static void i9xx_load_luts_internal(struct drm_crtc *crtc, } } -static void i9xx_load_luts(struct drm_crtc_state *crtc_state) +static void i9xx_load_luts(const struct intel_crtc_state *crtc_state) { - i9xx_load_luts_internal(crtc_state->crtc, crtc_state->gamma_lut, - to_intel_crtc_state(crtc_state)); + i9xx_load_luts_internal(crtc_state, crtc_state->base.gamma_lut); } -/* Loads the legacy palette/gamma unit for the CRTC on Haswell. */ -static void haswell_load_luts(struct drm_crtc_state *crtc_state) +static void hsw_color_commit(const struct intel_crtc_state *crtc_state) { - struct drm_crtc *crtc = crtc_state->crtc; - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct intel_crtc_state *intel_crtc_state = - to_intel_crtc_state(crtc_state); - bool reenable_ips = false; + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - /* - * Workaround : Do not read or write the pipe palette/gamma data while - * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled. - */ - if (IS_HASWELL(dev_priv) && intel_crtc_state->ips_enabled && - (intel_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)) { - hsw_disable_ips(intel_crtc_state); - reenable_ips = true; - } + I915_WRITE(GAMMA_MODE(crtc->pipe), crtc_state->gamma_mode); - intel_crtc_state->gamma_mode = GAMMA_MODE_MODE_8BIT; - I915_WRITE(GAMMA_MODE(intel_crtc->pipe), GAMMA_MODE_MODE_8BIT); - - i9xx_load_luts(crtc_state); - - if (reenable_ips) - hsw_enable_ips(intel_crtc_state); + ilk_load_csc_matrix(crtc_state); } -static void bdw_load_degamma_lut(struct drm_crtc_state *state) +static void bdw_load_degamma_lut(const struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = to_i915(state->crtc->dev); - enum pipe pipe = to_intel_crtc(state->crtc)->pipe; - uint32_t i, lut_size = INTEL_INFO(dev_priv)->color.degamma_lut_size; + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + const struct drm_property_blob *degamma_lut = crtc_state->base.degamma_lut; + u32 i, lut_size = INTEL_INFO(dev_priv)->color.degamma_lut_size; + enum pipe pipe = crtc->pipe; I915_WRITE(PREC_PAL_INDEX(pipe), PAL_PREC_SPLIT_MODE | PAL_PREC_AUTO_INCREMENT); - if (state->degamma_lut) { - struct drm_color_lut *lut = state->degamma_lut->data; + if (degamma_lut) { + const struct drm_color_lut *lut = degamma_lut->data; for (i = 0; i < lut_size; i++) { - uint32_t word = + u32 word = drm_color_lut_extract(lut[i].red, 10) << 20 | drm_color_lut_extract(lut[i].green, 10) << 10 | drm_color_lut_extract(lut[i].blue, 10); @@ -410,7 +385,7 @@ static void bdw_load_degamma_lut(struct drm_crtc_state *state) } } else { for (i = 0; i < lut_size; i++) { - uint32_t v = (i * ((1 << 10) - 1)) / (lut_size - 1); + u32 v = (i * ((1 << 10) - 1)) / (lut_size - 1); I915_WRITE(PREC_PAL_DATA(pipe), (v << 20) | (v << 10) | v); @@ -418,11 +393,13 @@ static void bdw_load_degamma_lut(struct drm_crtc_state *state) } } -static void bdw_load_gamma_lut(struct drm_crtc_state *state, u32 offset) +static void bdw_load_gamma_lut(const struct intel_crtc_state *crtc_state, u32 offset) { - struct drm_i915_private *dev_priv = to_i915(state->crtc->dev); - enum pipe pipe = to_intel_crtc(state->crtc)->pipe; - uint32_t i, lut_size = INTEL_INFO(dev_priv)->color.gamma_lut_size; + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + const struct drm_property_blob *gamma_lut = crtc_state->base.gamma_lut; + u32 i, lut_size = INTEL_INFO(dev_priv)->color.gamma_lut_size; + enum pipe pipe = crtc->pipe; WARN_ON(offset & ~PAL_PREC_INDEX_VALUE_MASK); @@ -431,11 +408,11 @@ static void bdw_load_gamma_lut(struct drm_crtc_state *state, u32 offset) PAL_PREC_AUTO_INCREMENT | offset); - if (state->gamma_lut) { - struct drm_color_lut *lut = state->gamma_lut->data; + if (gamma_lut) { + const struct drm_color_lut *lut = gamma_lut->data; for (i = 0; i < lut_size; i++) { - uint32_t word = + u32 word = (drm_color_lut_extract(lut[i].red, 10) << 20) | (drm_color_lut_extract(lut[i].green, 10) << 10) | drm_color_lut_extract(lut[i].blue, 10); @@ -453,7 +430,7 @@ static void bdw_load_gamma_lut(struct drm_crtc_state *state, u32 offset) drm_color_lut_extract(lut[i].blue, 16)); } else { for (i = 0; i < lut_size; i++) { - uint32_t v = (i * ((1 << 10) - 1)) / (lut_size - 1); + u32 v = (i * ((1 << 10) - 1)) / (lut_size - 1); I915_WRITE(PREC_PAL_DATA(pipe), (v << 20) | (v << 10) | v); @@ -466,38 +443,34 @@ static void bdw_load_gamma_lut(struct drm_crtc_state *state, u32 offset) } /* Loads the palette/gamma unit for the CRTC on Broadwell+. */ -static void broadwell_load_luts(struct drm_crtc_state *state) +static void broadwell_load_luts(const struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = to_i915(state->crtc->dev); - struct intel_crtc_state *intel_state = to_intel_crtc_state(state); - enum pipe pipe = to_intel_crtc(state->crtc)->pipe; - - if (crtc_state_is_legacy_gamma(state)) { - haswell_load_luts(state); - return; - } + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; - bdw_load_degamma_lut(state); - bdw_load_gamma_lut(state, - INTEL_INFO(dev_priv)->color.degamma_lut_size); - - intel_state->gamma_mode = GAMMA_MODE_MODE_SPLIT; - I915_WRITE(GAMMA_MODE(pipe), GAMMA_MODE_MODE_SPLIT); - POSTING_READ(GAMMA_MODE(pipe)); + if (crtc_state_is_legacy_gamma(crtc_state)) { + i9xx_load_luts(crtc_state); + } else { + bdw_load_degamma_lut(crtc_state); + bdw_load_gamma_lut(crtc_state, + INTEL_INFO(dev_priv)->color.degamma_lut_size); - /* - * Reset the index, otherwise it prevents the legacy palette to be - * written properly. - */ - I915_WRITE(PREC_PAL_INDEX(pipe), 0); + /* + * Reset the index, otherwise it prevents the legacy palette to be + * written properly. + */ + I915_WRITE(PREC_PAL_INDEX(pipe), 0); + } } -static void glk_load_degamma_lut(struct drm_crtc_state *state) +static void glk_load_degamma_lut(const struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = to_i915(state->crtc->dev); - enum pipe pipe = to_intel_crtc(state->crtc)->pipe; - const uint32_t lut_size = 33; - uint32_t i; + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; + const u32 lut_size = 33; + u32 i; /* * When setting the auto-increment bit, the hardware seems to @@ -512,7 +485,7 @@ static void glk_load_degamma_lut(struct drm_crtc_state *state) * different values per channel, so this just loads a linear table. */ for (i = 0; i < lut_size; i++) { - uint32_t v = (i * (1 << 16)) / (lut_size - 1); + u32 v = (i * (1 << 16)) / (lut_size - 1); I915_WRITE(PRE_CSC_GAMC_DATA(pipe), v); } @@ -522,51 +495,49 @@ static void glk_load_degamma_lut(struct drm_crtc_state *state) I915_WRITE(PRE_CSC_GAMC_DATA(pipe), (1 << 16)); } -static void glk_load_luts(struct drm_crtc_state *state) +static void glk_load_luts(const struct intel_crtc_state *crtc_state) { - struct drm_crtc *crtc = state->crtc; - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_crtc_state *intel_state = to_intel_crtc_state(state); - enum pipe pipe = to_intel_crtc(crtc)->pipe; - - glk_load_degamma_lut(state); + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; - if (crtc_state_is_legacy_gamma(state)) { - haswell_load_luts(state); - return; - } + glk_load_degamma_lut(crtc_state); - bdw_load_gamma_lut(state, 0); + if (crtc_state_is_legacy_gamma(crtc_state)) { + i9xx_load_luts(crtc_state); + } else { + bdw_load_gamma_lut(crtc_state, 0); - intel_state->gamma_mode = GAMMA_MODE_MODE_10BIT; - I915_WRITE(GAMMA_MODE(pipe), GAMMA_MODE_MODE_10BIT); - POSTING_READ(GAMMA_MODE(pipe)); + /* + * Reset the index, otherwise it prevents the legacy palette to be + * written properly. + */ + I915_WRITE(PREC_PAL_INDEX(pipe), 0); + } } -/* Loads the palette/gamma unit for the CRTC on CherryView. */ -static void cherryview_load_luts(struct drm_crtc_state *state) +static void cherryview_load_luts(const struct intel_crtc_state *crtc_state) { - struct drm_crtc *crtc = state->crtc; - struct drm_i915_private *dev_priv = to_i915(crtc->dev); - enum pipe pipe = to_intel_crtc(crtc)->pipe; - struct drm_color_lut *lut; - uint32_t i, lut_size; - uint32_t word0, word1; - - if (crtc_state_is_legacy_gamma(state)) { - /* Turn off degamma/gamma on CGM block. */ - I915_WRITE(CGM_PIPE_MODE(pipe), - (state->ctm ? CGM_PIPE_MODE_CSC : 0)); - i9xx_load_luts_internal(crtc, state->gamma_lut, - to_intel_crtc_state(state)); + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + const struct drm_property_blob *gamma_lut = crtc_state->base.gamma_lut; + const struct drm_property_blob *degamma_lut = crtc_state->base.degamma_lut; + enum pipe pipe = crtc->pipe; + + cherryview_load_csc_matrix(crtc_state); + + if (crtc_state_is_legacy_gamma(crtc_state)) { + i9xx_load_luts_internal(crtc_state, gamma_lut); return; } - if (state->degamma_lut) { - lut = state->degamma_lut->data; - lut_size = INTEL_INFO(dev_priv)->color.degamma_lut_size; + if (degamma_lut) { + const struct drm_color_lut *lut = degamma_lut->data; + int i, lut_size = INTEL_INFO(dev_priv)->color.degamma_lut_size; + for (i = 0; i < lut_size; i++) { + u32 word0, word1; + /* Write LUT in U0.14 format. */ word0 = (drm_color_lut_extract(lut[i].green, 14) << 16) | @@ -578,10 +549,13 @@ static void cherryview_load_luts(struct drm_crtc_state *state) } } - if (state->gamma_lut) { - lut = state->gamma_lut->data; - lut_size = INTEL_INFO(dev_priv)->color.gamma_lut_size; + if (gamma_lut) { + const struct drm_color_lut *lut = gamma_lut->data; + int i, lut_size = INTEL_INFO(dev_priv)->color.gamma_lut_size; + for (i = 0; i < lut_size; i++) { + u32 word0, word1; + /* Write LUT in U0.10 format. */ word0 = (drm_color_lut_extract(lut[i].green, 10) << 16) | @@ -593,74 +567,100 @@ static void cherryview_load_luts(struct drm_crtc_state *state) } } - I915_WRITE(CGM_PIPE_MODE(pipe), - (state->ctm ? CGM_PIPE_MODE_CSC : 0) | - (state->degamma_lut ? CGM_PIPE_MODE_DEGAMMA : 0) | - (state->gamma_lut ? CGM_PIPE_MODE_GAMMA : 0)); - /* * Also program a linear LUT in the legacy block (behind the * CGM block). */ - i9xx_load_luts_internal(crtc, NULL, to_intel_crtc_state(state)); + i9xx_load_luts_internal(crtc_state, NULL); } -void intel_color_load_luts(struct drm_crtc_state *crtc_state) +void intel_color_load_luts(const struct intel_crtc_state *crtc_state) { - struct drm_device *dev = crtc_state->crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); dev_priv->display.load_luts(crtc_state); } -int intel_color_check(struct drm_crtc *crtc, - struct drm_crtc_state *crtc_state) +void intel_color_commit(const struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = to_i915(crtc->dev); - size_t gamma_length, degamma_length; + struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); + + if (dev_priv->display.color_commit) + dev_priv->display.color_commit(crtc_state); +} + +static int check_lut_size(const struct drm_property_blob *lut, int expected) +{ + int len; + + if (!lut) + return 0; + + len = drm_color_lut_size(lut); + if (len != expected) { + DRM_DEBUG_KMS("Invalid LUT size; got %d, expected %d\n", + len, expected); + return -EINVAL; + } + + return 0; +} + +int intel_color_check(struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); + const struct drm_property_blob *gamma_lut = crtc_state->base.gamma_lut; + const struct drm_property_blob *degamma_lut = crtc_state->base.degamma_lut; + int gamma_length, degamma_length; + u32 gamma_tests, degamma_tests; degamma_length = INTEL_INFO(dev_priv)->color.degamma_lut_size; gamma_length = INTEL_INFO(dev_priv)->color.gamma_lut_size; + degamma_tests = INTEL_INFO(dev_priv)->color.degamma_lut_tests; + gamma_tests = INTEL_INFO(dev_priv)->color.gamma_lut_tests; - /* - * We allow both degamma & gamma luts at the right size or - * NULL. - */ - if ((!crtc_state->degamma_lut || - drm_color_lut_size(crtc_state->degamma_lut) == degamma_length) && - (!crtc_state->gamma_lut || - drm_color_lut_size(crtc_state->gamma_lut) == gamma_length)) + /* Always allow legacy gamma LUT with no further checking. */ + if (crtc_state_is_legacy_gamma(crtc_state)) { + crtc_state->gamma_mode = GAMMA_MODE_MODE_8BIT; return 0; + } - /* - * We also allow no degamma lut/ctm and a gamma lut at the legacy - * size (256 entries). - */ - if (crtc_state_is_legacy_gamma(crtc_state)) - return 0; + if (check_lut_size(degamma_lut, degamma_length) || + check_lut_size(gamma_lut, gamma_length)) + return -EINVAL; - return -EINVAL; + if (drm_color_lut_check(degamma_lut, degamma_tests) || + drm_color_lut_check(gamma_lut, gamma_tests)) + return -EINVAL; + + if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) + crtc_state->gamma_mode = GAMMA_MODE_MODE_10BIT; + else if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv)) + crtc_state->gamma_mode = GAMMA_MODE_MODE_SPLIT; + else + crtc_state->gamma_mode = GAMMA_MODE_MODE_8BIT; + + return 0; } -void intel_color_init(struct drm_crtc *crtc) +void intel_color_init(struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = to_i915(crtc->dev); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - drm_mode_crtc_set_gamma_size(crtc, 256); + drm_mode_crtc_set_gamma_size(&crtc->base, 256); if (IS_CHERRYVIEW(dev_priv)) { - dev_priv->display.load_csc_matrix = cherryview_load_csc_matrix; dev_priv->display.load_luts = cherryview_load_luts; } else if (IS_HASWELL(dev_priv)) { - dev_priv->display.load_csc_matrix = ilk_load_csc_matrix; - dev_priv->display.load_luts = haswell_load_luts; + dev_priv->display.load_luts = i9xx_load_luts; + dev_priv->display.color_commit = hsw_color_commit; } else if (IS_BROADWELL(dev_priv) || IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv)) { - dev_priv->display.load_csc_matrix = ilk_load_csc_matrix; dev_priv->display.load_luts = broadwell_load_luts; + dev_priv->display.color_commit = hsw_color_commit; } else if (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) { - dev_priv->display.load_csc_matrix = ilk_load_csc_matrix; dev_priv->display.load_luts = glk_load_luts; + dev_priv->display.color_commit = hsw_color_commit; } else { dev_priv->display.load_luts = i9xx_load_luts; } @@ -668,7 +668,7 @@ void intel_color_init(struct drm_crtc *crtc) /* Enable color management support when we have degamma & gamma LUTs. */ if (INTEL_INFO(dev_priv)->color.degamma_lut_size != 0 && INTEL_INFO(dev_priv)->color.gamma_lut_size != 0) - drm_crtc_enable_color_mgmt(crtc, + drm_crtc_enable_color_mgmt(&crtc->base, INTEL_INFO(dev_priv)->color.degamma_lut_size, true, INTEL_INFO(dev_priv)->color.gamma_lut_size); diff --git a/drivers/gpu/drm/i915/intel_combo_phy.c b/drivers/gpu/drm/i915/intel_combo_phy.c new file mode 100644 index 000000000000..3d0271cebf99 --- /dev/null +++ b/drivers/gpu/drm/i915/intel_combo_phy.c @@ -0,0 +1,254 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2018 Intel Corporation + */ + +#include "intel_drv.h" + +#define for_each_combo_port(__dev_priv, __port) \ + for ((__port) = PORT_A; (__port) < I915_MAX_PORTS; (__port)++) \ + for_each_if(intel_port_is_combophy(__dev_priv, __port)) + +#define for_each_combo_port_reverse(__dev_priv, __port) \ + for ((__port) = I915_MAX_PORTS; (__port)-- > PORT_A;) \ + for_each_if(intel_port_is_combophy(__dev_priv, __port)) + +enum { + PROCMON_0_85V_DOT_0, + PROCMON_0_95V_DOT_0, + PROCMON_0_95V_DOT_1, + PROCMON_1_05V_DOT_0, + PROCMON_1_05V_DOT_1, +}; + +static const struct cnl_procmon { + u32 dw1, dw9, dw10; +} cnl_procmon_values[] = { + [PROCMON_0_85V_DOT_0] = + { .dw1 = 0x00000000, .dw9 = 0x62AB67BB, .dw10 = 0x51914F96, }, + [PROCMON_0_95V_DOT_0] = + { .dw1 = 0x00000000, .dw9 = 0x86E172C7, .dw10 = 0x77CA5EAB, }, + [PROCMON_0_95V_DOT_1] = + { .dw1 = 0x00000000, .dw9 = 0x93F87FE1, .dw10 = 0x8AE871C5, }, + [PROCMON_1_05V_DOT_0] = + { .dw1 = 0x00000000, .dw9 = 0x98FA82DD, .dw10 = 0x89E46DC1, }, + [PROCMON_1_05V_DOT_1] = + { .dw1 = 0x00440000, .dw9 = 0x9A00AB25, .dw10 = 0x8AE38FF1, }, +}; + +/* + * CNL has just one set of registers, while ICL has two sets: one for port A and + * the other for port B. The CNL registers are equivalent to the ICL port A + * registers, that's why we call the ICL macros even though the function has CNL + * on its name. + */ +static const struct cnl_procmon * +cnl_get_procmon_ref_values(struct drm_i915_private *dev_priv, enum port port) +{ + const struct cnl_procmon *procmon; + u32 val; + + val = I915_READ(ICL_PORT_COMP_DW3(port)); + switch (val & (PROCESS_INFO_MASK | VOLTAGE_INFO_MASK)) { + default: + MISSING_CASE(val); + /* fall through */ + case VOLTAGE_INFO_0_85V | PROCESS_INFO_DOT_0: + procmon = &cnl_procmon_values[PROCMON_0_85V_DOT_0]; + break; + case VOLTAGE_INFO_0_95V | PROCESS_INFO_DOT_0: + procmon = &cnl_procmon_values[PROCMON_0_95V_DOT_0]; + break; + case VOLTAGE_INFO_0_95V | PROCESS_INFO_DOT_1: + procmon = &cnl_procmon_values[PROCMON_0_95V_DOT_1]; + break; + case VOLTAGE_INFO_1_05V | PROCESS_INFO_DOT_0: + procmon = &cnl_procmon_values[PROCMON_1_05V_DOT_0]; + break; + case VOLTAGE_INFO_1_05V | PROCESS_INFO_DOT_1: + procmon = &cnl_procmon_values[PROCMON_1_05V_DOT_1]; + break; + } + + return procmon; +} + +static void cnl_set_procmon_ref_values(struct drm_i915_private *dev_priv, + enum port port) +{ + const struct cnl_procmon *procmon; + u32 val; + + procmon = cnl_get_procmon_ref_values(dev_priv, port); + + val = I915_READ(ICL_PORT_COMP_DW1(port)); + val &= ~((0xff << 16) | 0xff); + val |= procmon->dw1; + I915_WRITE(ICL_PORT_COMP_DW1(port), val); + + I915_WRITE(ICL_PORT_COMP_DW9(port), procmon->dw9); + I915_WRITE(ICL_PORT_COMP_DW10(port), procmon->dw10); +} + +static bool check_phy_reg(struct drm_i915_private *dev_priv, + enum port port, i915_reg_t reg, u32 mask, + u32 expected_val) +{ + u32 val = I915_READ(reg); + + if ((val & mask) != expected_val) { + DRM_DEBUG_DRIVER("Port %c combo PHY reg %08x state mismatch: " + "current %08x mask %08x expected %08x\n", + port_name(port), + reg.reg, val, mask, expected_val); + return false; + } + + return true; +} + +static bool cnl_verify_procmon_ref_values(struct drm_i915_private *dev_priv, + enum port port) +{ + const struct cnl_procmon *procmon; + bool ret; + + procmon = cnl_get_procmon_ref_values(dev_priv, port); + + ret = check_phy_reg(dev_priv, port, ICL_PORT_COMP_DW1(port), + (0xff << 16) | 0xff, procmon->dw1); + ret &= check_phy_reg(dev_priv, port, ICL_PORT_COMP_DW9(port), + -1U, procmon->dw9); + ret &= check_phy_reg(dev_priv, port, ICL_PORT_COMP_DW10(port), + -1U, procmon->dw10); + + return ret; +} + +static bool cnl_combo_phy_enabled(struct drm_i915_private *dev_priv) +{ + return !(I915_READ(CHICKEN_MISC_2) & CNL_COMP_PWR_DOWN) && + (I915_READ(CNL_PORT_COMP_DW0) & COMP_INIT); +} + +static bool cnl_combo_phy_verify_state(struct drm_i915_private *dev_priv) +{ + enum port port = PORT_A; + bool ret; + + if (!cnl_combo_phy_enabled(dev_priv)) + return false; + + ret = cnl_verify_procmon_ref_values(dev_priv, port); + + ret &= check_phy_reg(dev_priv, port, CNL_PORT_CL1CM_DW5, + CL_POWER_DOWN_ENABLE, CL_POWER_DOWN_ENABLE); + + return ret; +} + +void cnl_combo_phys_init(struct drm_i915_private *dev_priv) +{ + u32 val; + + val = I915_READ(CHICKEN_MISC_2); + val &= ~CNL_COMP_PWR_DOWN; + I915_WRITE(CHICKEN_MISC_2, val); + + /* Dummy PORT_A to get the correct CNL register from the ICL macro */ + cnl_set_procmon_ref_values(dev_priv, PORT_A); + + val = I915_READ(CNL_PORT_COMP_DW0); + val |= COMP_INIT; + I915_WRITE(CNL_PORT_COMP_DW0, val); + + val = I915_READ(CNL_PORT_CL1CM_DW5); + val |= CL_POWER_DOWN_ENABLE; + I915_WRITE(CNL_PORT_CL1CM_DW5, val); +} + +void cnl_combo_phys_uninit(struct drm_i915_private *dev_priv) +{ + u32 val; + + if (!cnl_combo_phy_verify_state(dev_priv)) + DRM_WARN("Combo PHY HW state changed unexpectedly.\n"); + + val = I915_READ(CHICKEN_MISC_2); + val |= CNL_COMP_PWR_DOWN; + I915_WRITE(CHICKEN_MISC_2, val); +} + +static bool icl_combo_phy_enabled(struct drm_i915_private *dev_priv, + enum port port) +{ + return !(I915_READ(ICL_PHY_MISC(port)) & + ICL_PHY_MISC_DE_IO_COMP_PWR_DOWN) && + (I915_READ(ICL_PORT_COMP_DW0(port)) & COMP_INIT); +} + +static bool icl_combo_phy_verify_state(struct drm_i915_private *dev_priv, + enum port port) +{ + bool ret; + + if (!icl_combo_phy_enabled(dev_priv, port)) + return false; + + ret = cnl_verify_procmon_ref_values(dev_priv, port); + + ret &= check_phy_reg(dev_priv, port, ICL_PORT_CL_DW5(port), + CL_POWER_DOWN_ENABLE, CL_POWER_DOWN_ENABLE); + + return ret; +} + +void icl_combo_phys_init(struct drm_i915_private *dev_priv) +{ + enum port port; + + for_each_combo_port(dev_priv, port) { + u32 val; + + if (icl_combo_phy_verify_state(dev_priv, port)) { + DRM_DEBUG_DRIVER("Port %c combo PHY already enabled, won't reprogram it.\n", + port_name(port)); + continue; + } + + val = I915_READ(ICL_PHY_MISC(port)); + val &= ~ICL_PHY_MISC_DE_IO_COMP_PWR_DOWN; + I915_WRITE(ICL_PHY_MISC(port), val); + + cnl_set_procmon_ref_values(dev_priv, port); + + val = I915_READ(ICL_PORT_COMP_DW0(port)); + val |= COMP_INIT; + I915_WRITE(ICL_PORT_COMP_DW0(port), val); + + val = I915_READ(ICL_PORT_CL_DW5(port)); + val |= CL_POWER_DOWN_ENABLE; + I915_WRITE(ICL_PORT_CL_DW5(port), val); + } +} + +void icl_combo_phys_uninit(struct drm_i915_private *dev_priv) +{ + enum port port; + + for_each_combo_port_reverse(dev_priv, port) { + u32 val; + + if (!icl_combo_phy_verify_state(dev_priv, port)) + DRM_WARN("Port %c combo PHY HW state changed unexpectedly\n", + port_name(port)); + + val = I915_READ(ICL_PHY_MISC(port)); + val |= ICL_PHY_MISC_DE_IO_COMP_PWR_DOWN; + I915_WRITE(ICL_PHY_MISC(port), val); + + val = I915_READ(ICL_PORT_COMP_DW0(port)); + val &= ~COMP_INIT; + I915_WRITE(ICL_PORT_COMP_DW0(port), val); + } +} diff --git a/drivers/gpu/drm/i915/intel_modes.c b/drivers/gpu/drm/i915/intel_connector.c index ca44bf368e24..ee16758747c5 100644 --- a/drivers/gpu/drm/i915/intel_modes.c +++ b/drivers/gpu/drm/i915/intel_connector.c @@ -25,11 +25,143 @@ #include <linux/slab.h> #include <linux/i2c.h> +#include <drm/drm_atomic_helper.h> #include <drm/drm_edid.h> -#include <drm/drmP.h> #include "intel_drv.h" #include "i915_drv.h" +int intel_connector_init(struct intel_connector *connector) +{ + struct intel_digital_connector_state *conn_state; + + /* + * Allocate enough memory to hold intel_digital_connector_state, + * This might be a few bytes too many, but for connectors that don't + * need it we'll free the state and allocate a smaller one on the first + * successful commit anyway. + */ + conn_state = kzalloc(sizeof(*conn_state), GFP_KERNEL); + if (!conn_state) + return -ENOMEM; + + __drm_atomic_helper_connector_reset(&connector->base, + &conn_state->base); + + return 0; +} + +struct intel_connector *intel_connector_alloc(void) +{ + struct intel_connector *connector; + + connector = kzalloc(sizeof(*connector), GFP_KERNEL); + if (!connector) + return NULL; + + if (intel_connector_init(connector) < 0) { + kfree(connector); + return NULL; + } + + return connector; +} + +/* + * Free the bits allocated by intel_connector_alloc. + * This should only be used after intel_connector_alloc has returned + * successfully, and before drm_connector_init returns successfully. + * Otherwise the destroy callbacks for the connector and the state should + * take care of proper cleanup/free (see intel_connector_destroy). + */ +void intel_connector_free(struct intel_connector *connector) +{ + kfree(to_intel_digital_connector_state(connector->base.state)); + kfree(connector); +} + +/* + * Connector type independent destroy hook for drm_connector_funcs. + */ +void intel_connector_destroy(struct drm_connector *connector) +{ + struct intel_connector *intel_connector = to_intel_connector(connector); + + kfree(intel_connector->detect_edid); + + if (!IS_ERR_OR_NULL(intel_connector->edid)) + kfree(intel_connector->edid); + + intel_panel_fini(&intel_connector->panel); + + drm_connector_cleanup(connector); + + if (intel_connector->port) + drm_dp_mst_put_port_malloc(intel_connector->port); + + kfree(connector); +} + +int intel_connector_register(struct drm_connector *connector) +{ + struct intel_connector *intel_connector = to_intel_connector(connector); + int ret; + + ret = intel_backlight_device_register(intel_connector); + if (ret) + goto err; + + if (i915_inject_load_failure()) { + ret = -EFAULT; + goto err_backlight; + } + + return 0; + +err_backlight: + intel_backlight_device_unregister(intel_connector); +err: + return ret; +} + +void intel_connector_unregister(struct drm_connector *connector) +{ + struct intel_connector *intel_connector = to_intel_connector(connector); + + intel_backlight_device_unregister(intel_connector); +} + +void intel_connector_attach_encoder(struct intel_connector *connector, + struct intel_encoder *encoder) +{ + connector->encoder = encoder; + drm_connector_attach_encoder(&connector->base, &encoder->base); +} + +/* + * Simple connector->get_hw_state implementation for encoders that support only + * one connector and no cloning and hence the encoder state determines the state + * of the connector. + */ +bool intel_connector_get_hw_state(struct intel_connector *connector) +{ + enum pipe pipe = 0; + struct intel_encoder *encoder = connector->encoder; + + return encoder->get_hw_state(encoder, &pipe); +} + +enum pipe intel_connector_get_pipe(struct intel_connector *connector) +{ + struct drm_device *dev = connector->base.dev; + + WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex)); + + if (!connector->base.state->crtc) + return INVALID_PIPE; + + return to_intel_crtc(connector->base.state->crtc)->pipe; +} + /** * intel_connector_update_modes - update connector from edid * @connector: DRM connector device to use diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c index 0c6bf82bb059..3716b2ee362f 100644 --- a/drivers/gpu/drm/i915/intel_crt.c +++ b/drivers/gpu/drm/i915/intel_crt.c @@ -27,11 +27,10 @@ #include <linux/dmi.h> #include <linux/i2c.h> #include <linux/slab.h> -#include <drm/drmP.h> #include <drm/drm_atomic_helper.h> #include <drm/drm_crtc.h> -#include <drm/drm_crtc_helper.h> #include <drm/drm_edid.h> +#include <drm/drm_probe_helper.h> #include "intel_drv.h" #include <drm/i915_drm.h> #include "i915_drv.h" @@ -84,15 +83,17 @@ static bool intel_crt_get_hw_state(struct intel_encoder *encoder, { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_crt *crt = intel_encoder_to_crt(encoder); + intel_wakeref_t wakeref; bool ret; - if (!intel_display_power_get_if_enabled(dev_priv, - encoder->power_domain)) + wakeref = intel_display_power_get_if_enabled(dev_priv, + encoder->power_domain); + if (!wakeref) return false; ret = intel_crt_port_enabled(dev_priv, crt->adpa_reg, pipe); - intel_display_power_put(dev_priv, encoder->power_domain); + intel_display_power_put(dev_priv, encoder->power_domain, wakeref); return ret; } @@ -322,7 +323,7 @@ intel_crt_mode_valid(struct drm_connector *connector, * DAC limit supposedly 355 MHz. */ max_clock = 270000; - else if (IS_GEN3(dev_priv) || IS_GEN4(dev_priv)) + else if (IS_GEN_RANGE(dev_priv, 3, 4)) max_clock = 400000; else max_clock = 350000; @@ -344,57 +345,61 @@ intel_crt_mode_valid(struct drm_connector *connector, return MODE_OK; } -static bool intel_crt_compute_config(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config, - struct drm_connector_state *conn_state) +static int intel_crt_compute_config(struct intel_encoder *encoder, + struct intel_crtc_state *pipe_config, + struct drm_connector_state *conn_state) { struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) - return false; + return -EINVAL; + + pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; - return true; + return 0; } -static bool pch_crt_compute_config(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config, - struct drm_connector_state *conn_state) +static int pch_crt_compute_config(struct intel_encoder *encoder, + struct intel_crtc_state *pipe_config, + struct drm_connector_state *conn_state) { struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) - return false; + return -EINVAL; pipe_config->has_pch_encoder = true; + pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; - return true; + return 0; } -static bool hsw_crt_compute_config(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config, - struct drm_connector_state *conn_state) +static int hsw_crt_compute_config(struct intel_encoder *encoder, + struct intel_crtc_state *pipe_config, + struct drm_connector_state *conn_state) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) - return false; + return -EINVAL; /* HSW/BDW FDI limited to 4k */ if (adjusted_mode->crtc_hdisplay > 4096 || adjusted_mode->crtc_hblank_start > 4096) - return false; + return -EINVAL; pipe_config->has_pch_encoder = true; + pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; /* LPT FDI RX only supports 8bpc. */ if (HAS_PCH_LPT(dev_priv)) { if (pipe_config->bw_constrained && pipe_config->pipe_bpp < 24) { DRM_DEBUG_KMS("LPT only supports 24bpp\n"); - return false; + return -EINVAL; } pipe_config->pipe_bpp = 24; @@ -403,7 +408,7 @@ static bool hsw_crt_compute_config(struct intel_encoder *encoder, /* FDI must always be 2.7 GHz */ pipe_config->port_clock = 135000 * 2; - return true; + return 0; } static bool intel_ironlake_crt_detect_hotplug(struct drm_connector *connector) @@ -626,19 +631,19 @@ static bool intel_crt_detect_ddc(struct drm_connector *connector) } static enum drm_connector_status -intel_crt_load_detect(struct intel_crt *crt, uint32_t pipe) +intel_crt_load_detect(struct intel_crt *crt, u32 pipe) { struct drm_device *dev = crt->base.base.dev; struct drm_i915_private *dev_priv = to_i915(dev); - uint32_t save_bclrpat; - uint32_t save_vtotal; - uint32_t vtotal, vactive; - uint32_t vsample; - uint32_t vblank, vblank_start, vblank_end; - uint32_t dsl; + u32 save_bclrpat; + u32 save_vtotal; + u32 vtotal, vactive; + u32 vsample; + u32 vblank, vblank_start, vblank_end; + u32 dsl; i915_reg_t bclrpat_reg, vtotal_reg, vblank_reg, vsync_reg, pipeconf_reg, pipe_dsl_reg; - uint8_t st00; + u8 st00; enum drm_connector_status status; DRM_DEBUG_KMS("starting load-detect on CRT\n"); @@ -663,8 +668,8 @@ intel_crt_load_detect(struct intel_crt *crt, uint32_t pipe) /* Set the border color to purple. */ I915_WRITE(bclrpat_reg, 0x500050); - if (!IS_GEN2(dev_priv)) { - uint32_t pipeconf = I915_READ(pipeconf_reg); + if (!IS_GEN(dev_priv, 2)) { + u32 pipeconf = I915_READ(pipeconf_reg); I915_WRITE(pipeconf_reg, pipeconf | PIPECONF_FORCE_BORDER); POSTING_READ(pipeconf_reg); /* Wait for next Vblank to substitue @@ -685,8 +690,8 @@ intel_crt_load_detect(struct intel_crt *crt, uint32_t pipe) * Yes, this will flicker */ if (vblank_start <= vactive && vblank_end >= vtotal) { - uint32_t vsync = I915_READ(vsync_reg); - uint32_t vsync_start = (vsync & 0xffff) + 1; + u32 vsync = I915_READ(vsync_reg); + u32 vsync_start = (vsync & 0xffff) + 1; vblank_start = vsync_start; I915_WRITE(vblank_reg, @@ -774,6 +779,7 @@ intel_crt_detect(struct drm_connector *connector, struct drm_i915_private *dev_priv = to_i915(connector->dev); struct intel_crt *crt = intel_attached_crt(connector); struct intel_encoder *intel_encoder = &crt->base; + intel_wakeref_t wakeref; int status, ret; struct intel_load_detect_pipe tmp; @@ -782,7 +788,8 @@ intel_crt_detect(struct drm_connector *connector, force); if (i915_modparams.load_detect_test) { - intel_display_power_get(dev_priv, intel_encoder->power_domain); + wakeref = intel_display_power_get(dev_priv, + intel_encoder->power_domain); goto load_detect; } @@ -790,7 +797,8 @@ intel_crt_detect(struct drm_connector *connector, if (dmi_check_system(intel_spurious_crt_detect)) return connector_status_disconnected; - intel_display_power_get(dev_priv, intel_encoder->power_domain); + wakeref = intel_display_power_get(dev_priv, + intel_encoder->power_domain); if (I915_HAS_HOTPLUG(dev_priv)) { /* We can not rely on the HPD pin always being correctly wired @@ -845,26 +853,22 @@ load_detect: } out: - intel_display_power_put(dev_priv, intel_encoder->power_domain); + intel_display_power_put(dev_priv, intel_encoder->power_domain, wakeref); return status; } -static void intel_crt_destroy(struct drm_connector *connector) -{ - drm_connector_cleanup(connector); - kfree(connector); -} - static int intel_crt_get_modes(struct drm_connector *connector) { struct drm_device *dev = connector->dev; struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crt *crt = intel_attached_crt(connector); struct intel_encoder *intel_encoder = &crt->base; - int ret; + intel_wakeref_t wakeref; struct i2c_adapter *i2c; + int ret; - intel_display_power_get(dev_priv, intel_encoder->power_domain); + wakeref = intel_display_power_get(dev_priv, + intel_encoder->power_domain); i2c = intel_gmbus_get_adapter(dev_priv, dev_priv->vbt.crt_ddc_pin); ret = intel_crt_ddc_get_modes(connector, i2c); @@ -876,7 +880,7 @@ static int intel_crt_get_modes(struct drm_connector *connector) ret = intel_crt_ddc_get_modes(connector, i2c); out: - intel_display_power_put(dev_priv, intel_encoder->power_domain); + intel_display_power_put(dev_priv, intel_encoder->power_domain, wakeref); return ret; } @@ -909,7 +913,7 @@ static const struct drm_connector_funcs intel_crt_connector_funcs = { .fill_modes = drm_helper_probe_single_connector_modes, .late_register = intel_connector_register, .early_unregister = intel_connector_unregister, - .destroy = intel_crt_destroy, + .destroy = intel_connector_destroy, .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, }; @@ -984,7 +988,7 @@ void intel_crt_init(struct drm_i915_private *dev_priv) else crt->base.crtc_mask = (1 << 0) | (1 << 1) | (1 << 2); - if (IS_GEN2(dev_priv)) + if (IS_GEN(dev_priv, 2)) connector->interlace_allowed = 0; else connector->interlace_allowed = 1; diff --git a/drivers/gpu/drm/i915/intel_csr.c b/drivers/gpu/drm/i915/intel_csr.c index d48186e9ddad..e8ac04c33e29 100644 --- a/drivers/gpu/drm/i915/intel_csr.c +++ b/drivers/gpu/drm/i915/intel_csr.c @@ -34,82 +34,86 @@ * low-power state and comes back to normal. */ -#define I915_CSR_ICL "i915/icl_dmc_ver1_07.bin" -MODULE_FIRMWARE(I915_CSR_ICL); -#define ICL_CSR_VERSION_REQUIRED CSR_VERSION(1, 7) +#define GEN12_CSR_MAX_FW_SIZE ICL_CSR_MAX_FW_SIZE -#define I915_CSR_GLK "i915/glk_dmc_ver1_04.bin" -MODULE_FIRMWARE(I915_CSR_GLK); -#define GLK_CSR_VERSION_REQUIRED CSR_VERSION(1, 4) +#define ICL_CSR_PATH "i915/icl_dmc_ver1_07.bin" +#define ICL_CSR_VERSION_REQUIRED CSR_VERSION(1, 7) +#define ICL_CSR_MAX_FW_SIZE 0x6000 +MODULE_FIRMWARE(ICL_CSR_PATH); -#define I915_CSR_CNL "i915/cnl_dmc_ver1_07.bin" -MODULE_FIRMWARE(I915_CSR_CNL); +#define CNL_CSR_PATH "i915/cnl_dmc_ver1_07.bin" #define CNL_CSR_VERSION_REQUIRED CSR_VERSION(1, 7) +#define CNL_CSR_MAX_FW_SIZE GLK_CSR_MAX_FW_SIZE +MODULE_FIRMWARE(CNL_CSR_PATH); -#define I915_CSR_KBL "i915/kbl_dmc_ver1_04.bin" -MODULE_FIRMWARE(I915_CSR_KBL); +#define GLK_CSR_PATH "i915/glk_dmc_ver1_04.bin" +#define GLK_CSR_VERSION_REQUIRED CSR_VERSION(1, 4) +#define GLK_CSR_MAX_FW_SIZE 0x4000 +MODULE_FIRMWARE(GLK_CSR_PATH); + +#define KBL_CSR_PATH "i915/kbl_dmc_ver1_04.bin" #define KBL_CSR_VERSION_REQUIRED CSR_VERSION(1, 4) +#define KBL_CSR_MAX_FW_SIZE BXT_CSR_MAX_FW_SIZE +MODULE_FIRMWARE(KBL_CSR_PATH); -#define I915_CSR_SKL "i915/skl_dmc_ver1_27.bin" -MODULE_FIRMWARE(I915_CSR_SKL); +#define SKL_CSR_PATH "i915/skl_dmc_ver1_27.bin" #define SKL_CSR_VERSION_REQUIRED CSR_VERSION(1, 27) +#define SKL_CSR_MAX_FW_SIZE BXT_CSR_MAX_FW_SIZE +MODULE_FIRMWARE(SKL_CSR_PATH); -#define I915_CSR_BXT "i915/bxt_dmc_ver1_07.bin" -MODULE_FIRMWARE(I915_CSR_BXT); +#define BXT_CSR_PATH "i915/bxt_dmc_ver1_07.bin" #define BXT_CSR_VERSION_REQUIRED CSR_VERSION(1, 7) - - #define BXT_CSR_MAX_FW_SIZE 0x3000 -#define GLK_CSR_MAX_FW_SIZE 0x4000 -#define ICL_CSR_MAX_FW_SIZE 0x6000 +MODULE_FIRMWARE(BXT_CSR_PATH); + #define CSR_DEFAULT_FW_OFFSET 0xFFFFFFFF struct intel_css_header { /* 0x09 for DMC */ - uint32_t module_type; + u32 module_type; /* Includes the DMC specific header in dwords */ - uint32_t header_len; + u32 header_len; /* always value would be 0x10000 */ - uint32_t header_ver; + u32 header_ver; /* Not used */ - uint32_t module_id; + u32 module_id; /* Not used */ - uint32_t module_vendor; + u32 module_vendor; /* in YYYYMMDD format */ - uint32_t date; + u32 date; /* Size in dwords (CSS_Headerlen + PackageHeaderLen + dmc FWsLen)/4 */ - uint32_t size; + u32 size; /* Not used */ - uint32_t key_size; + u32 key_size; /* Not used */ - uint32_t modulus_size; + u32 modulus_size; /* Not used */ - uint32_t exponent_size; + u32 exponent_size; /* Not used */ - uint32_t reserved1[12]; + u32 reserved1[12]; /* Major Minor */ - uint32_t version; + u32 version; /* Not used */ - uint32_t reserved2[8]; + u32 reserved2[8]; /* Not used */ - uint32_t kernel_header_info; + u32 kernel_header_info; } __packed; struct intel_fw_info { - uint16_t reserved1; + u16 reserved1; /* Stepping (A, B, C, ..., *). * is a wildcard */ char stepping; @@ -117,8 +121,8 @@ struct intel_fw_info { /* Sub-stepping (0, 1, ..., *). * is a wildcard */ char substepping; - uint32_t offset; - uint32_t reserved2; + u32 offset; + u32 reserved2; } __packed; struct intel_package_header { @@ -131,14 +135,14 @@ struct intel_package_header { unsigned char reserved[10]; /* Number of valid entries in the FWInfo array below */ - uint32_t num_entries; + u32 num_entries; struct intel_fw_info fw_info[20]; } __packed; struct intel_dmc_header { /* always value would be 0x40403E3E */ - uint32_t signature; + u32 signature; /* DMC binary header length */ unsigned char header_len; @@ -147,30 +151,30 @@ struct intel_dmc_header { unsigned char header_ver; /* Reserved */ - uint16_t dmcc_ver; + u16 dmcc_ver; /* Major, Minor */ - uint32_t project; + u32 project; /* Firmware program size (excluding header) in dwords */ - uint32_t fw_size; + u32 fw_size; /* Major Minor version */ - uint32_t fw_version; + u32 fw_version; /* Number of valid MMIO cycles present. */ - uint32_t mmio_count; + u32 mmio_count; /* MMIO address */ - uint32_t mmioaddr[8]; + u32 mmioaddr[8]; /* MMIO data */ - uint32_t mmiodata[8]; + u32 mmiodata[8]; /* FW filename */ unsigned char dfile[32]; - uint32_t reserved1[2]; + u32 reserved1[2]; } __packed; struct stepping_info { @@ -190,6 +194,12 @@ static const struct stepping_info bxt_stepping_info[] = { {'B', '0'}, {'B', '1'}, {'B', '2'} }; +static const struct stepping_info icl_stepping_info[] = { + {'A', '0'}, {'A', '1'}, {'A', '2'}, + {'B', '0'}, {'B', '2'}, + {'C', '0'} +}; + static const struct stepping_info no_stepping_info = { '*', '*' }; static const struct stepping_info * @@ -198,7 +208,10 @@ intel_get_stepping_info(struct drm_i915_private *dev_priv) const struct stepping_info *si; unsigned int size; - if (IS_SKYLAKE(dev_priv)) { + if (IS_ICELAKE(dev_priv)) { + size = ARRAY_SIZE(icl_stepping_info); + si = icl_stepping_info; + } else if (IS_SKYLAKE(dev_priv)) { size = ARRAY_SIZE(skl_stepping_info); si = skl_stepping_info; } else if (IS_BROXTON(dev_priv)) { @@ -217,7 +230,7 @@ intel_get_stepping_info(struct drm_i915_private *dev_priv) static void gen9_set_dc_state_debugmask(struct drm_i915_private *dev_priv) { - uint32_t val, mask; + u32 val, mask; mask = DC_STATE_DEBUG_MASK_MEMORY_UP; @@ -244,7 +257,7 @@ static void gen9_set_dc_state_debugmask(struct drm_i915_private *dev_priv) void intel_csr_load_program(struct drm_i915_private *dev_priv) { u32 *payload = dev_priv->csr.dmc_payload; - uint32_t i, fw_size; + u32 i, fw_size; if (!HAS_CSR(dev_priv)) { DRM_ERROR("No CSR support available for this platform\n"); @@ -276,19 +289,17 @@ void intel_csr_load_program(struct drm_i915_private *dev_priv) gen9_set_dc_state_debugmask(dev_priv); } -static uint32_t *parse_csr_fw(struct drm_i915_private *dev_priv, - const struct firmware *fw) +static u32 *parse_csr_fw(struct drm_i915_private *dev_priv, + const struct firmware *fw) { struct intel_css_header *css_header; struct intel_package_header *package_header; struct intel_dmc_header *dmc_header; struct intel_csr *csr = &dev_priv->csr; const struct stepping_info *si = intel_get_stepping_info(dev_priv); - uint32_t dmc_offset = CSR_DEFAULT_FW_OFFSET, readcount = 0, nbytes; - uint32_t max_fw_size = 0; - uint32_t i; - uint32_t *dmc_payload; - uint32_t required_version; + u32 dmc_offset = CSR_DEFAULT_FW_OFFSET, readcount = 0, nbytes; + u32 i; + u32 *dmc_payload; if (!fw) return NULL; @@ -303,38 +314,19 @@ static uint32_t *parse_csr_fw(struct drm_i915_private *dev_priv, return NULL; } - csr->version = css_header->version; - - if (csr->fw_path == i915_modparams.dmc_firmware_path) { - /* Bypass version check for firmware override. */ - required_version = csr->version; - } else if (IS_ICELAKE(dev_priv)) { - required_version = ICL_CSR_VERSION_REQUIRED; - } else if (IS_CANNONLAKE(dev_priv)) { - required_version = CNL_CSR_VERSION_REQUIRED; - } else if (IS_GEMINILAKE(dev_priv)) { - required_version = GLK_CSR_VERSION_REQUIRED; - } else if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) { - required_version = KBL_CSR_VERSION_REQUIRED; - } else if (IS_SKYLAKE(dev_priv)) { - required_version = SKL_CSR_VERSION_REQUIRED; - } else if (IS_BROXTON(dev_priv)) { - required_version = BXT_CSR_VERSION_REQUIRED; - } else { - MISSING_CASE(INTEL_REVID(dev_priv)); - required_version = 0; - } - - if (csr->version != required_version) { + if (csr->required_version && + css_header->version != csr->required_version) { DRM_INFO("Refusing to load DMC firmware v%u.%u," " please use v%u.%u\n", - CSR_VERSION_MAJOR(csr->version), - CSR_VERSION_MINOR(csr->version), - CSR_VERSION_MAJOR(required_version), - CSR_VERSION_MINOR(required_version)); + CSR_VERSION_MAJOR(css_header->version), + CSR_VERSION_MINOR(css_header->version), + CSR_VERSION_MAJOR(csr->required_version), + CSR_VERSION_MINOR(csr->required_version)); return NULL; } + csr->version = css_header->version; + readcount += sizeof(struct intel_css_header); /* Extract Package Header information*/ @@ -402,15 +394,7 @@ static uint32_t *parse_csr_fw(struct drm_i915_private *dev_priv, /* fw_size is in dwords, so multiplied by 4 to convert into bytes. */ nbytes = dmc_header->fw_size * 4; - if (INTEL_GEN(dev_priv) >= 11) - max_fw_size = ICL_CSR_MAX_FW_SIZE; - else if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv)) - max_fw_size = GLK_CSR_MAX_FW_SIZE; - else if (IS_GEN9(dev_priv)) - max_fw_size = BXT_CSR_MAX_FW_SIZE; - else - MISSING_CASE(INTEL_REVID(dev_priv)); - if (nbytes > max_fw_size) { + if (nbytes > csr->max_fw_size) { DRM_ERROR("DMC FW too big (%u bytes)\n", nbytes); return NULL; } @@ -425,6 +409,21 @@ static uint32_t *parse_csr_fw(struct drm_i915_private *dev_priv, return memcpy(dmc_payload, &fw->data[readcount], nbytes); } +static void intel_csr_runtime_pm_get(struct drm_i915_private *dev_priv) +{ + WARN_ON(dev_priv->csr.wakeref); + dev_priv->csr.wakeref = + intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); +} + +static void intel_csr_runtime_pm_put(struct drm_i915_private *dev_priv) +{ + intel_wakeref_t wakeref __maybe_unused = + fetch_and_zero(&dev_priv->csr.wakeref); + + intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, wakeref); +} + static void csr_load_work_fn(struct work_struct *work) { struct drm_i915_private *dev_priv; @@ -440,8 +439,7 @@ static void csr_load_work_fn(struct work_struct *work) if (dev_priv->csr.dmc_payload) { intel_csr_load_program(dev_priv); - - intel_display_power_put(dev_priv, POWER_DOMAIN_INIT); + intel_csr_runtime_pm_put(dev_priv); DRM_INFO("Finished loading DMC firmware %s (v%u.%u)\n", dev_priv->csr.fw_path, @@ -475,26 +473,56 @@ void intel_csr_ucode_init(struct drm_i915_private *dev_priv) if (!HAS_CSR(dev_priv)) return; - if (i915_modparams.dmc_firmware_path) - csr->fw_path = i915_modparams.dmc_firmware_path; - else if (IS_ICELAKE(dev_priv)) - csr->fw_path = I915_CSR_ICL; - else if (IS_CANNONLAKE(dev_priv)) - csr->fw_path = I915_CSR_CNL; - else if (IS_GEMINILAKE(dev_priv)) - csr->fw_path = I915_CSR_GLK; - else if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) - csr->fw_path = I915_CSR_KBL; - else if (IS_SKYLAKE(dev_priv)) - csr->fw_path = I915_CSR_SKL; - else if (IS_BROXTON(dev_priv)) - csr->fw_path = I915_CSR_BXT; - /* - * Obtain a runtime pm reference, until CSR is loaded, - * to avoid entering runtime-suspend. + * Obtain a runtime pm reference, until CSR is loaded, to avoid entering + * runtime-suspend. + * + * On error, we return with the rpm wakeref held to prevent runtime + * suspend as runtime suspend *requires* a working CSR for whatever + * reason. */ - intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); + intel_csr_runtime_pm_get(dev_priv); + + if (INTEL_GEN(dev_priv) >= 12) { + /* Allow to load fw via parameter using the last known size */ + csr->max_fw_size = GEN12_CSR_MAX_FW_SIZE; + } else if (IS_ICELAKE(dev_priv)) { + csr->fw_path = ICL_CSR_PATH; + csr->required_version = ICL_CSR_VERSION_REQUIRED; + csr->max_fw_size = ICL_CSR_MAX_FW_SIZE; + } else if (IS_CANNONLAKE(dev_priv)) { + csr->fw_path = CNL_CSR_PATH; + csr->required_version = CNL_CSR_VERSION_REQUIRED; + csr->max_fw_size = CNL_CSR_MAX_FW_SIZE; + } else if (IS_GEMINILAKE(dev_priv)) { + csr->fw_path = GLK_CSR_PATH; + csr->required_version = GLK_CSR_VERSION_REQUIRED; + csr->max_fw_size = GLK_CSR_MAX_FW_SIZE; + } else if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) { + csr->fw_path = KBL_CSR_PATH; + csr->required_version = KBL_CSR_VERSION_REQUIRED; + csr->max_fw_size = KBL_CSR_MAX_FW_SIZE; + } else if (IS_SKYLAKE(dev_priv)) { + csr->fw_path = SKL_CSR_PATH; + csr->required_version = SKL_CSR_VERSION_REQUIRED; + csr->max_fw_size = SKL_CSR_MAX_FW_SIZE; + } else if (IS_BROXTON(dev_priv)) { + csr->fw_path = BXT_CSR_PATH; + csr->required_version = BXT_CSR_VERSION_REQUIRED; + csr->max_fw_size = BXT_CSR_MAX_FW_SIZE; + } + + if (i915_modparams.dmc_firmware_path) { + if (strlen(i915_modparams.dmc_firmware_path) == 0) { + csr->fw_path = NULL; + DRM_INFO("Disabling CSR firmware and runtime PM\n"); + return; + } + + csr->fw_path = i915_modparams.dmc_firmware_path; + /* Bypass version check for firmware override. */ + csr->required_version = 0; + } if (csr->fw_path == NULL) { DRM_DEBUG_KMS("No known CSR firmware for platform, disabling runtime PM\n"); @@ -524,7 +552,7 @@ void intel_csr_ucode_suspend(struct drm_i915_private *dev_priv) /* Drop the reference held in case DMC isn't loaded. */ if (!dev_priv->csr.dmc_payload) - intel_display_power_put(dev_priv, POWER_DOMAIN_INIT); + intel_csr_runtime_pm_put(dev_priv); } /** @@ -544,7 +572,7 @@ void intel_csr_ucode_resume(struct drm_i915_private *dev_priv) * loaded. */ if (!dev_priv->csr.dmc_payload) - intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); + intel_csr_runtime_pm_get(dev_priv); } /** @@ -560,6 +588,7 @@ void intel_csr_ucode_fini(struct drm_i915_private *dev_priv) return; intel_csr_ucode_suspend(dev_priv); + WARN_ON(dev_priv->csr.wakeref); kfree(dev_priv->csr.dmc_payload); } diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c index 5186cd7075f9..98cea1f4b3bf 100644 --- a/drivers/gpu/drm/i915/intel_ddi.c +++ b/drivers/gpu/drm/i915/intel_ddi.c @@ -28,6 +28,7 @@ #include <drm/drm_scdc_helper.h> #include "i915_drv.h" #include "intel_drv.h" +#include "intel_dsi.h" struct ddi_buf_trans { u32 trans1; /* balance leg enable, de-emph level */ @@ -493,103 +494,58 @@ static const struct cnl_ddi_buf_trans cnl_ddi_translations_edp_1_05V[] = { { 0x2, 0x7F, 0x3F, 0x00, 0x00 }, /* 400 400 0.0 */ }; -struct icl_combo_phy_ddi_buf_trans { - u32 dw2_swing_select; - u32 dw2_swing_scalar; - u32 dw4_scaling; -}; - -/* Voltage Swing Programming for VccIO 0.85V for DP */ -static const struct icl_combo_phy_ddi_buf_trans icl_combo_phy_ddi_translations_dp_hdmi_0_85V[] = { - /* Voltage mV db */ - { 0x2, 0x98, 0x0018 }, /* 400 0.0 */ - { 0x2, 0x98, 0x3015 }, /* 400 3.5 */ - { 0x2, 0x98, 0x6012 }, /* 400 6.0 */ - { 0x2, 0x98, 0x900F }, /* 400 9.5 */ - { 0xB, 0x70, 0x0018 }, /* 600 0.0 */ - { 0xB, 0x70, 0x3015 }, /* 600 3.5 */ - { 0xB, 0x70, 0x6012 }, /* 600 6.0 */ - { 0x5, 0x00, 0x0018 }, /* 800 0.0 */ - { 0x5, 0x00, 0x3015 }, /* 800 3.5 */ - { 0x6, 0x98, 0x0018 }, /* 1200 0.0 */ -}; - -/* FIXME - After table is updated in Bspec */ -/* Voltage Swing Programming for VccIO 0.85V for eDP */ -static const struct icl_combo_phy_ddi_buf_trans icl_combo_phy_ddi_translations_edp_0_85V[] = { - /* Voltage mV db */ - { 0x0, 0x00, 0x00 }, /* 200 0.0 */ - { 0x0, 0x00, 0x00 }, /* 200 1.5 */ - { 0x0, 0x00, 0x00 }, /* 200 4.0 */ - { 0x0, 0x00, 0x00 }, /* 200 6.0 */ - { 0x0, 0x00, 0x00 }, /* 250 0.0 */ - { 0x0, 0x00, 0x00 }, /* 250 1.5 */ - { 0x0, 0x00, 0x00 }, /* 250 4.0 */ - { 0x0, 0x00, 0x00 }, /* 300 0.0 */ - { 0x0, 0x00, 0x00 }, /* 300 1.5 */ - { 0x0, 0x00, 0x00 }, /* 350 0.0 */ -}; - -/* Voltage Swing Programming for VccIO 0.95V for DP */ -static const struct icl_combo_phy_ddi_buf_trans icl_combo_phy_ddi_translations_dp_hdmi_0_95V[] = { - /* Voltage mV db */ - { 0x2, 0x98, 0x0018 }, /* 400 0.0 */ - { 0x2, 0x98, 0x3015 }, /* 400 3.5 */ - { 0x2, 0x98, 0x6012 }, /* 400 6.0 */ - { 0x2, 0x98, 0x900F }, /* 400 9.5 */ - { 0x4, 0x98, 0x0018 }, /* 600 0.0 */ - { 0x4, 0x98, 0x3015 }, /* 600 3.5 */ - { 0x4, 0x98, 0x6012 }, /* 600 6.0 */ - { 0x5, 0x76, 0x0018 }, /* 800 0.0 */ - { 0x5, 0x76, 0x3015 }, /* 800 3.5 */ - { 0x6, 0x98, 0x0018 }, /* 1200 0.0 */ +/* icl_combo_phy_ddi_translations */ +static const struct cnl_ddi_buf_trans icl_combo_phy_ddi_translations_dp_hbr2[] = { + /* NT mV Trans mV db */ + { 0xA, 0x35, 0x3F, 0x00, 0x00 }, /* 350 350 0.0 */ + { 0xA, 0x4F, 0x37, 0x00, 0x08 }, /* 350 500 3.1 */ + { 0xC, 0x71, 0x2F, 0x00, 0x10 }, /* 350 700 6.0 */ + { 0x6, 0x7F, 0x2B, 0x00, 0x14 }, /* 350 900 8.2 */ + { 0xA, 0x4C, 0x3F, 0x00, 0x00 }, /* 500 500 0.0 */ + { 0xC, 0x73, 0x34, 0x00, 0x0B }, /* 500 700 2.9 */ + { 0x6, 0x7F, 0x2F, 0x00, 0x10 }, /* 500 900 5.1 */ + { 0xC, 0x6C, 0x3C, 0x00, 0x03 }, /* 650 700 0.6 */ + { 0x6, 0x7F, 0x35, 0x00, 0x0A }, /* 600 900 3.5 */ + { 0x6, 0x7F, 0x3F, 0x00, 0x00 }, /* 900 900 0.0 */ }; -/* FIXME - After table is updated in Bspec */ -/* Voltage Swing Programming for VccIO 0.95V for eDP */ -static const struct icl_combo_phy_ddi_buf_trans icl_combo_phy_ddi_translations_edp_0_95V[] = { - /* Voltage mV db */ - { 0x0, 0x00, 0x00 }, /* 200 0.0 */ - { 0x0, 0x00, 0x00 }, /* 200 1.5 */ - { 0x0, 0x00, 0x00 }, /* 200 4.0 */ - { 0x0, 0x00, 0x00 }, /* 200 6.0 */ - { 0x0, 0x00, 0x00 }, /* 250 0.0 */ - { 0x0, 0x00, 0x00 }, /* 250 1.5 */ - { 0x0, 0x00, 0x00 }, /* 250 4.0 */ - { 0x0, 0x00, 0x00 }, /* 300 0.0 */ - { 0x0, 0x00, 0x00 }, /* 300 1.5 */ - { 0x0, 0x00, 0x00 }, /* 350 0.0 */ +static const struct cnl_ddi_buf_trans icl_combo_phy_ddi_translations_edp_hbr2[] = { + /* NT mV Trans mV db */ + { 0x0, 0x7F, 0x3F, 0x00, 0x00 }, /* 200 200 0.0 */ + { 0x8, 0x7F, 0x38, 0x00, 0x07 }, /* 200 250 1.9 */ + { 0x1, 0x7F, 0x33, 0x00, 0x0C }, /* 200 300 3.5 */ + { 0x9, 0x7F, 0x31, 0x00, 0x0E }, /* 200 350 4.9 */ + { 0x8, 0x7F, 0x3F, 0x00, 0x00 }, /* 250 250 0.0 */ + { 0x1, 0x7F, 0x38, 0x00, 0x07 }, /* 250 300 1.6 */ + { 0x9, 0x7F, 0x35, 0x00, 0x0A }, /* 250 350 2.9 */ + { 0x1, 0x7F, 0x3F, 0x00, 0x00 }, /* 300 300 0.0 */ + { 0x9, 0x7F, 0x38, 0x00, 0x07 }, /* 300 350 1.3 */ + { 0x9, 0x7F, 0x3F, 0x00, 0x00 }, /* 350 350 0.0 */ }; -/* Voltage Swing Programming for VccIO 1.05V for DP */ -static const struct icl_combo_phy_ddi_buf_trans icl_combo_phy_ddi_translations_dp_hdmi_1_05V[] = { - /* Voltage mV db */ - { 0x2, 0x98, 0x0018 }, /* 400 0.0 */ - { 0x2, 0x98, 0x3015 }, /* 400 3.5 */ - { 0x2, 0x98, 0x6012 }, /* 400 6.0 */ - { 0x2, 0x98, 0x900F }, /* 400 9.5 */ - { 0x4, 0x98, 0x0018 }, /* 600 0.0 */ - { 0x4, 0x98, 0x3015 }, /* 600 3.5 */ - { 0x4, 0x98, 0x6012 }, /* 600 6.0 */ - { 0x5, 0x71, 0x0018 }, /* 800 0.0 */ - { 0x5, 0x71, 0x3015 }, /* 800 3.5 */ - { 0x6, 0x98, 0x0018 }, /* 1200 0.0 */ +static const struct cnl_ddi_buf_trans icl_combo_phy_ddi_translations_edp_hbr3[] = { + /* NT mV Trans mV db */ + { 0xA, 0x35, 0x3F, 0x00, 0x00 }, /* 350 350 0.0 */ + { 0xA, 0x4F, 0x37, 0x00, 0x08 }, /* 350 500 3.1 */ + { 0xC, 0x71, 0x2F, 0x00, 0x10 }, /* 350 700 6.0 */ + { 0x6, 0x7F, 0x2B, 0x00, 0x14 }, /* 350 900 8.2 */ + { 0xA, 0x4C, 0x3F, 0x00, 0x00 }, /* 500 500 0.0 */ + { 0xC, 0x73, 0x34, 0x00, 0x0B }, /* 500 700 2.9 */ + { 0x6, 0x7F, 0x2F, 0x00, 0x10 }, /* 500 900 5.1 */ + { 0xC, 0x6C, 0x3C, 0x00, 0x03 }, /* 650 700 0.6 */ + { 0x6, 0x7F, 0x35, 0x00, 0x0A }, /* 600 900 3.5 */ + { 0x6, 0x7F, 0x3F, 0x00, 0x00 }, /* 900 900 0.0 */ }; -/* FIXME - After table is updated in Bspec */ -/* Voltage Swing Programming for VccIO 1.05V for eDP */ -static const struct icl_combo_phy_ddi_buf_trans icl_combo_phy_ddi_translations_edp_1_05V[] = { - /* Voltage mV db */ - { 0x0, 0x00, 0x00 }, /* 200 0.0 */ - { 0x0, 0x00, 0x00 }, /* 200 1.5 */ - { 0x0, 0x00, 0x00 }, /* 200 4.0 */ - { 0x0, 0x00, 0x00 }, /* 200 6.0 */ - { 0x0, 0x00, 0x00 }, /* 250 0.0 */ - { 0x0, 0x00, 0x00 }, /* 250 1.5 */ - { 0x0, 0x00, 0x00 }, /* 250 4.0 */ - { 0x0, 0x00, 0x00 }, /* 300 0.0 */ - { 0x0, 0x00, 0x00 }, /* 300 1.5 */ - { 0x0, 0x00, 0x00 }, /* 350 0.0 */ +static const struct cnl_ddi_buf_trans icl_combo_phy_ddi_translations_hdmi[] = { + /* NT mV Trans mV db */ + { 0xA, 0x60, 0x3F, 0x00, 0x00 }, /* 450 450 0.0 */ + { 0xB, 0x73, 0x36, 0x00, 0x09 }, /* 450 650 3.2 */ + { 0x6, 0x7F, 0x31, 0x00, 0x0E }, /* 450 850 5.5 */ + { 0xB, 0x73, 0x3F, 0x00, 0x00 }, /* 650 650 0.0 ALS */ + { 0x6, 0x7F, 0x37, 0x00, 0x08 }, /* 650 850 2.3 */ + { 0x6, 0x7F, 0x3F, 0x00, 0x00 }, /* 850 850 0.0 */ + { 0x6, 0x7F, 0x35, 0x00, 0x0A }, /* 600 850 3.0 */ }; struct icl_mg_phy_ddi_buf_trans { @@ -642,7 +598,7 @@ skl_get_buf_trans_dp(struct drm_i915_private *dev_priv, int *n_entries) static const struct ddi_buf_trans * kbl_get_buf_trans_dp(struct drm_i915_private *dev_priv, int *n_entries) { - if (IS_KBL_ULX(dev_priv)) { + if (IS_KBL_ULX(dev_priv) || IS_AML_ULX(dev_priv)) { *n_entries = ARRAY_SIZE(kbl_y_ddi_translations_dp); return kbl_y_ddi_translations_dp; } else if (IS_KBL_ULT(dev_priv) || IS_CFL_ULT(dev_priv)) { @@ -658,7 +614,7 @@ static const struct ddi_buf_trans * skl_get_buf_trans_edp(struct drm_i915_private *dev_priv, int *n_entries) { if (dev_priv->vbt.edp.low_vswing) { - if (IS_SKL_ULX(dev_priv) || IS_KBL_ULX(dev_priv)) { + if (IS_SKL_ULX(dev_priv) || IS_KBL_ULX(dev_priv) || IS_AML_ULX(dev_priv)) { *n_entries = ARRAY_SIZE(skl_y_ddi_translations_edp); return skl_y_ddi_translations_edp; } else if (IS_SKL_ULT(dev_priv) || IS_KBL_ULT(dev_priv) || @@ -680,7 +636,7 @@ skl_get_buf_trans_edp(struct drm_i915_private *dev_priv, int *n_entries) static const struct ddi_buf_trans * skl_get_buf_trans_hdmi(struct drm_i915_private *dev_priv, int *n_entries) { - if (IS_SKL_ULX(dev_priv) || IS_KBL_ULX(dev_priv)) { + if (IS_SKL_ULX(dev_priv) || IS_KBL_ULX(dev_priv) || IS_AML_ULX(dev_priv)) { *n_entries = ARRAY_SIZE(skl_y_ddi_translations_hdmi); return skl_y_ddi_translations_hdmi; } else { @@ -870,43 +826,23 @@ cnl_get_buf_trans_edp(struct drm_i915_private *dev_priv, int *n_entries) } } -static const struct icl_combo_phy_ddi_buf_trans * +static const struct cnl_ddi_buf_trans * icl_get_combo_buf_trans(struct drm_i915_private *dev_priv, enum port port, - int type, int *n_entries) -{ - u32 voltage = I915_READ(ICL_PORT_COMP_DW3(port)) & VOLTAGE_INFO_MASK; - - if (type == INTEL_OUTPUT_EDP && dev_priv->vbt.edp.low_vswing) { - switch (voltage) { - case VOLTAGE_INFO_0_85V: - *n_entries = ARRAY_SIZE(icl_combo_phy_ddi_translations_edp_0_85V); - return icl_combo_phy_ddi_translations_edp_0_85V; - case VOLTAGE_INFO_0_95V: - *n_entries = ARRAY_SIZE(icl_combo_phy_ddi_translations_edp_0_95V); - return icl_combo_phy_ddi_translations_edp_0_95V; - case VOLTAGE_INFO_1_05V: - *n_entries = ARRAY_SIZE(icl_combo_phy_ddi_translations_edp_1_05V); - return icl_combo_phy_ddi_translations_edp_1_05V; - default: - MISSING_CASE(voltage); - return NULL; - } - } else { - switch (voltage) { - case VOLTAGE_INFO_0_85V: - *n_entries = ARRAY_SIZE(icl_combo_phy_ddi_translations_dp_hdmi_0_85V); - return icl_combo_phy_ddi_translations_dp_hdmi_0_85V; - case VOLTAGE_INFO_0_95V: - *n_entries = ARRAY_SIZE(icl_combo_phy_ddi_translations_dp_hdmi_0_95V); - return icl_combo_phy_ddi_translations_dp_hdmi_0_95V; - case VOLTAGE_INFO_1_05V: - *n_entries = ARRAY_SIZE(icl_combo_phy_ddi_translations_dp_hdmi_1_05V); - return icl_combo_phy_ddi_translations_dp_hdmi_1_05V; - default: - MISSING_CASE(voltage); - return NULL; - } + int type, int rate, int *n_entries) +{ + if (type == INTEL_OUTPUT_HDMI) { + *n_entries = ARRAY_SIZE(icl_combo_phy_ddi_translations_hdmi); + return icl_combo_phy_ddi_translations_hdmi; + } else if (rate > 540000 && type == INTEL_OUTPUT_EDP) { + *n_entries = ARRAY_SIZE(icl_combo_phy_ddi_translations_edp_hbr3); + return icl_combo_phy_ddi_translations_edp_hbr3; + } else if (type == INTEL_OUTPUT_EDP && dev_priv->vbt.edp.low_vswing) { + *n_entries = ARRAY_SIZE(icl_combo_phy_ddi_translations_edp_hbr2); + return icl_combo_phy_ddi_translations_edp_hbr2; } + + *n_entries = ARRAY_SIZE(icl_combo_phy_ddi_translations_dp_hbr2); + return icl_combo_phy_ddi_translations_dp_hbr2; } static int intel_ddi_hdmi_level(struct drm_i915_private *dev_priv, enum port port) @@ -917,8 +853,8 @@ static int intel_ddi_hdmi_level(struct drm_i915_private *dev_priv, enum port por if (IS_ICELAKE(dev_priv)) { if (intel_port_is_combophy(dev_priv, port)) - icl_get_combo_buf_trans(dev_priv, port, - INTEL_OUTPUT_HDMI, &n_entries); + icl_get_combo_buf_trans(dev_priv, port, INTEL_OUTPUT_HDMI, + 0, &n_entries); else n_entries = ARRAY_SIZE(icl_mg_phy_ddi_translations); default_entry = n_entries - 1; @@ -1038,7 +974,7 @@ static void intel_wait_ddi_buf_idle(struct drm_i915_private *dev_priv, DRM_ERROR("Timeout waiting for DDI BUF %c idle bit\n", port_name(port)); } -static uint32_t hsw_pll_to_ddi_pll_sel(const struct intel_shared_dpll *pll) +static u32 hsw_pll_to_ddi_pll_sel(const struct intel_shared_dpll *pll) { switch (pll->info->id) { case DPLL_ID_WRPLL1: @@ -1059,19 +995,20 @@ static uint32_t hsw_pll_to_ddi_pll_sel(const struct intel_shared_dpll *pll) } } -static uint32_t icl_pll_to_ddi_pll_sel(struct intel_encoder *encoder, - const struct intel_shared_dpll *pll) +static u32 icl_pll_to_ddi_clk_sel(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state) { - struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); - int clock = crtc->config->port_clock; + const struct intel_shared_dpll *pll = crtc_state->shared_dpll; + int clock = crtc_state->port_clock; const enum intel_dpll_id id = pll->info->id; switch (id) { default: + /* + * DPLL_ID_ICL_DPLL0 and DPLL_ID_ICL_DPLL1 should not be used + * here, so do warn if this get passed in + */ MISSING_CASE(id); - /* fall through */ - case DPLL_ID_ICL_DPLL0: - case DPLL_ID_ICL_DPLL1: return DDI_CLK_SEL_NONE; case DPLL_ID_ICL_TBTPLL: switch (clock) { @@ -1085,7 +1022,7 @@ static uint32_t icl_pll_to_ddi_pll_sel(struct intel_encoder *encoder, return DDI_CLK_SEL_TBT_810; default: MISSING_CASE(clock); - break; + return DDI_CLK_SEL_NONE; } case DPLL_ID_ICL_MGPLL1: case DPLL_ID_ICL_MGPLL2: @@ -1307,8 +1244,8 @@ static int skl_calc_wrpll_link(struct drm_i915_private *dev_priv, enum intel_dpll_id pll_id) { i915_reg_t cfgcr1_reg, cfgcr2_reg; - uint32_t cfgcr1_val, cfgcr2_val; - uint32_t p0, p1, p2, dco_freq; + u32 cfgcr1_val, cfgcr2_val; + u32 p0, p1, p2, dco_freq; cfgcr1_reg = DPLL_CFGCR1(pll_id); cfgcr2_reg = DPLL_CFGCR2(pll_id); @@ -1360,14 +1297,17 @@ static int skl_calc_wrpll_link(struct drm_i915_private *dev_priv, dco_freq += (((cfgcr1_val & DPLL_CFGCR1_DCO_FRACTION_MASK) >> 9) * 24 * 1000) / 0x8000; + if (WARN_ON(p0 == 0 || p1 == 0 || p2 == 0)) + return 0; + return dco_freq / (p0 * p1 * p2 * 5); } -static int cnl_calc_wrpll_link(struct drm_i915_private *dev_priv, - enum intel_dpll_id pll_id) +int cnl_calc_wrpll_link(struct drm_i915_private *dev_priv, + enum intel_dpll_id pll_id) { - uint32_t cfgcr0, cfgcr1; - uint32_t p0, p1, p2, dco_freq, ref_clock; + u32 cfgcr0, cfgcr1; + u32 p0, p1, p2, dco_freq, ref_clock; if (INTEL_GEN(dev_priv) >= 11) { cfgcr0 = I915_READ(ICL_DPLL_CFGCR0(pll_id)); @@ -1452,16 +1392,17 @@ static int icl_calc_tbt_pll_link(struct drm_i915_private *dev_priv, static int icl_calc_mg_pll_link(struct drm_i915_private *dev_priv, enum port port) { + enum tc_port tc_port = intel_port_to_tc(dev_priv, port); u32 mg_pll_div0, mg_clktop_hsclkctl; u32 m1, m2_int, m2_frac, div1, div2, refclk; u64 tmp; refclk = dev_priv->cdclk.hw.ref; - mg_pll_div0 = I915_READ(MG_PLL_DIV0(port)); - mg_clktop_hsclkctl = I915_READ(MG_CLKTOP2_HSCLKCTL(port)); + mg_pll_div0 = I915_READ(MG_PLL_DIV0(tc_port)); + mg_clktop_hsclkctl = I915_READ(MG_CLKTOP2_HSCLKCTL(tc_port)); - m1 = I915_READ(MG_PLL_DIV1(port)) & MG_PLL_DIV1_FBPREDIV_MASK; + m1 = I915_READ(MG_PLL_DIV1(tc_port)) & MG_PLL_DIV1_FBPREDIV_MASK; m2_int = mg_pll_div0 & MG_PLL_DIV0_FBDIV_INT_MASK; m2_frac = (mg_pll_div0 & MG_PLL_DIV0_FRACNEN_H) ? (mg_pll_div0 & MG_PLL_DIV0_FBDIV_FRAC_MASK) >> @@ -1517,7 +1458,7 @@ static void ddi_dotclock_get(struct intel_crtc_state *pipe_config) else dotclock = pipe_config->port_clock; - if (pipe_config->ycbcr420) + if (pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420) dotclock *= 2; if (pipe_config->pixel_multiplier) @@ -1532,7 +1473,7 @@ static void icl_ddi_clock_get(struct intel_encoder *encoder, struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); enum port port = encoder->port; int link_clock = 0; - uint32_t pll_id; + u32 pll_id; pll_id = intel_get_shared_dpll_id(dev_priv, pipe_config->shared_dpll); if (intel_port_is_combophy(dev_priv, port)) { @@ -1557,7 +1498,7 @@ static void cnl_ddi_clock_get(struct intel_encoder *encoder, { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); int link_clock = 0; - uint32_t cfgcr0; + u32 cfgcr0; enum intel_dpll_id pll_id; pll_id = intel_get_shared_dpll_id(dev_priv, pipe_config->shared_dpll); @@ -1611,7 +1552,7 @@ static void skl_ddi_clock_get(struct intel_encoder *encoder, { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); int link_clock = 0; - uint32_t dpll_ctl1; + u32 dpll_ctl1; enum intel_dpll_id pll_id; pll_id = intel_get_shared_dpll_id(dev_priv, pipe_config->shared_dpll); @@ -1737,16 +1678,16 @@ static void intel_ddi_clock_get(struct intel_encoder *encoder, { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - if (INTEL_GEN(dev_priv) <= 8) - hsw_ddi_clock_get(encoder, pipe_config); - else if (IS_GEN9_BC(dev_priv)) - skl_ddi_clock_get(encoder, pipe_config); - else if (IS_GEN9_LP(dev_priv)) - bxt_ddi_clock_get(encoder, pipe_config); + if (IS_ICELAKE(dev_priv)) + icl_ddi_clock_get(encoder, pipe_config); else if (IS_CANNONLAKE(dev_priv)) cnl_ddi_clock_get(encoder, pipe_config); - else if (IS_ICELAKE(dev_priv)) - icl_ddi_clock_get(encoder, pipe_config); + else if (IS_GEN9_LP(dev_priv)) + bxt_ddi_clock_get(encoder, pipe_config); + else if (IS_GEN9_BC(dev_priv)) + skl_ddi_clock_get(encoder, pipe_config); + else if (INTEL_GEN(dev_priv) <= 8) + hsw_ddi_clock_get(encoder, pipe_config); } void intel_ddi_set_pipe_settings(const struct intel_crtc_state *crtc_state) @@ -1784,6 +1725,13 @@ void intel_ddi_set_pipe_settings(const struct intel_crtc_state *crtc_state) break; } + /* + * As per DP 1.2 spec section 2.3.4.3 while sending + * YCBCR 444 signals we should program MSA MISC1/0 fields with + * colorspace information. The output colorspace encoding is BT601. + */ + if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444) + temp |= TRANS_MSA_SAMPLING_444 | TRANS_MSA_CLRSP_YCBCR; I915_WRITE(TRANS_MSA_MISC(cpu_transcoder), temp); } @@ -1793,7 +1741,7 @@ void intel_ddi_set_vc_payload_alloc(const struct intel_crtc_state *crtc_state, struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; - uint32_t temp; + u32 temp; temp = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder)); if (state == true) @@ -1811,7 +1759,7 @@ void intel_ddi_enable_transcoder_func(const struct intel_crtc_state *crtc_state) enum pipe pipe = crtc->pipe; enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; enum port port = encoder->port; - uint32_t temp; + u32 temp; /* Enable TRANS_DDI_FUNC_CTL for the pipe to work in HDMI mode */ temp = TRANS_DDI_FUNC_ENABLE; @@ -1872,7 +1820,7 @@ void intel_ddi_enable_transcoder_func(const struct intel_crtc_state *crtc_state) temp |= TRANS_DDI_MODE_SELECT_DVI; if (crtc_state->hdmi_scrambling) - temp |= TRANS_DDI_HDMI_SCRAMBLING_MASK; + temp |= TRANS_DDI_HDMI_SCRAMBLING; if (crtc_state->hdmi_high_tmds_clock_ratio) temp |= TRANS_DDI_HIGH_TMDS_CHAR_RATE; } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) { @@ -1895,7 +1843,7 @@ void intel_ddi_disable_transcoder_func(const struct intel_crtc_state *crtc_state struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; i915_reg_t reg = TRANS_DDI_FUNC_CTL(cpu_transcoder); - uint32_t val = I915_READ(reg); + u32 val = I915_READ(reg); val &= ~(TRANS_DDI_FUNC_ENABLE | TRANS_DDI_PORT_MASK | TRANS_DDI_DP_VC_PAYLOAD_ALLOC); val |= TRANS_DDI_PORT_NONE; @@ -1914,12 +1862,14 @@ int intel_ddi_toggle_hdcp_signalling(struct intel_encoder *intel_encoder, { struct drm_device *dev = intel_encoder->base.dev; struct drm_i915_private *dev_priv = to_i915(dev); + intel_wakeref_t wakeref; enum pipe pipe = 0; int ret = 0; - uint32_t tmp; + u32 tmp; - if (WARN_ON(!intel_display_power_get_if_enabled(dev_priv, - intel_encoder->power_domain))) + wakeref = intel_display_power_get_if_enabled(dev_priv, + intel_encoder->power_domain); + if (WARN_ON(!wakeref)) return -ENXIO; if (WARN_ON(!intel_encoder->get_hw_state(intel_encoder, &pipe))) { @@ -1934,7 +1884,7 @@ int intel_ddi_toggle_hdcp_signalling(struct intel_encoder *intel_encoder, tmp &= ~TRANS_DDI_HDCP_SIGNALLING; I915_WRITE(TRANS_DDI_FUNC_CTL(pipe), tmp); out: - intel_display_power_put(dev_priv, intel_encoder->power_domain); + intel_display_power_put(dev_priv, intel_encoder->power_domain, wakeref); return ret; } @@ -1945,13 +1895,15 @@ bool intel_ddi_connector_get_hw_state(struct intel_connector *intel_connector) struct intel_encoder *encoder = intel_connector->encoder; int type = intel_connector->base.connector_type; enum port port = encoder->port; - enum pipe pipe = 0; enum transcoder cpu_transcoder; - uint32_t tmp; + intel_wakeref_t wakeref; + enum pipe pipe = 0; + u32 tmp; bool ret; - if (!intel_display_power_get_if_enabled(dev_priv, - encoder->power_domain)) + wakeref = intel_display_power_get_if_enabled(dev_priv, + encoder->power_domain); + if (!wakeref) return false; if (!encoder->get_hw_state(encoder, &pipe)) { @@ -1993,29 +1945,31 @@ bool intel_ddi_connector_get_hw_state(struct intel_connector *intel_connector) } out: - intel_display_power_put(dev_priv, encoder->power_domain); + intel_display_power_put(dev_priv, encoder->power_domain, wakeref); return ret; } -bool intel_ddi_get_hw_state(struct intel_encoder *encoder, - enum pipe *pipe) +static void intel_ddi_get_encoder_pipes(struct intel_encoder *encoder, + u8 *pipe_mask, bool *is_dp_mst) { struct drm_device *dev = encoder->base.dev; struct drm_i915_private *dev_priv = to_i915(dev); enum port port = encoder->port; + intel_wakeref_t wakeref; enum pipe p; u32 tmp; - bool ret; + u8 mst_pipe_mask; - if (!intel_display_power_get_if_enabled(dev_priv, - encoder->power_domain)) - return false; + *pipe_mask = 0; + *is_dp_mst = false; - ret = false; + wakeref = intel_display_power_get_if_enabled(dev_priv, + encoder->power_domain); + if (!wakeref) + return; tmp = I915_READ(DDI_BUF_CTL(port)); - if (!(tmp & DDI_BUF_CTL_ENABLE)) goto out; @@ -2023,44 +1977,58 @@ bool intel_ddi_get_hw_state(struct intel_encoder *encoder, tmp = I915_READ(TRANS_DDI_FUNC_CTL(TRANSCODER_EDP)); switch (tmp & TRANS_DDI_EDP_INPUT_MASK) { + default: + MISSING_CASE(tmp & TRANS_DDI_EDP_INPUT_MASK); + /* fallthrough */ case TRANS_DDI_EDP_INPUT_A_ON: case TRANS_DDI_EDP_INPUT_A_ONOFF: - *pipe = PIPE_A; + *pipe_mask = BIT(PIPE_A); break; case TRANS_DDI_EDP_INPUT_B_ONOFF: - *pipe = PIPE_B; + *pipe_mask = BIT(PIPE_B); break; case TRANS_DDI_EDP_INPUT_C_ONOFF: - *pipe = PIPE_C; + *pipe_mask = BIT(PIPE_C); break; } - ret = true; - goto out; } + mst_pipe_mask = 0; for_each_pipe(dev_priv, p) { - enum transcoder cpu_transcoder = (enum transcoder) p; + enum transcoder cpu_transcoder = (enum transcoder)p; tmp = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder)); - if ((tmp & TRANS_DDI_PORT_MASK) == TRANS_DDI_SELECT_PORT(port)) { - if ((tmp & TRANS_DDI_MODE_SELECT_MASK) == - TRANS_DDI_MODE_SELECT_DP_MST) - goto out; + if ((tmp & TRANS_DDI_PORT_MASK) != TRANS_DDI_SELECT_PORT(port)) + continue; - *pipe = p; - ret = true; + if ((tmp & TRANS_DDI_MODE_SELECT_MASK) == + TRANS_DDI_MODE_SELECT_DP_MST) + mst_pipe_mask |= BIT(p); - goto out; - } + *pipe_mask |= BIT(p); } - DRM_DEBUG_KMS("No pipe for ddi port %c found\n", port_name(port)); + if (!*pipe_mask) + DRM_DEBUG_KMS("No pipe for ddi port %c found\n", + port_name(port)); + + if (!mst_pipe_mask && hweight8(*pipe_mask) > 1) { + DRM_DEBUG_KMS("Multiple pipes for non DP-MST port %c (pipe_mask %02x)\n", + port_name(port), *pipe_mask); + *pipe_mask = BIT(ffs(*pipe_mask) - 1); + } + + if (mst_pipe_mask && mst_pipe_mask != *pipe_mask) + DRM_DEBUG_KMS("Conflicting MST and non-MST encoders for port %c (pipe_mask %02x mst_pipe_mask %02x)\n", + port_name(port), *pipe_mask, mst_pipe_mask); + else + *is_dp_mst = mst_pipe_mask; out: - if (ret && IS_GEN9_LP(dev_priv)) { + if (*pipe_mask && IS_GEN9_LP(dev_priv)) { tmp = I915_READ(BXT_PHY_CTL(port)); if ((tmp & (BXT_PHY_CMNLANE_POWERDOWN_ACK | BXT_PHY_LANE_POWERDOWN_ACK | @@ -2069,13 +2037,27 @@ out: "(PHY_CTL %08x)\n", port_name(port), tmp); } - intel_display_power_put(dev_priv, encoder->power_domain); + intel_display_power_put(dev_priv, encoder->power_domain, wakeref); +} - return ret; +bool intel_ddi_get_hw_state(struct intel_encoder *encoder, + enum pipe *pipe) +{ + u8 pipe_mask; + bool is_mst; + + intel_ddi_get_encoder_pipes(encoder, &pipe_mask, &is_mst); + + if (is_mst || !pipe_mask) + return false; + + *pipe = ffs(pipe_mask) - 1; + + return true; } static inline enum intel_display_power_domain -intel_ddi_main_link_aux_domain(struct intel_dp *intel_dp) +intel_ddi_main_link_aux_domain(struct intel_digital_port *dig_port) { /* CNL+ HW requires corresponding AUX IOs to be powered up for PSR with * DC states enabled at the same time, while for driver initiated AUX @@ -2089,15 +2071,15 @@ intel_ddi_main_link_aux_domain(struct intel_dp *intel_dp) * Note that PSR is enabled only on Port A even though this function * returns the correct domain for other ports too. */ - return intel_dp->aux_ch == AUX_CH_A ? POWER_DOMAIN_AUX_IO_A : - intel_dp->aux_power_domain; + return dig_port->aux_ch == AUX_CH_A ? POWER_DOMAIN_AUX_IO_A : + intel_aux_power_domain(dig_port); } -static u64 intel_ddi_get_power_domains(struct intel_encoder *encoder, - struct intel_crtc_state *crtc_state) +static void intel_ddi_get_power_domains(struct intel_encoder *encoder, + struct intel_crtc_state *crtc_state) { + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_digital_port *dig_port; - u64 domains; /* * TODO: Add support for MST encoders. Atm, the following should never @@ -2105,19 +2087,26 @@ static u64 intel_ddi_get_power_domains(struct intel_encoder *encoder, * hook. */ if (WARN_ON(intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST))) - return 0; + return; dig_port = enc_to_dig_port(&encoder->base); - domains = BIT_ULL(dig_port->ddi_io_power_domain); - - /* AUX power is only needed for (e)DP mode, not for HDMI. */ - if (intel_crtc_has_dp_encoder(crtc_state)) { - struct intel_dp *intel_dp = &dig_port->dp; + intel_display_power_get(dev_priv, dig_port->ddi_io_power_domain); - domains |= BIT_ULL(intel_ddi_main_link_aux_domain(intel_dp)); - } + /* + * AUX power is only needed for (e)DP mode, and for HDMI mode on TC + * ports. + */ + if (intel_crtc_has_dp_encoder(crtc_state) || + intel_port_is_tc(dev_priv, encoder->port)) + intel_display_power_get(dev_priv, + intel_ddi_main_link_aux_domain(dig_port)); - return domains; + /* + * VDSC power is needed when DSC is enabled + */ + if (crtc_state->dsc_params.compression_enable) + intel_display_power_get(dev_priv, + intel_dsc_power_domain(crtc_state)); } void intel_ddi_enable_pipe_clock(const struct intel_crtc_state *crtc_state) @@ -2144,7 +2133,7 @@ void intel_ddi_disable_pipe_clock(const struct intel_crtc_state *crtc_state) } static void _skl_ddi_set_iboost(struct drm_i915_private *dev_priv, - enum port port, uint8_t iboost) + enum port port, u8 iboost) { u32 tmp; @@ -2163,7 +2152,7 @@ static void skl_ddi_set_iboost(struct intel_encoder *encoder, struct intel_digital_port *intel_dig_port = enc_to_dig_port(&encoder->base); struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); enum port port = encoder->port; - uint8_t iboost; + u8 iboost; if (type == INTEL_OUTPUT_HDMI) iboost = dev_priv->vbt.ddi_port_info[port].hdmi_boost_level; @@ -2231,13 +2220,14 @@ static void bxt_ddi_vswing_sequence(struct intel_encoder *encoder, u8 intel_ddi_dp_voltage_max(struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); enum port port = encoder->port; int n_entries; if (IS_ICELAKE(dev_priv)) { if (intel_port_is_combophy(dev_priv, port)) icl_get_combo_buf_trans(dev_priv, port, encoder->type, - &n_entries); + intel_dp->link_rate, &n_entries); else n_entries = ARRAY_SIZE(icl_mg_phy_ddi_translations); } else if (IS_CANNONLAKE(dev_priv)) { @@ -2418,14 +2408,15 @@ static void cnl_ddi_vswing_sequence(struct intel_encoder *encoder, } static void icl_ddi_combo_vswing_program(struct drm_i915_private *dev_priv, - u32 level, enum port port, int type) + u32 level, enum port port, int type, + int rate) { - const struct icl_combo_phy_ddi_buf_trans *ddi_translations = NULL; + const struct cnl_ddi_buf_trans *ddi_translations = NULL; u32 n_entries, val; int ln; ddi_translations = icl_get_combo_buf_trans(dev_priv, port, type, - &n_entries); + rate, &n_entries); if (!ddi_translations) return; @@ -2434,34 +2425,23 @@ static void icl_ddi_combo_vswing_program(struct drm_i915_private *dev_priv, level = n_entries - 1; } - /* Set PORT_TX_DW5 Rterm Sel to 110b. */ + /* Set PORT_TX_DW5 */ val = I915_READ(ICL_PORT_TX_DW5_LN0(port)); - val &= ~RTERM_SELECT_MASK; + val &= ~(SCALING_MODE_SEL_MASK | RTERM_SELECT_MASK | + TAP2_DISABLE | TAP3_DISABLE); + val |= SCALING_MODE_SEL(0x2); val |= RTERM_SELECT(0x6); - I915_WRITE(ICL_PORT_TX_DW5_GRP(port), val); - - /* Program PORT_TX_DW5 */ - val = I915_READ(ICL_PORT_TX_DW5_LN0(port)); - /* Set DisableTap2 and DisableTap3 if MIPI DSI - * Clear DisableTap2 and DisableTap3 for all other Ports - */ - if (type == INTEL_OUTPUT_DSI) { - val |= TAP2_DISABLE; - val |= TAP3_DISABLE; - } else { - val &= ~TAP2_DISABLE; - val &= ~TAP3_DISABLE; - } + val |= TAP3_DISABLE; I915_WRITE(ICL_PORT_TX_DW5_GRP(port), val); /* Program PORT_TX_DW2 */ val = I915_READ(ICL_PORT_TX_DW2_LN0(port)); val &= ~(SWING_SEL_LOWER_MASK | SWING_SEL_UPPER_MASK | RCOMP_SCALAR_MASK); - val |= SWING_SEL_UPPER(ddi_translations[level].dw2_swing_select); - val |= SWING_SEL_LOWER(ddi_translations[level].dw2_swing_select); + val |= SWING_SEL_UPPER(ddi_translations[level].dw2_swing_sel); + val |= SWING_SEL_LOWER(ddi_translations[level].dw2_swing_sel); /* Program Rcomp scalar for every table entry */ - val |= RCOMP_SCALAR(ddi_translations[level].dw2_swing_scalar); + val |= RCOMP_SCALAR(0x98); I915_WRITE(ICL_PORT_TX_DW2_GRP(port), val); /* Program PORT_TX_DW4 */ @@ -2470,9 +2450,17 @@ static void icl_ddi_combo_vswing_program(struct drm_i915_private *dev_priv, val = I915_READ(ICL_PORT_TX_DW4_LN(port, ln)); val &= ~(POST_CURSOR_1_MASK | POST_CURSOR_2_MASK | CURSOR_COEFF_MASK); - val |= ddi_translations[level].dw4_scaling; + val |= POST_CURSOR_1(ddi_translations[level].dw4_post_cursor_1); + val |= POST_CURSOR_2(ddi_translations[level].dw4_post_cursor_2); + val |= CURSOR_COEFF(ddi_translations[level].dw4_cursor_coeff); I915_WRITE(ICL_PORT_TX_DW4_LN(port, ln), val); } + + /* Program PORT_TX_DW7 */ + val = I915_READ(ICL_PORT_TX_DW7_LN0(port)); + val &= ~N_SCALAR_MASK; + val |= N_SCALAR(ddi_translations[level].dw7_n_scalar); + I915_WRITE(ICL_PORT_TX_DW7_GRP(port), val); } static void icl_combo_phy_ddi_vswing_sequence(struct intel_encoder *encoder, @@ -2537,7 +2525,7 @@ static void icl_combo_phy_ddi_vswing_sequence(struct intel_encoder *encoder, I915_WRITE(ICL_PORT_TX_DW5_GRP(port), val); /* 5. Program swing and de-emphasis */ - icl_ddi_combo_vswing_program(dev_priv, level, port, type); + icl_ddi_combo_vswing_program(dev_priv, level, port, type, rate); /* 6. Set training enable to trigger update */ val = I915_READ(ICL_PORT_TX_DW5_LN0(port)); @@ -2678,7 +2666,7 @@ static void icl_ddi_vswing_sequence(struct intel_encoder *encoder, icl_mg_phy_ddi_vswing_sequence(encoder, link_clock, level); } -static uint32_t translate_signal_level(int signal_levels) +static u32 translate_signal_level(int signal_levels) { int i; @@ -2693,9 +2681,9 @@ static uint32_t translate_signal_level(int signal_levels) return 0; } -static uint32_t intel_ddi_dp_level(struct intel_dp *intel_dp) +static u32 intel_ddi_dp_level(struct intel_dp *intel_dp) { - uint8_t train_set = intel_dp->train_set[0]; + u8 train_set = intel_dp->train_set[0]; int signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK | DP_TRAIN_PRE_EMPHASIS_MASK); @@ -2720,7 +2708,7 @@ u32 bxt_signal_levels(struct intel_dp *intel_dp) return 0; } -uint32_t ddi_signal_levels(struct intel_dp *intel_dp) +u32 ddi_signal_levels(struct intel_dp *intel_dp) { struct intel_digital_port *dport = dp_to_dig_port(intel_dp); struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev); @@ -2734,8 +2722,8 @@ uint32_t ddi_signal_levels(struct intel_dp *intel_dp) } static inline -uint32_t icl_dpclka_cfgcr0_clk_off(struct drm_i915_private *dev_priv, - enum port port) +u32 icl_dpclka_cfgcr0_clk_off(struct drm_i915_private *dev_priv, + enum port port) { if (intel_port_is_combophy(dev_priv, port)) { return ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(port); @@ -2748,77 +2736,130 @@ uint32_t icl_dpclka_cfgcr0_clk_off(struct drm_i915_private *dev_priv, return 0; } -void icl_map_plls_to_ports(struct drm_crtc *crtc, - struct intel_crtc_state *crtc_state, - struct drm_atomic_state *old_state) +static void icl_map_plls_to_ports(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state) { + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_shared_dpll *pll = crtc_state->shared_dpll; - struct drm_i915_private *dev_priv = to_i915(crtc->dev); - struct drm_connector_state *conn_state; - struct drm_connector *conn; - int i; + enum port port = encoder->port; + u32 val; - for_each_new_connector_in_state(old_state, conn, conn_state, i) { - struct intel_encoder *encoder = - to_intel_encoder(conn_state->best_encoder); - enum port port; - uint32_t val; + mutex_lock(&dev_priv->dpll_lock); - if (conn_state->crtc != crtc) - continue; + val = I915_READ(DPCLKA_CFGCR0_ICL); + WARN_ON((val & icl_dpclka_cfgcr0_clk_off(dev_priv, port)) == 0); - port = encoder->port; - mutex_lock(&dev_priv->dpll_lock); + if (intel_port_is_combophy(dev_priv, port)) { + val &= ~DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port); + val |= DPCLKA_CFGCR0_DDI_CLK_SEL(pll->info->id, port); + I915_WRITE(DPCLKA_CFGCR0_ICL, val); + POSTING_READ(DPCLKA_CFGCR0_ICL); + } - val = I915_READ(DPCLKA_CFGCR0_ICL); - WARN_ON((val & icl_dpclka_cfgcr0_clk_off(dev_priv, port)) == 0); + val &= ~icl_dpclka_cfgcr0_clk_off(dev_priv, port); + I915_WRITE(DPCLKA_CFGCR0_ICL, val); - if (intel_port_is_combophy(dev_priv, port)) { - val &= ~DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port); - val |= DPCLKA_CFGCR0_DDI_CLK_SEL(pll->info->id, port); - I915_WRITE(DPCLKA_CFGCR0_ICL, val); - POSTING_READ(DPCLKA_CFGCR0_ICL); - } + mutex_unlock(&dev_priv->dpll_lock); +} - val &= ~icl_dpclka_cfgcr0_clk_off(dev_priv, port); - I915_WRITE(DPCLKA_CFGCR0_ICL, val); +static void icl_unmap_plls_to_ports(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + enum port port = encoder->port; + u32 val; - mutex_unlock(&dev_priv->dpll_lock); - } + mutex_lock(&dev_priv->dpll_lock); + + val = I915_READ(DPCLKA_CFGCR0_ICL); + val |= icl_dpclka_cfgcr0_clk_off(dev_priv, port); + I915_WRITE(DPCLKA_CFGCR0_ICL, val); + + mutex_unlock(&dev_priv->dpll_lock); } -void icl_unmap_plls_to_ports(struct drm_crtc *crtc, - struct intel_crtc_state *crtc_state, - struct drm_atomic_state *old_state) +void icl_sanitize_encoder_pll_mapping(struct intel_encoder *encoder) { - struct drm_i915_private *dev_priv = to_i915(crtc->dev); - struct drm_connector_state *old_conn_state; - struct drm_connector *conn; - int i; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + u32 val; + enum port port; + u32 port_mask; + bool ddi_clk_needed; - for_each_old_connector_in_state(old_state, conn, old_conn_state, i) { - struct intel_encoder *encoder = - to_intel_encoder(old_conn_state->best_encoder); - enum port port; + /* + * In case of DP MST, we sanitize the primary encoder only, not the + * virtual ones. + */ + if (encoder->type == INTEL_OUTPUT_DP_MST) + return; + + if (!encoder->base.crtc && intel_encoder_is_dp(encoder)) { + u8 pipe_mask; + bool is_mst; + + intel_ddi_get_encoder_pipes(encoder, &pipe_mask, &is_mst); + /* + * In the unlikely case that BIOS enables DP in MST mode, just + * warn since our MST HW readout is incomplete. + */ + if (WARN_ON(is_mst)) + return; + } + + port_mask = BIT(encoder->port); + ddi_clk_needed = encoder->base.crtc; + + if (encoder->type == INTEL_OUTPUT_DSI) { + struct intel_encoder *other_encoder; + + port_mask = intel_dsi_encoder_ports(encoder); + /* + * Sanity check that we haven't incorrectly registered another + * encoder using any of the ports of this DSI encoder. + */ + for_each_intel_encoder(&dev_priv->drm, other_encoder) { + if (other_encoder == encoder) + continue; - if (old_conn_state->crtc != crtc) + if (WARN_ON(port_mask & BIT(other_encoder->port))) + return; + } + /* + * For DSI we keep the ddi clocks gated + * except during enable/disable sequence. + */ + ddi_clk_needed = false; + } + + val = I915_READ(DPCLKA_CFGCR0_ICL); + for_each_port_masked(port, port_mask) { + bool ddi_clk_ungated = !(val & + icl_dpclka_cfgcr0_clk_off(dev_priv, + port)); + + if (ddi_clk_needed == ddi_clk_ungated) continue; - port = encoder->port; - mutex_lock(&dev_priv->dpll_lock); - I915_WRITE(DPCLKA_CFGCR0_ICL, - I915_READ(DPCLKA_CFGCR0_ICL) | - icl_dpclka_cfgcr0_clk_off(dev_priv, port)); - mutex_unlock(&dev_priv->dpll_lock); + /* + * Punt on the case now where clock is gated, but it would + * be needed by the port. Something else is really broken then. + */ + if (WARN_ON(ddi_clk_needed)) + continue; + + DRM_NOTE("Port %c is disabled/in DSI mode with an ungated DDI clock, gate it\n", + port_name(port)); + val |= icl_dpclka_cfgcr0_clk_off(dev_priv, port); + I915_WRITE(DPCLKA_CFGCR0_ICL, val); } } static void intel_ddi_clk_select(struct intel_encoder *encoder, - const struct intel_shared_dpll *pll) + const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); enum port port = encoder->port; - uint32_t val; + u32 val; + const struct intel_shared_dpll *pll = crtc_state->shared_dpll; if (WARN_ON(!pll)) return; @@ -2828,7 +2869,7 @@ static void intel_ddi_clk_select(struct intel_encoder *encoder, if (IS_ICELAKE(dev_priv)) { if (!intel_port_is_combophy(dev_priv, port)) I915_WRITE(DDI_CLK_SEL(port), - icl_pll_to_ddi_pll_sel(encoder, pll)); + icl_pll_to_ddi_clk_sel(encoder, crtc_state)); } else if (IS_CANNONLAKE(dev_priv)) { /* Configure DPCLKA_CFGCR0 to map the DPLL to the DDI. */ val = I915_READ(DPCLKA_CFGCR0); @@ -2881,6 +2922,184 @@ static void intel_ddi_clk_disable(struct intel_encoder *encoder) } } +static void icl_enable_phy_clock_gating(struct intel_digital_port *dig_port) +{ + struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); + enum port port = dig_port->base.port; + enum tc_port tc_port = intel_port_to_tc(dev_priv, port); + i915_reg_t mg_regs[2] = { MG_DP_MODE(port, 0), MG_DP_MODE(port, 1) }; + u32 val; + int i; + + if (tc_port == PORT_TC_NONE) + return; + + for (i = 0; i < ARRAY_SIZE(mg_regs); i++) { + val = I915_READ(mg_regs[i]); + val |= MG_DP_MODE_CFG_TR2PWR_GATING | + MG_DP_MODE_CFG_TRPWR_GATING | + MG_DP_MODE_CFG_CLNPWR_GATING | + MG_DP_MODE_CFG_DIGPWR_GATING | + MG_DP_MODE_CFG_GAONPWR_GATING; + I915_WRITE(mg_regs[i], val); + } + + val = I915_READ(MG_MISC_SUS0(tc_port)); + val |= MG_MISC_SUS0_SUSCLK_DYNCLKGATE_MODE(3) | + MG_MISC_SUS0_CFG_TR2PWR_GATING | + MG_MISC_SUS0_CFG_CL2PWR_GATING | + MG_MISC_SUS0_CFG_GAONPWR_GATING | + MG_MISC_SUS0_CFG_TRPWR_GATING | + MG_MISC_SUS0_CFG_CL1PWR_GATING | + MG_MISC_SUS0_CFG_DGPWR_GATING; + I915_WRITE(MG_MISC_SUS0(tc_port), val); +} + +static void icl_disable_phy_clock_gating(struct intel_digital_port *dig_port) +{ + struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); + enum port port = dig_port->base.port; + enum tc_port tc_port = intel_port_to_tc(dev_priv, port); + i915_reg_t mg_regs[2] = { MG_DP_MODE(port, 0), MG_DP_MODE(port, 1) }; + u32 val; + int i; + + if (tc_port == PORT_TC_NONE) + return; + + for (i = 0; i < ARRAY_SIZE(mg_regs); i++) { + val = I915_READ(mg_regs[i]); + val &= ~(MG_DP_MODE_CFG_TR2PWR_GATING | + MG_DP_MODE_CFG_TRPWR_GATING | + MG_DP_MODE_CFG_CLNPWR_GATING | + MG_DP_MODE_CFG_DIGPWR_GATING | + MG_DP_MODE_CFG_GAONPWR_GATING); + I915_WRITE(mg_regs[i], val); + } + + val = I915_READ(MG_MISC_SUS0(tc_port)); + val &= ~(MG_MISC_SUS0_SUSCLK_DYNCLKGATE_MODE_MASK | + MG_MISC_SUS0_CFG_TR2PWR_GATING | + MG_MISC_SUS0_CFG_CL2PWR_GATING | + MG_MISC_SUS0_CFG_GAONPWR_GATING | + MG_MISC_SUS0_CFG_TRPWR_GATING | + MG_MISC_SUS0_CFG_CL1PWR_GATING | + MG_MISC_SUS0_CFG_DGPWR_GATING); + I915_WRITE(MG_MISC_SUS0(tc_port), val); +} + +static void icl_program_mg_dp_mode(struct intel_digital_port *intel_dig_port) +{ + struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev); + enum port port = intel_dig_port->base.port; + enum tc_port tc_port = intel_port_to_tc(dev_priv, port); + u32 ln0, ln1, lane_info; + + if (tc_port == PORT_TC_NONE || intel_dig_port->tc_type == TC_PORT_TBT) + return; + + ln0 = I915_READ(MG_DP_MODE(port, 0)); + ln1 = I915_READ(MG_DP_MODE(port, 1)); + + switch (intel_dig_port->tc_type) { + case TC_PORT_TYPEC: + ln0 &= ~(MG_DP_MODE_CFG_DP_X1_MODE | MG_DP_MODE_CFG_DP_X2_MODE); + ln1 &= ~(MG_DP_MODE_CFG_DP_X1_MODE | MG_DP_MODE_CFG_DP_X2_MODE); + + lane_info = (I915_READ(PORT_TX_DFLEXDPSP) & + DP_LANE_ASSIGNMENT_MASK(tc_port)) >> + DP_LANE_ASSIGNMENT_SHIFT(tc_port); + + switch (lane_info) { + case 0x1: + case 0x4: + break; + case 0x2: + ln0 |= MG_DP_MODE_CFG_DP_X1_MODE; + break; + case 0x3: + ln0 |= MG_DP_MODE_CFG_DP_X1_MODE | + MG_DP_MODE_CFG_DP_X2_MODE; + break; + case 0x8: + ln1 |= MG_DP_MODE_CFG_DP_X1_MODE; + break; + case 0xC: + ln1 |= MG_DP_MODE_CFG_DP_X1_MODE | + MG_DP_MODE_CFG_DP_X2_MODE; + break; + case 0xF: + ln0 |= MG_DP_MODE_CFG_DP_X1_MODE | + MG_DP_MODE_CFG_DP_X2_MODE; + ln1 |= MG_DP_MODE_CFG_DP_X1_MODE | + MG_DP_MODE_CFG_DP_X2_MODE; + break; + default: + MISSING_CASE(lane_info); + } + break; + + case TC_PORT_LEGACY: + ln0 |= MG_DP_MODE_CFG_DP_X1_MODE | MG_DP_MODE_CFG_DP_X2_MODE; + ln1 |= MG_DP_MODE_CFG_DP_X1_MODE | MG_DP_MODE_CFG_DP_X2_MODE; + break; + + default: + MISSING_CASE(intel_dig_port->tc_type); + return; + } + + I915_WRITE(MG_DP_MODE(port, 0), ln0); + I915_WRITE(MG_DP_MODE(port, 1), ln1); +} + +static void intel_dp_sink_set_fec_ready(struct intel_dp *intel_dp, + const struct intel_crtc_state *crtc_state) +{ + if (!crtc_state->fec_enable) + return; + + if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_FEC_CONFIGURATION, DP_FEC_READY) <= 0) + DRM_DEBUG_KMS("Failed to set FEC_READY in the sink\n"); +} + +static void intel_ddi_enable_fec(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + enum port port = encoder->port; + u32 val; + + if (!crtc_state->fec_enable) + return; + + val = I915_READ(DP_TP_CTL(port)); + val |= DP_TP_CTL_FEC_ENABLE; + I915_WRITE(DP_TP_CTL(port), val); + + if (intel_wait_for_register(dev_priv, DP_TP_STATUS(port), + DP_TP_STATUS_FEC_ENABLE_LIVE, + DP_TP_STATUS_FEC_ENABLE_LIVE, + 1)) + DRM_ERROR("Timed out waiting for FEC Enable Status\n"); +} + +static void intel_ddi_disable_fec_state(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + enum port port = encoder->port; + u32 val; + + if (!crtc_state->fec_enable) + return; + + val = I915_READ(DP_TP_CTL(port)); + val &= ~DP_TP_CTL_FEC_ENABLE; + I915_WRITE(DP_TP_CTL(port), val); + POSTING_READ(DP_TP_CTL(port)); +} + static void intel_ddi_pre_enable_dp(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) @@ -2894,19 +3113,16 @@ static void intel_ddi_pre_enable_dp(struct intel_encoder *encoder, WARN_ON(is_mst && (port == PORT_A || port == PORT_E)); - intel_display_power_get(dev_priv, - intel_ddi_main_link_aux_domain(intel_dp)); - intel_dp_set_link_params(intel_dp, crtc_state->port_clock, crtc_state->lane_count, is_mst); intel_edp_panel_on(intel_dp); - intel_ddi_clk_select(encoder, crtc_state->shared_dpll); + intel_ddi_clk_select(encoder, crtc_state); intel_display_power_get(dev_priv, dig_port->ddi_io_power_domain); - icl_program_mg_dp_mode(intel_dp); + icl_program_mg_dp_mode(dig_port); icl_disable_phy_clock_gating(dig_port); if (IS_ICELAKE(dev_priv)) @@ -2922,14 +3138,21 @@ static void intel_ddi_pre_enable_dp(struct intel_encoder *encoder, intel_ddi_init_dp_buf_reg(encoder); if (!is_mst) intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON); + intel_dp_sink_set_decompression_state(intel_dp, crtc_state, + true); + intel_dp_sink_set_fec_ready(intel_dp, crtc_state); intel_dp_start_link_train(intel_dp); if (port != PORT_A || INTEL_GEN(dev_priv) >= 9) intel_dp_stop_link_train(intel_dp); + intel_ddi_enable_fec(encoder, crtc_state); + icl_enable_phy_clock_gating(dig_port); if (!is_mst) intel_ddi_enable_pipe_clock(crtc_state); + + intel_dsc_enable(encoder, crtc_state); } static void intel_ddi_pre_enable_hdmi(struct intel_encoder *encoder, @@ -2944,10 +3167,13 @@ static void intel_ddi_pre_enable_hdmi(struct intel_encoder *encoder, struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base); intel_dp_dual_mode_set_tmds_output(intel_hdmi, true); - intel_ddi_clk_select(encoder, crtc_state->shared_dpll); + intel_ddi_clk_select(encoder, crtc_state); intel_display_power_get(dev_priv, dig_port->ddi_io_power_domain); + icl_program_mg_dp_mode(dig_port); + icl_disable_phy_clock_gating(dig_port); + if (IS_ICELAKE(dev_priv)) icl_ddi_vswing_sequence(encoder, crtc_state->port_clock, level, INTEL_OUTPUT_HDMI); @@ -2958,12 +3184,14 @@ static void intel_ddi_pre_enable_hdmi(struct intel_encoder *encoder, else intel_prepare_hdmi_ddi_buffers(encoder, level); + icl_enable_phy_clock_gating(dig_port); + if (IS_GEN9_BC(dev_priv)) skl_ddi_set_iboost(encoder, level, INTEL_OUTPUT_HDMI); intel_ddi_enable_pipe_clock(crtc_state); - intel_dig_port->set_infoframes(&encoder->base, + intel_dig_port->set_infoframes(encoder, crtc_state->has_infoframe, crtc_state, conn_state); } @@ -2991,15 +3219,31 @@ static void intel_ddi_pre_enable(struct intel_encoder *encoder, WARN_ON(crtc_state->has_pch_encoder); + if (INTEL_GEN(dev_priv) >= 11) + icl_map_plls_to_ports(encoder, crtc_state); + intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); - if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) { intel_ddi_pre_enable_hdmi(encoder, crtc_state, conn_state); - else + } else { + struct intel_lspcon *lspcon = + enc_to_intel_lspcon(&encoder->base); + intel_ddi_pre_enable_dp(encoder, crtc_state, conn_state); + if (lspcon->active) { + struct intel_digital_port *dig_port = + enc_to_dig_port(&encoder->base); + + dig_port->set_infoframes(encoder, + crtc_state->has_infoframe, + crtc_state, conn_state); + } + } } -static void intel_disable_ddi_buf(struct intel_encoder *encoder) +static void intel_disable_ddi_buf(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); enum port port = encoder->port; @@ -3018,6 +3262,9 @@ static void intel_disable_ddi_buf(struct intel_encoder *encoder) val |= DP_TP_CTL_LINK_TRAIN_PAT1; I915_WRITE(DP_TP_CTL(port), val); + /* Disable FEC in DP Sink */ + intel_ddi_disable_fec_state(encoder, crtc_state); + if (wait) intel_wait_ddi_buf_idle(dev_priv, port); } @@ -3041,17 +3288,15 @@ static void intel_ddi_post_disable_dp(struct intel_encoder *encoder, intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF); } - intel_disable_ddi_buf(encoder); + intel_disable_ddi_buf(encoder, old_crtc_state); intel_edp_panel_vdd_on(intel_dp); intel_edp_panel_off(intel_dp); - intel_display_power_put(dev_priv, dig_port->ddi_io_power_domain); + intel_display_power_put_unchecked(dev_priv, + dig_port->ddi_io_power_domain); intel_ddi_clk_disable(encoder); - - intel_display_power_put(dev_priv, - intel_ddi_main_link_aux_domain(intel_dp)); } static void intel_ddi_post_disable_hdmi(struct intel_encoder *encoder, @@ -3062,14 +3307,15 @@ static void intel_ddi_post_disable_hdmi(struct intel_encoder *encoder, struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base); struct intel_hdmi *intel_hdmi = &dig_port->hdmi; - dig_port->set_infoframes(&encoder->base, false, + dig_port->set_infoframes(encoder, false, old_crtc_state, old_conn_state); intel_ddi_disable_pipe_clock(old_crtc_state); - intel_disable_ddi_buf(encoder); + intel_disable_ddi_buf(encoder, old_crtc_state); - intel_display_power_put(dev_priv, dig_port->ddi_io_power_domain); + intel_display_power_put_unchecked(dev_priv, + dig_port->ddi_io_power_domain); intel_ddi_clk_disable(encoder); @@ -3080,6 +3326,8 @@ static void intel_ddi_post_disable(struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + /* * When called from DP MST code: * - old_conn_state will be NULL @@ -3099,6 +3347,9 @@ static void intel_ddi_post_disable(struct intel_encoder *encoder, else intel_ddi_post_disable_dp(encoder, old_crtc_state, old_conn_state); + + if (INTEL_GEN(dev_priv) >= 11) + icl_unmap_plls_to_ports(encoder); } void intel_ddi_fdi_post_disable(struct intel_encoder *encoder, @@ -3106,7 +3357,7 @@ void intel_ddi_fdi_post_disable(struct intel_encoder *encoder, const struct drm_connector_state *old_conn_state) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - uint32_t val; + u32 val; /* * Bspec lists this as both step 13 (before DDI_BUF_CTL disable) @@ -3118,7 +3369,7 @@ void intel_ddi_fdi_post_disable(struct intel_encoder *encoder, val &= ~FDI_RX_ENABLE; I915_WRITE(FDI_RX_CTL(PIPE_A), val); - intel_disable_ddi_buf(encoder); + intel_disable_ddi_buf(encoder, old_crtc_state); intel_ddi_clk_disable(encoder); val = I915_READ(FDI_RX_MISC(PIPE_A)); @@ -3154,6 +3405,26 @@ static void intel_enable_ddi_dp(struct intel_encoder *encoder, intel_audio_codec_enable(encoder, crtc_state, conn_state); } +static i915_reg_t +gen9_chicken_trans_reg_by_port(struct drm_i915_private *dev_priv, + enum port port) +{ + static const i915_reg_t regs[] = { + [PORT_A] = CHICKEN_TRANS_EDP, + [PORT_B] = CHICKEN_TRANS_A, + [PORT_C] = CHICKEN_TRANS_B, + [PORT_D] = CHICKEN_TRANS_C, + [PORT_E] = CHICKEN_TRANS_A, + }; + + WARN_ON(INTEL_GEN(dev_priv) < 9); + + if (WARN_ON(port < PORT_A || port > PORT_E)) + port = PORT_A; + + return regs[port]; +} + static void intel_enable_ddi_hdmi(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) @@ -3177,17 +3448,10 @@ static void intel_enable_ddi_hdmi(struct intel_encoder *encoder, * the bits affect a specific DDI port rather than * a specific transcoder. */ - static const enum transcoder port_to_transcoder[] = { - [PORT_A] = TRANSCODER_EDP, - [PORT_B] = TRANSCODER_A, - [PORT_C] = TRANSCODER_B, - [PORT_D] = TRANSCODER_C, - [PORT_E] = TRANSCODER_A, - }; - enum transcoder transcoder = port_to_transcoder[port]; + i915_reg_t reg = gen9_chicken_trans_reg_by_port(dev_priv, port); u32 val; - val = I915_READ(CHICKEN_TRANS(transcoder)); + val = I915_READ(reg); if (port == PORT_E) val |= DDIE_TRAINING_OVERRIDE_ENABLE | @@ -3196,8 +3460,8 @@ static void intel_enable_ddi_hdmi(struct intel_encoder *encoder, val |= DDI_TRAINING_OVERRIDE_ENABLE | DDI_TRAINING_OVERRIDE_VALUE; - I915_WRITE(CHICKEN_TRANS(transcoder), val); - POSTING_READ(CHICKEN_TRANS(transcoder)); + I915_WRITE(reg, val); + POSTING_READ(reg); udelay(1); @@ -3208,7 +3472,7 @@ static void intel_enable_ddi_hdmi(struct intel_encoder *encoder, val &= ~(DDI_TRAINING_OVERRIDE_ENABLE | DDI_TRAINING_OVERRIDE_VALUE); - I915_WRITE(CHICKEN_TRANS(transcoder), val); + I915_WRITE(reg, val); } /* In HDMI/DVI mode, the port width, and swing/emphasis values @@ -3252,6 +3516,9 @@ static void intel_disable_ddi_dp(struct intel_encoder *encoder, intel_edp_drrs_disable(intel_dp, old_crtc_state); intel_psr_disable(intel_dp, old_crtc_state); intel_edp_backlight_off(old_conn_state); + /* Disable the decompression in DP Sink */ + intel_dp_sink_set_decompression_state(intel_dp, old_crtc_state, + false); } static void intel_disable_ddi_hdmi(struct intel_encoder *encoder, @@ -3282,13 +3549,103 @@ static void intel_disable_ddi(struct intel_encoder *encoder, intel_disable_ddi_dp(encoder, old_crtc_state, old_conn_state); } -static void bxt_ddi_pre_pll_enable(struct intel_encoder *encoder, - const struct intel_crtc_state *pipe_config, - const struct drm_connector_state *conn_state) +static void intel_ddi_update_pipe_dp(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) +{ + struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); + + intel_psr_enable(intel_dp, crtc_state); + intel_edp_drrs_enable(intel_dp, crtc_state); + + intel_panel_update_backlight(encoder, crtc_state, conn_state); +} + +static void intel_ddi_update_pipe(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) +{ + if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) + intel_ddi_update_pipe_dp(encoder, crtc_state, conn_state); + + if (conn_state->content_protection == + DRM_MODE_CONTENT_PROTECTION_DESIRED) + intel_hdcp_enable(to_intel_connector(conn_state->connector)); + else if (conn_state->content_protection == + DRM_MODE_CONTENT_PROTECTION_UNDESIRED) + intel_hdcp_disable(to_intel_connector(conn_state->connector)); +} + +static void intel_ddi_set_fia_lane_count(struct intel_encoder *encoder, + const struct intel_crtc_state *pipe_config, + enum port port) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base); + enum tc_port tc_port = intel_port_to_tc(dev_priv, port); + u32 val = I915_READ(PORT_TX_DFLEXDPMLE1); + bool lane_reversal = dig_port->saved_port_bits & DDI_BUF_PORT_REVERSAL; + + val &= ~DFLEXDPMLE1_DPMLETC_MASK(tc_port); + switch (pipe_config->lane_count) { + case 1: + val |= (lane_reversal) ? DFLEXDPMLE1_DPMLETC_ML3(tc_port) : + DFLEXDPMLE1_DPMLETC_ML0(tc_port); + break; + case 2: + val |= (lane_reversal) ? DFLEXDPMLE1_DPMLETC_ML3_2(tc_port) : + DFLEXDPMLE1_DPMLETC_ML1_0(tc_port); + break; + case 4: + val |= DFLEXDPMLE1_DPMLETC_ML3_0(tc_port); + break; + default: + MISSING_CASE(pipe_config->lane_count); + } + I915_WRITE(PORT_TX_DFLEXDPMLE1, val); +} + +static void +intel_ddi_pre_pll_enable(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base); + enum port port = encoder->port; + + if (intel_crtc_has_dp_encoder(crtc_state) || + intel_port_is_tc(dev_priv, encoder->port)) + intel_display_power_get(dev_priv, + intel_ddi_main_link_aux_domain(dig_port)); + + if (IS_GEN9_LP(dev_priv)) + bxt_ddi_phy_set_lane_optim_mask(encoder, + crtc_state->lane_lat_optim_mask); + + /* + * Program the lane count for static/dynamic connections on Type-C ports. + * Skip this step for TBT. + */ + if (dig_port->tc_type == TC_PORT_UNKNOWN || + dig_port->tc_type == TC_PORT_TBT) + return; + + intel_ddi_set_fia_lane_count(encoder, crtc_state, port); +} + +static void +intel_ddi_post_pll_disable(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) { - uint8_t mask = pipe_config->lane_lat_optim_mask; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base); - bxt_ddi_phy_set_lane_optim_mask(encoder, mask); + if (intel_crtc_has_dp_encoder(crtc_state) || + intel_port_is_tc(dev_priv, encoder->port)) + intel_display_power_put_unchecked(dev_priv, + intel_ddi_main_link_aux_domain(dig_port)); } void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp) @@ -3297,7 +3654,7 @@ void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp) struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev); enum port port = intel_dig_port->base.port; - uint32_t val; + u32 val; bool wait = false; if (I915_READ(DP_TP_CTL(port)) & DP_TP_CTL_ENABLE) { @@ -3353,10 +3710,10 @@ static bool intel_ddi_is_audio_enabled(struct drm_i915_private *dev_priv, void intel_ddi_compute_min_voltage_level(struct drm_i915_private *dev_priv, struct intel_crtc_state *crtc_state) { - if (IS_CANNONLAKE(dev_priv) && crtc_state->port_clock > 594000) - crtc_state->min_voltage_level = 2; - else if (IS_ICELAKE(dev_priv) && crtc_state->port_clock > 594000) + if (IS_ICELAKE(dev_priv) && crtc_state->port_clock > 594000) crtc_state->min_voltage_level = 1; + else if (IS_CANNONLAKE(dev_priv) && crtc_state->port_clock > 594000) + crtc_state->min_voltage_level = 2; } void intel_ddi_get_config(struct intel_encoder *encoder, @@ -3406,11 +3763,10 @@ void intel_ddi_get_config(struct intel_encoder *encoder, pipe_config->has_hdmi_sink = true; intel_dig_port = enc_to_dig_port(&encoder->base); - if (intel_dig_port->infoframe_enabled(&encoder->base, pipe_config)) + if (intel_dig_port->infoframe_enabled(encoder, pipe_config)) pipe_config->has_infoframe = true; - if ((temp & TRANS_DDI_HDMI_SCRAMBLING_MASK) == - TRANS_DDI_HDMI_SCRAMBLING_MASK) + if (temp & TRANS_DDI_HDMI_SCRAMBLING) pipe_config->hdmi_scrambling = true; if (temp & TRANS_DDI_HIGH_TMDS_CHAR_RATE) pipe_config->hdmi_high_tmds_clock_ratio = true; @@ -3491,9 +3847,9 @@ intel_ddi_compute_output_type(struct intel_encoder *encoder, } } -static bool intel_ddi_compute_config(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config, - struct drm_connector_state *conn_state) +static int intel_ddi_compute_config(struct intel_encoder *encoder, + struct intel_crtc_state *pipe_config, + struct drm_connector_state *conn_state) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); enum port port = encoder->port; @@ -3506,20 +3862,63 @@ static bool intel_ddi_compute_config(struct intel_encoder *encoder, ret = intel_hdmi_compute_config(encoder, pipe_config, conn_state); else ret = intel_dp_compute_config(encoder, pipe_config, conn_state); + if (ret) + return ret; - if (IS_GEN9_LP(dev_priv) && ret) + if (IS_GEN9_LP(dev_priv)) pipe_config->lane_lat_optim_mask = bxt_ddi_phy_calc_lane_lat_optim_mask(pipe_config->lane_count); intel_ddi_compute_min_voltage_level(dev_priv, pipe_config); - return ret; + return 0; } +static void intel_ddi_encoder_suspend(struct intel_encoder *encoder) +{ + struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base); + struct drm_i915_private *i915 = to_i915(encoder->base.dev); + + intel_dp_encoder_suspend(encoder); + + /* + * TODO: disconnect also from USB DP alternate mode once we have a + * way to handle the modeset restore in that mode during resume + * even if the sink has disappeared while being suspended. + */ + if (dig_port->tc_legacy_port) + icl_tc_phy_disconnect(i915, dig_port); +} + +static void intel_ddi_encoder_reset(struct drm_encoder *drm_encoder) +{ + struct intel_digital_port *dig_port = enc_to_dig_port(drm_encoder); + struct drm_i915_private *i915 = to_i915(drm_encoder->dev); + + if (intel_port_is_tc(i915, dig_port->base.port)) + intel_digital_port_connected(&dig_port->base); + + intel_dp_encoder_reset(drm_encoder); +} + +static void intel_ddi_encoder_destroy(struct drm_encoder *encoder) +{ + struct intel_digital_port *dig_port = enc_to_dig_port(encoder); + struct drm_i915_private *i915 = to_i915(encoder->dev); + + intel_dp_encoder_flush_work(encoder); + + if (intel_port_is_tc(i915, dig_port->base.port)) + icl_tc_phy_disconnect(i915, dig_port); + + drm_encoder_cleanup(encoder); + kfree(dig_port); +} + static const struct drm_encoder_funcs intel_ddi_funcs = { - .reset = intel_dp_encoder_reset, - .destroy = intel_dp_encoder_destroy, + .reset = intel_ddi_encoder_reset, + .destroy = intel_ddi_encoder_destroy, }; static struct intel_connector * @@ -3571,12 +3970,7 @@ static int modeset_pipe(struct drm_crtc *crtc, goto out; ret = drm_atomic_commit(state); - if (ret) - goto out; - - return 0; - - out: +out: drm_atomic_state_put(state); return ret; @@ -3763,15 +4157,16 @@ intel_ddi_max_lanes(struct intel_digital_port *intel_dport) void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port) { + struct ddi_vbt_port_info *port_info = + &dev_priv->vbt.ddi_port_info[port]; struct intel_digital_port *intel_dig_port; struct intel_encoder *intel_encoder; struct drm_encoder *encoder; bool init_hdmi, init_dp, init_lspcon = false; + enum pipe pipe; - - init_hdmi = (dev_priv->vbt.ddi_port_info[port].supports_dvi || - dev_priv->vbt.ddi_port_info[port].supports_hdmi); - init_dp = dev_priv->vbt.ddi_port_info[port].supports_dp; + init_hdmi = port_info->supports_dvi || port_info->supports_hdmi; + init_dp = port_info->supports_dp; if (intel_bios_is_lspcon_present(dev_priv, port)) { /* @@ -3805,20 +4200,22 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port) intel_encoder->compute_output_type = intel_ddi_compute_output_type; intel_encoder->compute_config = intel_ddi_compute_config; intel_encoder->enable = intel_enable_ddi; - if (IS_GEN9_LP(dev_priv)) - intel_encoder->pre_pll_enable = bxt_ddi_pre_pll_enable; + intel_encoder->pre_pll_enable = intel_ddi_pre_pll_enable; + intel_encoder->post_pll_disable = intel_ddi_post_pll_disable; intel_encoder->pre_enable = intel_ddi_pre_enable; intel_encoder->disable = intel_disable_ddi; intel_encoder->post_disable = intel_ddi_post_disable; + intel_encoder->update_pipe = intel_ddi_update_pipe; intel_encoder->get_hw_state = intel_ddi_get_hw_state; intel_encoder->get_config = intel_ddi_get_config; - intel_encoder->suspend = intel_dp_encoder_suspend; + intel_encoder->suspend = intel_ddi_encoder_suspend; intel_encoder->get_power_domains = intel_ddi_get_power_domains; intel_encoder->type = INTEL_OUTPUT_DDI; intel_encoder->power_domain = intel_port_to_power_domain(port); intel_encoder->port = port; - intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2); intel_encoder->cloneable = 0; + for_each_pipe(dev_priv, pipe) + intel_encoder->crtc_mask |= BIT(pipe); if (INTEL_GEN(dev_priv) >= 11) intel_dig_port->saved_port_bits = I915_READ(DDI_BUF_CTL(port)) & @@ -3828,6 +4225,11 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port) (DDI_BUF_PORT_REVERSAL | DDI_A_4_LANES); intel_dig_port->dp.output_reg = INVALID_MMIO_REG; intel_dig_port->max_lanes = intel_ddi_max_lanes(intel_dig_port); + intel_dig_port->aux_ch = intel_bios_port_aux_ch(dev_priv, port); + + intel_dig_port->tc_legacy_port = intel_port_is_tc(dev_priv, port) && + !port_info->supports_typec_usb && + !port_info->supports_tbt; switch (port) { case PORT_A: @@ -3858,8 +4260,6 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port) MISSING_CASE(port); } - intel_infoframe_init(intel_dig_port); - if (init_dp) { if (!intel_ddi_init_dp_connector(intel_dig_port)) goto err; @@ -3888,6 +4288,11 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port) port_name(port)); } + intel_infoframe_init(intel_dig_port); + + if (intel_port_is_tc(dev_priv, port)) + intel_digital_port_connected(intel_encoder); + return; err: diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c index 01fa98299bae..855a5074ad77 100644 --- a/drivers/gpu/drm/i915/intel_device_info.c +++ b/drivers/gpu/drm/i915/intel_device_info.c @@ -77,6 +77,10 @@ void intel_device_info_dump_flags(const struct intel_device_info *info, #define PRINT_FLAG(name) drm_printf(p, "%s: %s\n", #name, yesno(info->name)); DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG); #undef PRINT_FLAG + +#define PRINT_FLAG(name) drm_printf(p, "%s: %s\n", #name, yesno(info->display.name)); + DEV_INFO_DISPLAY_FOR_EACH_FLAG(PRINT_FLAG); +#undef PRINT_FLAG } static void sseu_dump(const struct sseu_dev_info *sseu, struct drm_printer *p) @@ -100,7 +104,7 @@ static void sseu_dump(const struct sseu_dev_info *sseu, struct drm_printer *p) drm_printf(p, "has EU power gating: %s\n", yesno(sseu->has_eu_pg)); } -void intel_device_info_dump_runtime(const struct intel_device_info *info, +void intel_device_info_dump_runtime(const struct intel_runtime_info *info, struct drm_printer *p) { sseu_dump(&info->sseu, p); @@ -109,21 +113,6 @@ void intel_device_info_dump_runtime(const struct intel_device_info *info, info->cs_timestamp_frequency_khz); } -void intel_device_info_dump(const struct intel_device_info *info, - struct drm_printer *p) -{ - struct drm_i915_private *dev_priv = - container_of(info, struct drm_i915_private, info); - - drm_printf(p, "pciid=0x%04x rev=0x%02x platform=%s gen=%i\n", - INTEL_DEVID(dev_priv), - INTEL_REVID(dev_priv), - intel_platform_name(info->platform), - info->gen); - - intel_device_info_dump_flags(info, p); -} - void intel_device_info_dump_topology(const struct sseu_dev_info *sseu, struct drm_printer *p) { @@ -160,7 +149,7 @@ static u16 compute_eu_total(const struct sseu_dev_info *sseu) static void gen11_sseu_info_init(struct drm_i915_private *dev_priv) { - struct sseu_dev_info *sseu = &mkwrite_device_info(dev_priv)->sseu; + struct sseu_dev_info *sseu = &RUNTIME_INFO(dev_priv)->sseu; u8 s_en; u32 ss_en, ss_en_mask; u8 eu_en; @@ -199,7 +188,7 @@ static void gen11_sseu_info_init(struct drm_i915_private *dev_priv) static void gen10_sseu_info_init(struct drm_i915_private *dev_priv) { - struct sseu_dev_info *sseu = &mkwrite_device_info(dev_priv)->sseu; + struct sseu_dev_info *sseu = &RUNTIME_INFO(dev_priv)->sseu; const u32 fuse2 = I915_READ(GEN8_FUSE2); int s, ss; const int eu_mask = 0xff; @@ -276,7 +265,7 @@ static void gen10_sseu_info_init(struct drm_i915_private *dev_priv) static void cherryview_sseu_info_init(struct drm_i915_private *dev_priv) { - struct sseu_dev_info *sseu = &mkwrite_device_info(dev_priv)->sseu; + struct sseu_dev_info *sseu = &RUNTIME_INFO(dev_priv)->sseu; u32 fuse; fuse = I915_READ(CHV_FUSE_GT); @@ -330,7 +319,7 @@ static void cherryview_sseu_info_init(struct drm_i915_private *dev_priv) static void gen9_sseu_info_init(struct drm_i915_private *dev_priv) { struct intel_device_info *info = mkwrite_device_info(dev_priv); - struct sseu_dev_info *sseu = &info->sseu; + struct sseu_dev_info *sseu = &RUNTIME_INFO(dev_priv)->sseu; int s, ss; u32 fuse2, eu_disable, subslice_mask; const u8 eu_mask = 0xff; @@ -433,7 +422,7 @@ static void gen9_sseu_info_init(struct drm_i915_private *dev_priv) static void broadwell_sseu_info_init(struct drm_i915_private *dev_priv) { - struct sseu_dev_info *sseu = &mkwrite_device_info(dev_priv)->sseu; + struct sseu_dev_info *sseu = &RUNTIME_INFO(dev_priv)->sseu; int s, ss; u32 fuse2, subslice_mask, eu_disable[3]; /* s_max */ @@ -515,8 +504,7 @@ static void broadwell_sseu_info_init(struct drm_i915_private *dev_priv) static void haswell_sseu_info_init(struct drm_i915_private *dev_priv) { - struct intel_device_info *info = mkwrite_device_info(dev_priv); - struct sseu_dev_info *sseu = &info->sseu; + struct sseu_dev_info *sseu = &RUNTIME_INFO(dev_priv)->sseu; u32 fuse1; int s, ss; @@ -524,9 +512,9 @@ static void haswell_sseu_info_init(struct drm_i915_private *dev_priv) * There isn't a register to tell us how many slices/subslices. We * work off the PCI-ids here. */ - switch (info->gt) { + switch (INTEL_INFO(dev_priv)->gt) { default: - MISSING_CASE(info->gt); + MISSING_CASE(INTEL_INFO(dev_priv)->gt); /* fall through */ case 1: sseu->slice_mask = BIT(0); @@ -721,7 +709,7 @@ static u32 read_timestamp_frequency(struct drm_i915_private *dev_priv) /** * intel_device_info_runtime_init - initialize runtime info - * @info: intel device info struct + * @dev_priv: the i915 device * * Determine various intel_device_info fields at runtime. * @@ -735,52 +723,55 @@ static u32 read_timestamp_frequency(struct drm_i915_private *dev_priv) * - after the PCH has been detected, * - before the first usage of the fields it can tweak. */ -void intel_device_info_runtime_init(struct intel_device_info *info) +void intel_device_info_runtime_init(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = - container_of(info, struct drm_i915_private, info); + struct intel_device_info *info = mkwrite_device_info(dev_priv); + struct intel_runtime_info *runtime = RUNTIME_INFO(dev_priv); enum pipe pipe; if (INTEL_GEN(dev_priv) >= 10) { for_each_pipe(dev_priv, pipe) - info->num_scalers[pipe] = 2; - } else if (INTEL_GEN(dev_priv) == 9) { - info->num_scalers[PIPE_A] = 2; - info->num_scalers[PIPE_B] = 2; - info->num_scalers[PIPE_C] = 1; + runtime->num_scalers[pipe] = 2; + } else if (IS_GEN(dev_priv, 9)) { + runtime->num_scalers[PIPE_A] = 2; + runtime->num_scalers[PIPE_B] = 2; + runtime->num_scalers[PIPE_C] = 1; } - BUILD_BUG_ON(I915_NUM_ENGINES > - sizeof(intel_ring_mask_t) * BITS_PER_BYTE); + BUILD_BUG_ON(I915_NUM_ENGINES > BITS_PER_TYPE(intel_ring_mask_t)); - /* - * Skylake and Broxton currently don't expose the topmost plane as its - * use is exclusive with the legacy cursor and we only want to expose - * one of those, not both. Until we can safely expose the topmost plane - * as a DRM_PLANE_TYPE_CURSOR with all the features exposed/supported, - * we don't expose the topmost plane at all to prevent ABI breakage - * down the line. - */ - if (IS_GEN10(dev_priv) || IS_GEMINILAKE(dev_priv)) + if (IS_GEN(dev_priv, 11)) + for_each_pipe(dev_priv, pipe) + runtime->num_sprites[pipe] = 6; + else if (IS_GEN(dev_priv, 10) || IS_GEMINILAKE(dev_priv)) for_each_pipe(dev_priv, pipe) - info->num_sprites[pipe] = 3; + runtime->num_sprites[pipe] = 3; else if (IS_BROXTON(dev_priv)) { - info->num_sprites[PIPE_A] = 2; - info->num_sprites[PIPE_B] = 2; - info->num_sprites[PIPE_C] = 1; + /* + * Skylake and Broxton currently don't expose the topmost plane as its + * use is exclusive with the legacy cursor and we only want to expose + * one of those, not both. Until we can safely expose the topmost plane + * as a DRM_PLANE_TYPE_CURSOR with all the features exposed/supported, + * we don't expose the topmost plane at all to prevent ABI breakage + * down the line. + */ + + runtime->num_sprites[PIPE_A] = 2; + runtime->num_sprites[PIPE_B] = 2; + runtime->num_sprites[PIPE_C] = 1; } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { for_each_pipe(dev_priv, pipe) - info->num_sprites[pipe] = 2; + runtime->num_sprites[pipe] = 2; } else if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv)) { for_each_pipe(dev_priv, pipe) - info->num_sprites[pipe] = 1; + runtime->num_sprites[pipe] = 1; } if (i915_modparams.disable_display) { DRM_INFO("Display disabled (module parameter)\n"); info->num_pipes = 0; - } else if (info->num_pipes > 0 && - (IS_GEN7(dev_priv) || IS_GEN8(dev_priv)) && + } else if (HAS_DISPLAY(dev_priv) && + (IS_GEN_RANGE(dev_priv, 7, 8)) && HAS_PCH_SPLIT(dev_priv)) { u32 fuse_strap = I915_READ(FUSE_STRAP); u32 sfuse_strap = I915_READ(SFUSE_STRAP); @@ -804,7 +795,7 @@ void intel_device_info_runtime_init(struct intel_device_info *info) DRM_INFO("PipeC fused off\n"); info->num_pipes -= 1; } - } else if (info->num_pipes > 0 && IS_GEN9(dev_priv)) { + } else if (HAS_DISPLAY(dev_priv) && INTEL_GEN(dev_priv) >= 9) { u32 dfsm = I915_READ(SKL_DFSM); u8 disabled_mask = 0; bool invalid; @@ -844,15 +835,20 @@ void intel_device_info_runtime_init(struct intel_device_info *info) cherryview_sseu_info_init(dev_priv); else if (IS_BROADWELL(dev_priv)) broadwell_sseu_info_init(dev_priv); - else if (INTEL_GEN(dev_priv) == 9) + else if (IS_GEN(dev_priv, 9)) gen9_sseu_info_init(dev_priv); - else if (INTEL_GEN(dev_priv) == 10) + else if (IS_GEN(dev_priv, 10)) gen10_sseu_info_init(dev_priv); else if (INTEL_GEN(dev_priv) >= 11) gen11_sseu_info_init(dev_priv); + if (IS_GEN(dev_priv, 6) && intel_vtd_active()) { + DRM_INFO("Disabling ppGTT for VT-d support\n"); + info->ppgtt = INTEL_PPGTT_NONE; + } + /* Initialize command stream timestamp frequency */ - info->cs_timestamp_frequency_khz = read_timestamp_frequency(dev_priv); + runtime->cs_timestamp_frequency_khz = read_timestamp_frequency(dev_priv); } void intel_driver_caps_print(const struct intel_driver_caps *caps, @@ -872,40 +868,46 @@ void intel_driver_caps_print(const struct intel_driver_caps *caps, void intel_device_info_init_mmio(struct drm_i915_private *dev_priv) { struct intel_device_info *info = mkwrite_device_info(dev_priv); - u8 vdbox_disable, vebox_disable; + unsigned int logical_vdbox = 0; + unsigned int i; u32 media_fuse; - int i; if (INTEL_GEN(dev_priv) < 11) return; - media_fuse = I915_READ(GEN11_GT_VEBOX_VDBOX_DISABLE); + media_fuse = ~I915_READ(GEN11_GT_VEBOX_VDBOX_DISABLE); - vdbox_disable = media_fuse & GEN11_GT_VDBOX_DISABLE_MASK; - vebox_disable = (media_fuse & GEN11_GT_VEBOX_DISABLE_MASK) >> - GEN11_GT_VEBOX_DISABLE_SHIFT; + RUNTIME_INFO(dev_priv)->vdbox_enable = media_fuse & GEN11_GT_VDBOX_DISABLE_MASK; + RUNTIME_INFO(dev_priv)->vebox_enable = (media_fuse & GEN11_GT_VEBOX_DISABLE_MASK) >> + GEN11_GT_VEBOX_DISABLE_SHIFT; - DRM_DEBUG_DRIVER("vdbox disable: %04x\n", vdbox_disable); + DRM_DEBUG_DRIVER("vdbox enable: %04x\n", RUNTIME_INFO(dev_priv)->vdbox_enable); for (i = 0; i < I915_MAX_VCS; i++) { if (!HAS_ENGINE(dev_priv, _VCS(i))) continue; - if (!(BIT(i) & vdbox_disable)) + if (!(BIT(i) & RUNTIME_INFO(dev_priv)->vdbox_enable)) { + info->ring_mask &= ~ENGINE_MASK(_VCS(i)); + DRM_DEBUG_DRIVER("vcs%u fused off\n", i); continue; + } - info->ring_mask &= ~ENGINE_MASK(_VCS(i)); - DRM_DEBUG_DRIVER("vcs%u fused off\n", i); + /* + * In Gen11, only even numbered logical VDBOXes are + * hooked up to an SFC (Scaler & Format Converter) unit. + */ + if (logical_vdbox++ % 2 == 0) + RUNTIME_INFO(dev_priv)->vdbox_sfc_access |= BIT(i); } - DRM_DEBUG_DRIVER("vebox disable: %04x\n", vebox_disable); + DRM_DEBUG_DRIVER("vebox enable: %04x\n", RUNTIME_INFO(dev_priv)->vebox_enable); for (i = 0; i < I915_MAX_VECS; i++) { if (!HAS_ENGINE(dev_priv, _VECS(i))) continue; - if (!(BIT(i) & vebox_disable)) - continue; - - info->ring_mask &= ~ENGINE_MASK(_VECS(i)); - DRM_DEBUG_DRIVER("vecs%u fused off\n", i); + if (!(BIT(i) & RUNTIME_INFO(dev_priv)->vebox_enable)) { + info->ring_mask &= ~ENGINE_MASK(_VECS(i)); + DRM_DEBUG_DRIVER("vecs%u fused off\n", i); + } } } diff --git a/drivers/gpu/drm/i915/intel_device_info.h b/drivers/gpu/drm/i915/intel_device_info.h index 6eecd64734d5..e8b8661df746 100644 --- a/drivers/gpu/drm/i915/intel_device_info.h +++ b/drivers/gpu/drm/i915/intel_device_info.h @@ -25,6 +25,8 @@ #ifndef _INTEL_DEVICE_INFO_H_ #define _INTEL_DEVICE_INFO_H_ +#include <uapi/drm/i915_drm.h> + #include "intel_display.h" struct drm_printer; @@ -74,51 +76,59 @@ enum intel_platform { INTEL_MAX_PLATFORMS }; +enum intel_ppgtt { + INTEL_PPGTT_NONE = I915_GEM_PPGTT_NONE, + INTEL_PPGTT_ALIASING = I915_GEM_PPGTT_ALIASING, + INTEL_PPGTT_FULL = I915_GEM_PPGTT_FULL, + INTEL_PPGTT_FULL_4LVL, +}; + #define DEV_INFO_FOR_EACH_FLAG(func) \ func(is_mobile); \ func(is_lp); \ func(is_alpha_support); \ /* Keep has_* in alphabetical order */ \ func(has_64bit_reloc); \ - func(has_aliasing_ppgtt); \ - func(has_csr); \ - func(has_ddi); \ - func(has_dp_mst); \ + func(gpu_reset_clobbers_display); \ func(has_reset_engine); \ - func(has_fbc); \ func(has_fpga_dbg); \ - func(has_full_ppgtt); \ - func(has_full_48bit_ppgtt); \ - func(has_gmch_display); \ func(has_guc); \ func(has_guc_ct); \ - func(has_hotplug); \ func(has_l3_dpf); \ func(has_llc); \ func(has_logical_ring_contexts); \ func(has_logical_ring_elsq); \ func(has_logical_ring_preemption); \ - func(has_overlay); \ func(has_pooled_eu); \ - func(has_psr); \ func(has_rc6); \ func(has_rc6p); \ func(has_runtime_pm); \ func(has_snoop); \ func(has_coherent_ggtt); \ func(unfenced_needs_alignment); \ + func(hws_needs_physical); + +#define DEV_INFO_DISPLAY_FOR_EACH_FLAG(func) \ + /* Keep in alphabetical order */ \ func(cursor_needs_physical); \ - func(hws_needs_physical); \ + func(has_csr); \ + func(has_ddi); \ + func(has_dp_mst); \ + func(has_fbc); \ + func(has_gmch); \ + func(has_hotplug); \ + func(has_ipc); \ + func(has_overlay); \ + func(has_psr); \ func(overlay_needs_physical); \ - func(supports_tv); \ - func(has_ipc); + func(supports_tv); #define GEN_MAX_SLICES (6) /* CNL upper bound */ #define GEN_MAX_SUBSLICES (8) /* ICL upper bound */ struct sseu_dev_info { u8 slice_mask; - u8 subslice_mask[GEN_MAX_SUBSLICES]; + u8 subslice_mask[GEN_MAX_SLICES]; u16 eu_total; u8 eu_per_subslice; u8 min_eu_in_pool; @@ -143,45 +153,66 @@ struct sseu_dev_info { typedef u8 intel_ring_mask_t; struct intel_device_info { - u16 device_id; u16 gen_mask; u8 gen; u8 gt; /* GT number, 0 if undefined */ - u8 num_rings; intel_ring_mask_t ring_mask; /* Rings supported by the HW */ enum intel_platform platform; u32 platform_mask; + enum intel_ppgtt ppgtt; unsigned int page_sizes; /* page sizes supported by the HW */ u32 display_mmio_offset; u8 num_pipes; - u8 num_sprites[I915_MAX_PIPES]; - u8 num_scalers[I915_MAX_PIPES]; #define DEFINE_FLAG(name) u8 name:1 DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG); #undef DEFINE_FLAG + + struct { +#define DEFINE_FLAG(name) u8 name:1 + DEV_INFO_DISPLAY_FOR_EACH_FLAG(DEFINE_FLAG); +#undef DEFINE_FLAG + } display; + u16 ddb_size; /* in blocks */ /* Register offsets for the various display pipes and transcoders */ int pipe_offsets[I915_MAX_TRANSCODERS]; int trans_offsets[I915_MAX_TRANSCODERS]; - int palette_offsets[I915_MAX_PIPES]; int cursor_offsets[I915_MAX_PIPES]; + struct color_luts { + u16 degamma_lut_size; + u16 gamma_lut_size; + u32 degamma_lut_tests; + u32 gamma_lut_tests; + } color; +}; + +struct intel_runtime_info { + u16 device_id; + + u8 num_sprites[I915_MAX_PIPES]; + u8 num_scalers[I915_MAX_PIPES]; + + u8 num_rings; + /* Slice/subslice/EU info */ struct sseu_dev_info sseu; u32 cs_timestamp_frequency_khz; - struct color_luts { - u16 degamma_lut_size; - u16 gamma_lut_size; - } color; + /* Enabled (not fused off) media engine bitmasks. */ + u8 vdbox_enable; + u8 vebox_enable; + + /* Media engine access to SFC per instance */ + u8 vdbox_sfc_access; }; struct intel_driver_caps { @@ -238,12 +269,10 @@ static inline void sseu_set_eus(struct sseu_dev_info *sseu, const char *intel_platform_name(enum intel_platform platform); -void intel_device_info_runtime_init(struct intel_device_info *info); -void intel_device_info_dump(const struct intel_device_info *info, - struct drm_printer *p); +void intel_device_info_runtime_init(struct drm_i915_private *dev_priv); void intel_device_info_dump_flags(const struct intel_device_info *info, struct drm_printer *p); -void intel_device_info_dump_runtime(const struct intel_device_info *info, +void intel_device_info_dump_runtime(const struct intel_runtime_info *info, struct drm_printer *p); void intel_device_info_dump_topology(const struct sseu_dev_info *sseu, struct drm_printer *p); diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c index c9878dd1f7cd..421aac80a838 100644 --- a/drivers/gpu/drm/i915/intel_display.c +++ b/drivers/gpu/drm/i915/intel_display.c @@ -24,34 +24,44 @@ * Eric Anholt <eric@anholt.net> */ -#include <linux/dmi.h> -#include <linux/module.h> -#include <linux/input.h> #include <linux/i2c.h> +#include <linux/input.h> +#include <linux/intel-iommu.h> #include <linux/kernel.h> +#include <linux/module.h> +#include <linux/reservation.h> #include <linux/slab.h> #include <linux/vgaarb.h> + +#include <drm/drm_atomic.h> +#include <drm/drm_atomic_helper.h> +#include <drm/drm_atomic_uapi.h> +#include <drm/drm_dp_helper.h> #include <drm/drm_edid.h> -#include <drm/drmP.h> -#include "intel_drv.h" -#include "intel_frontbuffer.h" +#include <drm/drm_fourcc.h> +#include <drm/drm_plane_helper.h> +#include <drm/drm_probe_helper.h> +#include <drm/drm_rect.h> #include <drm/i915_drm.h> + #include "i915_drv.h" #include "i915_gem_clflush.h" +#include "i915_trace.h" +#include "intel_drv.h" #include "intel_dsi.h" +#include "intel_frontbuffer.h" + +#include "intel_drv.h" +#include "intel_dsi.h" +#include "intel_frontbuffer.h" + +#include "i915_drv.h" +#include "i915_gem_clflush.h" +#include "i915_reset.h" #include "i915_trace.h" -#include <drm/drm_atomic.h> -#include <drm/drm_atomic_helper.h> -#include <drm/drm_dp_helper.h> -#include <drm/drm_crtc_helper.h> -#include <drm/drm_plane_helper.h> -#include <drm/drm_rect.h> -#include <drm/drm_atomic_uapi.h> -#include <linux/dma_remapping.h> -#include <linux/reservation.h> /* Primary plane formats for gen <= 3 */ -static const uint32_t i8xx_primary_formats[] = { +static const u32 i8xx_primary_formats[] = { DRM_FORMAT_C8, DRM_FORMAT_RGB565, DRM_FORMAT_XRGB1555, @@ -59,7 +69,7 @@ static const uint32_t i8xx_primary_formats[] = { }; /* Primary plane formats for gen >= 4 */ -static const uint32_t i965_primary_formats[] = { +static const u32 i965_primary_formats[] = { DRM_FORMAT_C8, DRM_FORMAT_RGB565, DRM_FORMAT_XRGB8888, @@ -68,67 +78,18 @@ static const uint32_t i965_primary_formats[] = { DRM_FORMAT_XBGR2101010, }; -static const uint64_t i9xx_format_modifiers[] = { - I915_FORMAT_MOD_X_TILED, - DRM_FORMAT_MOD_LINEAR, - DRM_FORMAT_MOD_INVALID -}; - -static const uint32_t skl_primary_formats[] = { - DRM_FORMAT_C8, - DRM_FORMAT_RGB565, - DRM_FORMAT_XRGB8888, - DRM_FORMAT_XBGR8888, - DRM_FORMAT_ARGB8888, - DRM_FORMAT_ABGR8888, - DRM_FORMAT_XRGB2101010, - DRM_FORMAT_XBGR2101010, - DRM_FORMAT_YUYV, - DRM_FORMAT_YVYU, - DRM_FORMAT_UYVY, - DRM_FORMAT_VYUY, -}; - -static const uint32_t skl_pri_planar_formats[] = { - DRM_FORMAT_C8, - DRM_FORMAT_RGB565, - DRM_FORMAT_XRGB8888, - DRM_FORMAT_XBGR8888, - DRM_FORMAT_ARGB8888, - DRM_FORMAT_ABGR8888, - DRM_FORMAT_XRGB2101010, - DRM_FORMAT_XBGR2101010, - DRM_FORMAT_YUYV, - DRM_FORMAT_YVYU, - DRM_FORMAT_UYVY, - DRM_FORMAT_VYUY, - DRM_FORMAT_NV12, -}; - -static const uint64_t skl_format_modifiers_noccs[] = { - I915_FORMAT_MOD_Yf_TILED, - I915_FORMAT_MOD_Y_TILED, - I915_FORMAT_MOD_X_TILED, - DRM_FORMAT_MOD_LINEAR, - DRM_FORMAT_MOD_INVALID -}; - -static const uint64_t skl_format_modifiers_ccs[] = { - I915_FORMAT_MOD_Yf_TILED_CCS, - I915_FORMAT_MOD_Y_TILED_CCS, - I915_FORMAT_MOD_Yf_TILED, - I915_FORMAT_MOD_Y_TILED, +static const u64 i9xx_format_modifiers[] = { I915_FORMAT_MOD_X_TILED, DRM_FORMAT_MOD_LINEAR, DRM_FORMAT_MOD_INVALID }; /* Cursor formats */ -static const uint32_t intel_cursor_formats[] = { +static const u32 intel_cursor_formats[] = { DRM_FORMAT_ARGB8888, }; -static const uint64_t cursor_format_modifiers[] = { +static const u64 cursor_format_modifiers[] = { DRM_FORMAT_MOD_LINEAR, DRM_FORMAT_MOD_INVALID }; @@ -141,15 +102,15 @@ static void ironlake_pch_clock_get(struct intel_crtc *crtc, static int intel_framebuffer_init(struct intel_framebuffer *ifb, struct drm_i915_gem_object *obj, struct drm_mode_fb_cmd2 *mode_cmd); -static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc); -static void intel_set_pipe_timings(struct intel_crtc *intel_crtc); -static void intel_set_pipe_src_size(struct intel_crtc *intel_crtc); -static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc, - struct intel_link_m_n *m_n, - struct intel_link_m_n *m2_n2); -static void ironlake_set_pipeconf(struct drm_crtc *crtc); -static void haswell_set_pipeconf(struct drm_crtc *crtc); -static void haswell_set_pipemisc(struct drm_crtc *crtc); +static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state); +static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state); +static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state, + const struct intel_link_m_n *m_n, + const struct intel_link_m_n *m2_n2); +static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state); +static void ironlake_set_pipeconf(const struct intel_crtc_state *crtc_state); +static void haswell_set_pipeconf(const struct intel_crtc_state *crtc_state); +static void haswell_set_pipemisc(const struct intel_crtc_state *crtc_state); static void vlv_prepare_pll(struct intel_crtc *crtc, const struct intel_crtc_state *pipe_config); static void chv_prepare_pll(struct intel_crtc *crtc, @@ -158,9 +119,9 @@ static void intel_begin_crtc_commit(struct drm_crtc *, struct drm_crtc_state *); static void intel_finish_crtc_commit(struct drm_crtc *, struct drm_crtc_state *); static void intel_crtc_init_scalers(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state); -static void skylake_pfit_enable(struct intel_crtc *crtc); -static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force); -static void ironlake_pfit_enable(struct intel_crtc *crtc); +static void skylake_pfit_enable(const struct intel_crtc_state *crtc_state); +static void ironlake_pfit_disable(const struct intel_crtc_state *old_crtc_state); +static void ironlake_pfit_enable(const struct intel_crtc_state *crtc_state); static void intel_modeset_setup_hw_state(struct drm_device *dev, struct drm_modeset_acquire_ctx *ctx); static void intel_pre_disable_primary_noatomic(struct drm_crtc *crtc); @@ -506,23 +467,8 @@ static const struct intel_limit intel_limits_bxt = { }; static void -skl_wa_528(struct drm_i915_private *dev_priv, int pipe, bool enable) -{ - if (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv)) - return; - - if (enable) - I915_WRITE(CHICKEN_PIPESL_1(pipe), HSW_FBCQ_DIS); - else - I915_WRITE(CHICKEN_PIPESL_1(pipe), 0); -} - -static void skl_wa_clkgate(struct drm_i915_private *dev_priv, int pipe, bool enable) { - if (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv)) - return; - if (enable) I915_WRITE(CLKGATE_DIS_PSL(pipe), DUPS1_GATING_DIS | DUPS2_GATING_DIS); @@ -559,7 +505,7 @@ static int pnv_calc_dpll_params(int refclk, struct dpll *clock) return clock->dot; } -static uint32_t i9xx_dpll_compute_m(struct dpll *dpll) +static u32 i9xx_dpll_compute_m(struct dpll *dpll) { return 5 * (dpll->m1 + 2) + (dpll->m2 + 2); } @@ -594,8 +540,8 @@ int chv_calc_dpll_params(int refclk, struct dpll *clock) clock->p = clock->p1 * clock->p2; if (WARN_ON(clock->n == 0 || clock->p == 0)) return 0; - clock->vco = DIV_ROUND_CLOSEST_ULL((uint64_t)refclk * clock->m, - clock->n << 22); + clock->vco = DIV_ROUND_CLOSEST_ULL((u64)refclk * clock->m, + clock->n << 22); clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p); return clock->dot / 5; @@ -957,7 +903,7 @@ chv_find_best_dpll(const struct intel_limit *limit, struct drm_device *dev = crtc->base.dev; unsigned int best_error_ppm; struct dpll clock; - uint64_t m2; + u64 m2; int found = false; memset(best_clock, 0, sizeof(*best_clock)); @@ -979,7 +925,7 @@ chv_find_best_dpll(const struct intel_limit *limit, clock.p = clock.p1 * clock.p2; - m2 = DIV_ROUND_CLOSEST_ULL(((uint64_t)target * clock.p * + m2 = DIV_ROUND_CLOSEST_ULL(((u64)target * clock.p * clock.n) << 22, refclk * clock.m1); if (m2 > INT_MAX/clock.m1) @@ -1049,7 +995,7 @@ static bool pipe_scanline_is_moving(struct drm_i915_private *dev_priv, u32 line1, line2; u32 line_mask; - if (IS_GEN2(dev_priv)) + if (IS_GEN(dev_priv, 2)) line_mask = DSL_LINEMASK_GEN2; else line_mask = DSL_LINEMASK_GEN3; @@ -1175,7 +1121,7 @@ static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv, u32 val; /* ILK FDI PLL is always enabled */ - if (IS_GEN5(dev_priv)) + if (IS_GEN(dev_priv, 5)) return; /* On Haswell, DDI ports are responsible for the FDI PLL setup */ @@ -1263,17 +1209,19 @@ void assert_pipe(struct drm_i915_private *dev_priv, enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv, pipe); enum intel_display_power_domain power_domain; + intel_wakeref_t wakeref; /* we keep both pipes enabled on 830 */ if (IS_I830(dev_priv)) state = true; power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder); - if (intel_display_power_get_if_enabled(dev_priv, power_domain)) { + wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); + if (wakeref) { u32 val = I915_READ(PIPECONF(cpu_transcoder)); cur_state = !!(val & PIPECONF_ENABLE); - intel_display_power_put(dev_priv, power_domain); + intel_display_power_put(dev_priv, power_domain, wakeref); } else { cur_state = false; } @@ -1381,6 +1329,7 @@ static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv, "PCH LVDS enabled on transcoder %c, should be disabled\n", pipe_name(pipe)); + /* PCH SDVOB multiplex with HDMIB */ assert_pch_hdmi_disabled(dev_priv, pipe, PORT_B, PCH_HDMIB); assert_pch_hdmi_disabled(dev_priv, pipe, PORT_C, PCH_HDMIC); assert_pch_hdmi_disabled(dev_priv, pipe, PORT_D, PCH_HDMID); @@ -1565,14 +1514,15 @@ static void i9xx_enable_pll(struct intel_crtc *crtc, } } -static void i9xx_disable_pll(struct intel_crtc *crtc) +static void i9xx_disable_pll(const struct intel_crtc_state *crtc_state) { + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; /* Disable DVO 2x clock on both PLLs if necessary */ if (IS_I830(dev_priv) && - intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DVO) && + intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO) && !intel_num_dvo_pipes(dev_priv)) { I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) & ~DPLL_DVO_2X_MODE); @@ -1666,16 +1616,16 @@ void vlv_wait_port_ready(struct drm_i915_private *dev_priv, I915_READ(dpll_reg) & port_mask, expected_mask); } -static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv, - enum pipe pipe) +static void ironlake_enable_pch_transcoder(const struct intel_crtc_state *crtc_state) { - struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv, - pipe); + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; i915_reg_t reg; - uint32_t val, pipeconf_val; + u32 val, pipeconf_val; /* Make sure PCH DPLL is enabled */ - assert_shared_dpll_enabled(dev_priv, intel_crtc->config->shared_dpll); + assert_shared_dpll_enabled(dev_priv, crtc_state->shared_dpll); /* FDI must be feeding us bits for PCH ports */ assert_fdi_tx_enabled(dev_priv, pipe); @@ -1701,7 +1651,7 @@ static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv, * here for both 8bpc and 12bpc. */ val &= ~PIPECONF_BPC_MASK; - if (intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_HDMI)) + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) val |= PIPECONF_8BPC; else val |= pipeconf_val & PIPECONF_BPC_MASK; @@ -1710,7 +1660,7 @@ static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv, val &= ~TRANS_INTERLACE_MASK; if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK) if (HAS_PCH_IBX(dev_priv) && - intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_SDVO)) + intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO)) val |= TRANS_LEGACY_INTERLACED_ILK; else val |= TRANS_INTERLACED; @@ -1760,7 +1710,7 @@ static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv, enum pipe pipe) { i915_reg_t reg; - uint32_t val; + u32 val; /* FDI relies on the transcoder */ assert_fdi_tx_disabled(dev_priv, pipe); @@ -1817,6 +1767,35 @@ enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc) return crtc->pipe; } +static u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); + + /* + * On i965gm the hardware frame counter reads + * zero when the TV encoder is enabled :( + */ + if (IS_I965GM(dev_priv) && + (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT))) + return 0; + + if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv)) + return 0xffffffff; /* full 32 bit counter */ + else if (INTEL_GEN(dev_priv) >= 3) + return 0xffffff; /* only 24 bits of frame count */ + else + return 0; /* Gen2 doesn't have a hardware frame counter */ +} + +static void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + + drm_crtc_set_max_vblank_count(&crtc->base, + intel_crtc_max_vblank_count(crtc_state)); + drm_crtc_vblank_on(&crtc->base); +} + static void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state) { struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc); @@ -1835,7 +1814,7 @@ static void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state) * a plane. On ILK+ the pipe PLLs are integrated, so we don't * need the check. */ - if (HAS_GMCH_DISPLAY(dev_priv)) { + if (HAS_GMCH(dev_priv)) { if (intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI)) assert_dsi_pll_enabled(dev_priv); else @@ -1869,7 +1848,7 @@ static void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state) * when it's derived from the timestamps. So let's wait for the * pipe to start properly before we call drm_crtc_vblank_on() */ - if (dev_priv->drm.max_vblank_count == 0) + if (intel_crtc_max_vblank_count(new_crtc_state) == 0) intel_wait_for_pipe_scanline_moving(crtc); } @@ -1913,7 +1892,7 @@ static void intel_disable_pipe(const struct intel_crtc_state *old_crtc_state) static unsigned int intel_tile_size(const struct drm_i915_private *dev_priv) { - return IS_GEN2(dev_priv) ? 2048 : 4096; + return IS_GEN(dev_priv, 2) ? 2048 : 4096; } static unsigned int @@ -1926,7 +1905,7 @@ intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane) case DRM_FORMAT_MOD_LINEAR: return cpp; case I915_FORMAT_MOD_X_TILED: - if (IS_GEN2(dev_priv)) + if (IS_GEN(dev_priv, 2)) return 128; else return 512; @@ -1935,7 +1914,7 @@ intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane) return 128; /* fall through */ case I915_FORMAT_MOD_Y_TILED: - if (IS_GEN2(dev_priv) || HAS_128_BYTE_Y_TILING(dev_priv)) + if (IS_GEN(dev_priv, 2) || HAS_128_BYTE_Y_TILING(dev_priv)) return 128; else return 512; @@ -2087,6 +2066,7 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, struct drm_device *dev = fb->dev; struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_object *obj = intel_fb_obj(fb); + intel_wakeref_t wakeref; struct i915_vma *vma; unsigned int pinctl; u32 alignment; @@ -2110,7 +2090,7 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, * intel_runtime_pm_put(), so it is correct to wrap only the * pin/unpin/fence and not more. */ - intel_runtime_pm_get(dev_priv); + wakeref = intel_runtime_pm_get(dev_priv); atomic_inc(&dev_priv->gpu_error.pending_fb_pin); @@ -2123,7 +2103,7 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, * complicated than this. For example, Cherryview appears quite * happy to scanout from anywhere within its global aperture. */ - if (HAS_GMCH_DISPLAY(dev_priv)) + if (HAS_GMCH(dev_priv)) pinctl |= PIN_MAPPABLE; vma = i915_gem_object_pin_to_display_plane(obj, @@ -2165,7 +2145,7 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, err: atomic_dec(&dev_priv->gpu_error.pending_fb_pin); - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, wakeref); return vma; } @@ -2254,6 +2234,11 @@ static u32 intel_adjust_tile_offset(int *x, int *y, return new_offset; } +static bool is_surface_linear(u64 modifier, int color_plane) +{ + return modifier == DRM_FORMAT_MOD_LINEAR; +} + static u32 intel_adjust_aligned_offset(int *x, int *y, const struct drm_framebuffer *fb, int color_plane, @@ -2266,7 +2251,7 @@ static u32 intel_adjust_aligned_offset(int *x, int *y, WARN_ON(new_offset > old_offset); - if (fb->modifier != DRM_FORMAT_MOD_LINEAR) { + if (!is_surface_linear(fb->modifier, color_plane)) { unsigned int tile_size, tile_width, tile_height; unsigned int pitch_tiles; @@ -2330,14 +2315,13 @@ static u32 intel_compute_aligned_offset(struct drm_i915_private *dev_priv, unsigned int rotation, u32 alignment) { - uint64_t fb_modifier = fb->modifier; unsigned int cpp = fb->format->cpp[color_plane]; u32 offset, offset_aligned; if (alignment) alignment--; - if (fb_modifier != DRM_FORMAT_MOD_LINEAR) { + if (!is_surface_linear(fb->modifier, color_plane)) { unsigned int tile_size, tile_width, tile_height; unsigned int tile_rows, tiles, pitch_tiles; @@ -2400,10 +2384,26 @@ static int intel_fb_offset_to_xy(int *x, int *y, int color_plane) { struct drm_i915_private *dev_priv = to_i915(fb->dev); + unsigned int height; if (fb->modifier != DRM_FORMAT_MOD_LINEAR && - fb->offsets[color_plane] % intel_tile_size(dev_priv)) + fb->offsets[color_plane] % intel_tile_size(dev_priv)) { + DRM_DEBUG_KMS("Misaligned offset 0x%08x for color plane %d\n", + fb->offsets[color_plane], color_plane); return -EINVAL; + } + + height = drm_framebuffer_plane_height(fb->height, fb, color_plane); + height = ALIGN(height, intel_tile_height(fb, color_plane)); + + /* Catch potential overflows early */ + if (add_overflows_t(u32, mul_u32_u32(height, fb->pitches[color_plane]), + fb->offsets[color_plane])) { + DRM_DEBUG_KMS("Bad offset 0x%08x or pitch %d for color plane %d\n", + fb->offsets[color_plane], fb->pitches[color_plane], + color_plane); + return -ERANGE; + } *x = 0; *y = 0; @@ -2416,7 +2416,7 @@ static int intel_fb_offset_to_xy(int *x, int *y, return 0; } -static unsigned int intel_fb_modifier_to_tiling(uint64_t fb_modifier) +static unsigned int intel_fb_modifier_to_tiling(u64 fb_modifier) { switch (fb_modifier) { case I915_FORMAT_MOD_X_TILED: @@ -2574,7 +2574,7 @@ intel_fill_fb_info(struct drm_i915_private *dev_priv, tile_size); offset /= tile_size; - if (fb->modifier != DRM_FORMAT_MOD_LINEAR) { + if (!is_surface_linear(fb->modifier, i)) { unsigned int tile_width, tile_height; unsigned int pitch_tiles; struct drm_rect r; @@ -2788,10 +2788,6 @@ intel_set_plane_visible(struct intel_crtc_state *crtc_state, crtc_state->base.plane_mask |= drm_plane_mask(&plane->base); else crtc_state->base.plane_mask &= ~drm_plane_mask(&plane->base); - - DRM_DEBUG_KMS("%s active planes 0x%x\n", - crtc_state->base.crtc->name, - crtc_state->active_planes); } static void fixup_active_planes(struct intel_crtc_state *crtc_state) @@ -2819,6 +2815,10 @@ static void intel_plane_disable_noatomic(struct intel_crtc *crtc, struct intel_plane_state *plane_state = to_intel_plane_state(plane->base.state); + DRM_DEBUG_KMS("Disabling [PLANE:%d:%s] on [CRTC:%d:%s]\n", + plane->base.base.id, plane->base.name, + crtc->base.base.id, crtc->base.name); + intel_set_plane_visible(crtc_state, plane_state, false); fixup_active_planes(crtc_state); @@ -2826,7 +2826,7 @@ static void intel_plane_disable_noatomic(struct intel_crtc *crtc, intel_pre_disable_primary_noatomic(&crtc->base); trace_intel_disable_plane(&plane->base, crtc); - plane->disable_plane(plane, crtc); + plane->disable_plane(plane, crtc_state); } static void @@ -3099,28 +3099,6 @@ static int skl_check_main_surface(struct intel_plane_state *plane_state) return 0; } -static int -skl_check_nv12_surface(struct intel_plane_state *plane_state) -{ - /* Display WA #1106 */ - if (plane_state->base.rotation != - (DRM_MODE_REFLECT_X | DRM_MODE_ROTATE_90) && - plane_state->base.rotation != DRM_MODE_ROTATE_270) - return 0; - - /* - * src coordinates are rotated here. - * We check height but report it as width - */ - if (((drm_rect_height(&plane_state->base.src) >> 16) % 4) != 0) { - DRM_DEBUG_KMS("src width must be multiple " - "of 4 for rotated NV12\n"); - return -EINVAL; - } - - return 0; -} - static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state) { const struct drm_framebuffer *fb = plane_state->base.fb; @@ -3199,9 +3177,6 @@ int skl_check_plane_surface(struct intel_plane_state *plane_state) * the main surface setup depends on it. */ if (fb->format->format == DRM_FORMAT_NV12) { - ret = skl_check_nv12_surface(plane_state); - if (ret) - return ret; ret = skl_check_nv12_aux_surface(plane_state); if (ret) return ret; @@ -3229,7 +3204,7 @@ i9xx_plane_max_stride(struct intel_plane *plane, { struct drm_i915_private *dev_priv = to_i915(plane->base.dev); - if (!HAS_GMCH_DISPLAY(dev_priv)) { + if (!HAS_GMCH(dev_priv)) { return 32*1024; } else if (INTEL_GEN(dev_priv) >= 4) { if (modifier == I915_FORMAT_MOD_X_TILED) @@ -3249,28 +3224,38 @@ i9xx_plane_max_stride(struct intel_plane *plane, } } +static u32 i9xx_plane_ctl_crtc(const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + u32 dspcntr = 0; + + dspcntr |= DISPPLANE_GAMMA_ENABLE; + + if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) + dspcntr |= DISPPLANE_PIPE_CSC_ENABLE; + + if (INTEL_GEN(dev_priv) < 5) + dspcntr |= DISPPLANE_SEL_PIPE(crtc->pipe); + + return dspcntr; +} + static u32 i9xx_plane_ctl(const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state) { struct drm_i915_private *dev_priv = to_i915(plane_state->base.plane->dev); - struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); const struct drm_framebuffer *fb = plane_state->base.fb; unsigned int rotation = plane_state->base.rotation; u32 dspcntr; - dspcntr = DISPLAY_PLANE_ENABLE | DISPPLANE_GAMMA_ENABLE; + dspcntr = DISPLAY_PLANE_ENABLE; - if (IS_G4X(dev_priv) || IS_GEN5(dev_priv) || - IS_GEN6(dev_priv) || IS_IVYBRIDGE(dev_priv)) + if (IS_G4X(dev_priv) || IS_GEN(dev_priv, 5) || + IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv)) dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE; - if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) - dspcntr |= DISPPLANE_PIPE_CSC_ENABLE; - - if (INTEL_GEN(dev_priv) < 5) - dspcntr |= DISPPLANE_SEL_PIPE(crtc->pipe); - switch (fb->format->format) { case DRM_FORMAT_C8: dspcntr |= DISPPLANE_8BPP; @@ -3398,12 +3383,13 @@ static void i9xx_update_plane(struct intel_plane *plane, struct drm_i915_private *dev_priv = to_i915(plane->base.dev); enum i9xx_plane_id i9xx_plane = plane->i9xx_plane; u32 linear_offset; - u32 dspcntr = plane_state->ctl; - i915_reg_t reg = DSPCNTR(i9xx_plane); int x = plane_state->color_plane[0].x; int y = plane_state->color_plane[0].y; unsigned long irqflags; u32 dspaddr_offset; + u32 dspcntr; + + dspcntr = plane_state->ctl | i9xx_plane_ctl_crtc(crtc_state); linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0); @@ -3414,61 +3400,76 @@ static void i9xx_update_plane(struct intel_plane *plane, spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); + I915_WRITE_FW(DSPSTRIDE(i9xx_plane), plane_state->color_plane[0].stride); + if (INTEL_GEN(dev_priv) < 4) { /* pipesrc and dspsize control the size that is scaled from, * which should always be the user's requested size. */ + I915_WRITE_FW(DSPPOS(i9xx_plane), 0); I915_WRITE_FW(DSPSIZE(i9xx_plane), ((crtc_state->pipe_src_h - 1) << 16) | (crtc_state->pipe_src_w - 1)); - I915_WRITE_FW(DSPPOS(i9xx_plane), 0); } else if (IS_CHERRYVIEW(dev_priv) && i9xx_plane == PLANE_B) { + I915_WRITE_FW(PRIMPOS(i9xx_plane), 0); I915_WRITE_FW(PRIMSIZE(i9xx_plane), ((crtc_state->pipe_src_h - 1) << 16) | (crtc_state->pipe_src_w - 1)); - I915_WRITE_FW(PRIMPOS(i9xx_plane), 0); I915_WRITE_FW(PRIMCNSTALPHA(i9xx_plane), 0); } - I915_WRITE_FW(reg, dspcntr); - - I915_WRITE_FW(DSPSTRIDE(i9xx_plane), plane_state->color_plane[0].stride); if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { - I915_WRITE_FW(DSPSURF(i9xx_plane), - intel_plane_ggtt_offset(plane_state) + - dspaddr_offset); I915_WRITE_FW(DSPOFFSET(i9xx_plane), (y << 16) | x); } else if (INTEL_GEN(dev_priv) >= 4) { + I915_WRITE_FW(DSPLINOFF(i9xx_plane), linear_offset); + I915_WRITE_FW(DSPTILEOFF(i9xx_plane), (y << 16) | x); + } + + /* + * The control register self-arms if the plane was previously + * disabled. Try to make the plane enable atomic by writing + * the control register just before the surface register. + */ + I915_WRITE_FW(DSPCNTR(i9xx_plane), dspcntr); + if (INTEL_GEN(dev_priv) >= 4) I915_WRITE_FW(DSPSURF(i9xx_plane), intel_plane_ggtt_offset(plane_state) + dspaddr_offset); - I915_WRITE_FW(DSPTILEOFF(i9xx_plane), (y << 16) | x); - I915_WRITE_FW(DSPLINOFF(i9xx_plane), linear_offset); - } else { + else I915_WRITE_FW(DSPADDR(i9xx_plane), intel_plane_ggtt_offset(plane_state) + dspaddr_offset); - } - POSTING_READ_FW(reg); spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); } static void i9xx_disable_plane(struct intel_plane *plane, - struct intel_crtc *crtc) + const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(plane->base.dev); enum i9xx_plane_id i9xx_plane = plane->i9xx_plane; unsigned long irqflags; + u32 dspcntr; + + /* + * DSPCNTR pipe gamma enable on g4x+ and pipe csc + * enable on ilk+ affect the pipe bottom color as + * well, so we must configure them even if the plane + * is disabled. + * + * On pre-g4x there is no way to gamma correct the + * pipe bottom color but we'll keep on doing this + * anyway. + */ + dspcntr = i9xx_plane_ctl_crtc(crtc_state); spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); - I915_WRITE_FW(DSPCNTR(i9xx_plane), 0); + I915_WRITE_FW(DSPCNTR(i9xx_plane), dspcntr); if (INTEL_GEN(dev_priv) >= 4) I915_WRITE_FW(DSPSURF(i9xx_plane), 0); else I915_WRITE_FW(DSPADDR(i9xx_plane), 0); - POSTING_READ_FW(DSPCNTR(i9xx_plane)); spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); } @@ -3479,6 +3480,7 @@ static bool i9xx_plane_get_hw_state(struct intel_plane *plane, struct drm_i915_private *dev_priv = to_i915(plane->base.dev); enum intel_display_power_domain power_domain; enum i9xx_plane_id i9xx_plane = plane->i9xx_plane; + intel_wakeref_t wakeref; bool ret; u32 val; @@ -3488,7 +3490,8 @@ static bool i9xx_plane_get_hw_state(struct intel_plane *plane, * display power wells. */ power_domain = POWER_DOMAIN_PIPE(plane->pipe); - if (!intel_display_power_get_if_enabled(dev_priv, power_domain)) + wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); + if (!wakeref) return false; val = I915_READ(DSPCNTR(i9xx_plane)); @@ -3501,7 +3504,7 @@ static bool i9xx_plane_get_hw_state(struct intel_plane *plane, *pipe = (val & DISPPLANE_SEL_PIPE_MASK) >> DISPPLANE_SEL_PIPE_SHIFT; - intel_display_power_put(dev_priv, power_domain); + intel_display_power_put(dev_priv, power_domain, wakeref); return ret; } @@ -3528,13 +3531,13 @@ static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id) /* * This function detaches (aka. unbinds) unused scalers in hardware */ -static void skl_detach_scalers(struct intel_crtc *intel_crtc) +static void skl_detach_scalers(const struct intel_crtc_state *crtc_state) { - struct intel_crtc_scaler_state *scaler_state; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc); + const struct intel_crtc_scaler_state *scaler_state = + &crtc_state->scaler_state; int i; - scaler_state = &intel_crtc->config->scaler_state; - /* loop through and disable scalers that aren't in use */ for (i = 0; i < intel_crtc->num_scalers; i++) { if (!scaler_state->scalers[i].in_use) @@ -3542,6 +3545,21 @@ static void skl_detach_scalers(struct intel_crtc *intel_crtc) } } +static unsigned int skl_plane_stride_mult(const struct drm_framebuffer *fb, + int color_plane, unsigned int rotation) +{ + /* + * The stride is either expressed as a multiple of 64 bytes chunks for + * linear buffers or in number of tiles for tiled buffers. + */ + if (fb->modifier == DRM_FORMAT_MOD_LINEAR) + return 64; + else if (drm_rotation_90_or_270(rotation)) + return intel_tile_height(fb, color_plane); + else + return intel_tile_width_bytes(fb, color_plane); +} + u32 skl_plane_stride(const struct intel_plane_state *plane_state, int color_plane) { @@ -3552,19 +3570,10 @@ u32 skl_plane_stride(const struct intel_plane_state *plane_state, if (color_plane >= fb->format->num_planes) return 0; - /* - * The stride is either expressed as a multiple of 64 bytes chunks for - * linear buffers or in number of tiles for tiled buffers. - */ - if (drm_rotation_90_or_270(rotation)) - stride /= intel_tile_height(fb, color_plane); - else - stride /= intel_fb_stride_alignment(fb, color_plane); - - return stride; + return stride / skl_plane_stride_mult(fb, color_plane, rotation); } -static u32 skl_plane_ctl_format(uint32_t pixel_format) +static u32 skl_plane_ctl_format(u32 pixel_format) { switch (pixel_format) { case DRM_FORMAT_C8: @@ -3598,34 +3607,43 @@ static u32 skl_plane_ctl_format(uint32_t pixel_format) return 0; } -/* - * XXX: For ARBG/ABGR formats we default to expecting scanout buffers - * to be already pre-multiplied. We need to add a knob (or a different - * DRM_FORMAT) for user-space to configure that. - */ -static u32 skl_plane_ctl_alpha(uint32_t pixel_format) +static u32 skl_plane_ctl_alpha(const struct intel_plane_state *plane_state) { - switch (pixel_format) { - case DRM_FORMAT_ABGR8888: - case DRM_FORMAT_ARGB8888: + if (!plane_state->base.fb->format->has_alpha) + return PLANE_CTL_ALPHA_DISABLE; + + switch (plane_state->base.pixel_blend_mode) { + case DRM_MODE_BLEND_PIXEL_NONE: + return PLANE_CTL_ALPHA_DISABLE; + case DRM_MODE_BLEND_PREMULTI: return PLANE_CTL_ALPHA_SW_PREMULTIPLY; + case DRM_MODE_BLEND_COVERAGE: + return PLANE_CTL_ALPHA_HW_PREMULTIPLY; default: + MISSING_CASE(plane_state->base.pixel_blend_mode); return PLANE_CTL_ALPHA_DISABLE; } } -static u32 glk_plane_color_ctl_alpha(uint32_t pixel_format) +static u32 glk_plane_color_ctl_alpha(const struct intel_plane_state *plane_state) { - switch (pixel_format) { - case DRM_FORMAT_ABGR8888: - case DRM_FORMAT_ARGB8888: + if (!plane_state->base.fb->format->has_alpha) + return PLANE_COLOR_ALPHA_DISABLE; + + switch (plane_state->base.pixel_blend_mode) { + case DRM_MODE_BLEND_PIXEL_NONE: + return PLANE_COLOR_ALPHA_DISABLE; + case DRM_MODE_BLEND_PREMULTI: return PLANE_COLOR_ALPHA_SW_PREMULTIPLY; + case DRM_MODE_BLEND_COVERAGE: + return PLANE_COLOR_ALPHA_HW_PREMULTIPLY; default: + MISSING_CASE(plane_state->base.pixel_blend_mode); return PLANE_COLOR_ALPHA_DISABLE; } } -static u32 skl_plane_ctl_tiling(uint64_t fb_modifier) +static u32 skl_plane_ctl_tiling(u64 fb_modifier) { switch (fb_modifier) { case DRM_FORMAT_MOD_LINEAR: @@ -3684,6 +3702,20 @@ static u32 cnl_plane_ctl_flip(unsigned int reflect) return 0; } +u32 skl_plane_ctl_crtc(const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); + u32 plane_ctl = 0; + + if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) + return plane_ctl; + + plane_ctl |= PLANE_CTL_PIPE_GAMMA_ENABLE; + plane_ctl |= PLANE_CTL_PIPE_CSC_ENABLE; + + return plane_ctl; +} + u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state) { @@ -3697,11 +3729,8 @@ u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state, plane_ctl = PLANE_CTL_ENABLE; if (INTEL_GEN(dev_priv) < 10 && !IS_GEMINILAKE(dev_priv)) { - plane_ctl |= skl_plane_ctl_alpha(fb->format->format); - plane_ctl |= - PLANE_CTL_PIPE_GAMMA_ENABLE | - PLANE_CTL_PIPE_CSC_ENABLE | - PLANE_CTL_PLANE_GAMMA_DISABLE; + plane_ctl |= skl_plane_ctl_alpha(plane_state); + plane_ctl |= PLANE_CTL_PLANE_GAMMA_DISABLE; if (plane_state->base.color_encoding == DRM_COLOR_YCBCR_BT709) plane_ctl |= PLANE_CTL_YUV_TO_RGB_CSC_FORMAT_BT709; @@ -3726,22 +3755,31 @@ u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state, return plane_ctl; } +u32 glk_plane_color_ctl_crtc(const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); + u32 plane_color_ctl = 0; + + if (INTEL_GEN(dev_priv) >= 11) + return plane_color_ctl; + + plane_color_ctl |= PLANE_COLOR_PIPE_GAMMA_ENABLE; + plane_color_ctl |= PLANE_COLOR_PIPE_CSC_ENABLE; + + return plane_color_ctl; +} + u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state) { - struct drm_i915_private *dev_priv = - to_i915(plane_state->base.plane->dev); const struct drm_framebuffer *fb = plane_state->base.fb; + struct intel_plane *plane = to_intel_plane(plane_state->base.plane); u32 plane_color_ctl = 0; - if (INTEL_GEN(dev_priv) < 11) { - plane_color_ctl |= PLANE_COLOR_PIPE_GAMMA_ENABLE; - plane_color_ctl |= PLANE_COLOR_PIPE_CSC_ENABLE; - } plane_color_ctl |= PLANE_COLOR_PLANE_GAMMA_DISABLE; - plane_color_ctl |= glk_plane_color_ctl_alpha(fb->format->format); + plane_color_ctl |= glk_plane_color_ctl_alpha(plane_state); - if (fb->format->is_yuv) { + if (fb->format->is_yuv && !icl_is_hdr_plane(plane)) { if (plane_state->base.color_encoding == DRM_COLOR_YCBCR_BT709) plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709; else @@ -3749,6 +3787,8 @@ u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state, if (plane_state->base.color_range == DRM_COLOR_YCBCR_FULL_RANGE) plane_color_ctl |= PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE; + } else if (fb->format->is_yuv) { + plane_color_ctl |= PLANE_COLOR_INPUT_CSC_ENABLE; } return plane_color_ctl; @@ -3784,7 +3824,7 @@ __intel_display_resume(struct drm_device *dev, } /* ignore any reset values/BIOS leftovers in the WM registers */ - if (!HAS_GMCH_DISPLAY(to_i915(dev))) + if (!HAS_GMCH(to_i915(dev))) to_intel_atomic_state(state)->skip_intermediate_wm = true; ret = drm_atomic_helper_commit_duplicated_state(state, ctx); @@ -3795,8 +3835,8 @@ __intel_display_resume(struct drm_device *dev, static bool gpu_reset_clobbers_display(struct drm_i915_private *dev_priv) { - return intel_has_gpu_reset(dev_priv) && - INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv); + return (INTEL_INFO(dev_priv)->gpu_reset_clobbers_display && + intel_has_gpu_reset(dev_priv)); } void intel_prepare_reset(struct drm_i915_private *dev_priv) @@ -3909,6 +3949,30 @@ unlock: clear_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags); } +static void icl_set_pipe_chicken(struct intel_crtc *crtc) +{ + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; + u32 tmp; + + tmp = I915_READ(PIPE_CHICKEN(pipe)); + + /* + * Display WA #1153: icl + * enable hardware to bypass the alpha math + * and rounding for per-pixel values 00 and 0xff + */ + tmp |= PER_PIXEL_ALPHA_BYPASS_EN; + + /* + * W/A for underruns with linear/X-tiled with + * WM1+ disabled. + */ + tmp |= PM_FILL_MAINTAIN_DBUF_FULLNESS; + + I915_WRITE(PIPE_CHICKEN(pipe), tmp); +} + static void intel_update_pipe_config(const struct intel_crtc_state *old_crtc_state, const struct intel_crtc_state *new_crtc_state) { @@ -3933,16 +3997,29 @@ static void intel_update_pipe_config(const struct intel_crtc_state *old_crtc_sta /* on skylake this is done by detaching scalers */ if (INTEL_GEN(dev_priv) >= 9) { - skl_detach_scalers(crtc); + skl_detach_scalers(new_crtc_state); if (new_crtc_state->pch_pfit.enabled) - skylake_pfit_enable(crtc); + skylake_pfit_enable(new_crtc_state); } else if (HAS_PCH_SPLIT(dev_priv)) { if (new_crtc_state->pch_pfit.enabled) - ironlake_pfit_enable(crtc); + ironlake_pfit_enable(new_crtc_state); else if (old_crtc_state->pch_pfit.enabled) - ironlake_pfit_disable(crtc, true); + ironlake_pfit_disable(old_crtc_state); } + + /* + * We don't (yet) allow userspace to control the pipe background color, + * so force it to black, but apply pipe gamma and CSC so that its + * handling will match how we program our planes. + */ + if (INTEL_GEN(dev_priv) >= 9) + I915_WRITE(SKL_BOTTOM_COLOR(crtc->pipe), + SKL_BOTTOM_COLOR_GAMMA_ENABLE | + SKL_BOTTOM_COLOR_CSC_ENABLE); + + if (INTEL_GEN(dev_priv) >= 11) + icl_set_pipe_chicken(crtc); } static void intel_fdi_normal_train(struct intel_crtc *crtc) @@ -4169,7 +4246,7 @@ static void gen6_fdi_link_train(struct intel_crtc *crtc, temp = I915_READ(reg); temp &= ~FDI_LINK_TRAIN_NONE; temp |= FDI_LINK_TRAIN_PATTERN_2; - if (IS_GEN6(dev_priv)) { + if (IS_GEN(dev_priv, 6)) { temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK; /* SNB-B */ temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B; @@ -4340,10 +4417,10 @@ train_done: DRM_DEBUG_KMS("FDI train done.\n"); } -static void ironlake_fdi_pll_enable(struct intel_crtc *intel_crtc) +static void ironlake_fdi_pll_enable(const struct intel_crtc_state *crtc_state) { - struct drm_device *dev = intel_crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev); int pipe = intel_crtc->pipe; i915_reg_t reg; u32 temp; @@ -4352,7 +4429,7 @@ static void ironlake_fdi_pll_enable(struct intel_crtc *intel_crtc) reg = FDI_RX_CTL(pipe); temp = I915_READ(reg); temp &= ~(FDI_DP_PORT_WIDTH_MASK | (0x7 << 16)); - temp |= FDI_DP_PORT_WIDTH(intel_crtc->config->fdi_lanes); + temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes); temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11; I915_WRITE(reg, temp | FDI_RX_PLL_ENABLE); @@ -4501,10 +4578,11 @@ void lpt_disable_iclkip(struct drm_i915_private *dev_priv) } /* Program iCLKIP clock to the desired frequency */ -static void lpt_program_iclkip(struct intel_crtc *crtc) +static void lpt_program_iclkip(const struct intel_crtc_state *crtc_state) { + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - int clock = crtc->config->base.adjusted_mode.crtc_clock; + int clock = crtc_state->base.adjusted_mode.crtc_clock; u32 divsel, phaseinc, auxdiv, phasedir = 0; u32 temp; @@ -4615,12 +4693,12 @@ int lpt_get_iclkip(struct drm_i915_private *dev_priv) desired_divisor << auxdiv); } -static void ironlake_pch_transcoder_set_timings(struct intel_crtc *crtc, +static void ironlake_pch_transcoder_set_timings(const struct intel_crtc_state *crtc_state, enum pipe pch_transcoder) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - enum transcoder cpu_transcoder = crtc->config->cpu_transcoder; + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; I915_WRITE(PCH_TRANS_HTOTAL(pch_transcoder), I915_READ(HTOTAL(cpu_transcoder))); @@ -4639,10 +4717,9 @@ static void ironlake_pch_transcoder_set_timings(struct intel_crtc *crtc, I915_READ(VSYNCSHIFT(cpu_transcoder))); } -static void cpt_set_fdi_bc_bifurcation(struct drm_device *dev, bool enable) +static void cpt_set_fdi_bc_bifurcation(struct drm_i915_private *dev_priv, bool enable) { - struct drm_i915_private *dev_priv = to_i915(dev); - uint32_t temp; + u32 temp; temp = I915_READ(SOUTH_CHICKEN1); if (!!(temp & FDI_BC_BIFURCATION_SELECT) == enable) @@ -4660,22 +4737,23 @@ static void cpt_set_fdi_bc_bifurcation(struct drm_device *dev, bool enable) POSTING_READ(SOUTH_CHICKEN1); } -static void ivybridge_update_fdi_bc_bifurcation(struct intel_crtc *intel_crtc) +static void ivybridge_update_fdi_bc_bifurcation(const struct intel_crtc_state *crtc_state) { - struct drm_device *dev = intel_crtc->base.dev; + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - switch (intel_crtc->pipe) { + switch (crtc->pipe) { case PIPE_A: break; case PIPE_B: - if (intel_crtc->config->fdi_lanes > 2) - cpt_set_fdi_bc_bifurcation(dev, false); + if (crtc_state->fdi_lanes > 2) + cpt_set_fdi_bc_bifurcation(dev_priv, false); else - cpt_set_fdi_bc_bifurcation(dev, true); + cpt_set_fdi_bc_bifurcation(dev_priv, true); break; case PIPE_C: - cpt_set_fdi_bc_bifurcation(dev, true); + cpt_set_fdi_bc_bifurcation(dev_priv, true); break; default: @@ -4732,7 +4810,7 @@ static void ironlake_pch_enable(const struct intel_atomic_state *state, assert_pch_transcoder_disabled(dev_priv, pipe); if (IS_IVYBRIDGE(dev_priv)) - ivybridge_update_fdi_bc_bifurcation(crtc); + ivybridge_update_fdi_bc_bifurcation(crtc_state); /* Write the TU size bits before fdi link training, so that error * detection works. */ @@ -4765,11 +4843,11 @@ static void ironlake_pch_enable(const struct intel_atomic_state *state, * Note that enable_shared_dpll tries to do the right thing, but * get_shared_dpll unconditionally resets the pll - we need that to have * the right LVDS enable sequence. */ - intel_enable_shared_dpll(crtc); + intel_enable_shared_dpll(crtc_state); /* set transcoder timing, panel must allow it */ assert_panel_unlocked(dev_priv, pipe); - ironlake_pch_transcoder_set_timings(crtc, pipe); + ironlake_pch_transcoder_set_timings(crtc_state, pipe); intel_fdi_normal_train(crtc); @@ -4801,7 +4879,7 @@ static void ironlake_pch_enable(const struct intel_atomic_state *state, I915_WRITE(reg, temp); } - ironlake_enable_pch_transcoder(dev_priv, pipe); + ironlake_enable_pch_transcoder(crtc_state); } static void lpt_pch_enable(const struct intel_atomic_state *state, @@ -4813,10 +4891,10 @@ static void lpt_pch_enable(const struct intel_atomic_state *state, assert_pch_transcoder_disabled(dev_priv, PIPE_A); - lpt_program_iclkip(crtc); + lpt_program_iclkip(crtc_state); /* Set transcoder timing. */ - ironlake_pch_transcoder_set_timings(crtc, PIPE_A); + ironlake_pch_transcoder_set_timings(crtc_state, PIPE_A); lpt_enable_pch_transcoder(dev_priv, cpu_transcoder); } @@ -4904,8 +4982,7 @@ static int skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach, unsigned int scaler_user, int *scaler_id, int src_w, int src_h, int dst_w, int dst_h, - bool plane_scaler_check, - uint32_t pixel_format) + const struct drm_format_info *format, bool need_scaler) { struct intel_crtc_scaler_state *scaler_state = &crtc_state->scaler_state; @@ -4914,21 +4991,14 @@ skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach, struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev); const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode; - int need_scaling; /* * Src coordinates are already rotated by 270 degrees for * the 90/270 degree plane rotation cases (to match the * GTT mapping), hence no need to account for rotation here. */ - need_scaling = src_w != dst_w || src_h != dst_h; - - if (plane_scaler_check) - if (pixel_format == DRM_FORMAT_NV12) - need_scaling = true; - - if (crtc_state->ycbcr420 && scaler_user == SKL_CRTC_INDEX) - need_scaling = true; + if (src_w != dst_w || src_h != dst_h) + need_scaler = true; /* * Scaling/fitting not supported in IF-ID mode in GEN9+ @@ -4937,7 +5007,7 @@ skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach, * for NV12. */ if (INTEL_GEN(dev_priv) >= 9 && crtc_state->base.enable && - need_scaling && adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) { + need_scaler && adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) { DRM_DEBUG_KMS("Pipe/Plane scaling not supported with IF-ID mode\n"); return -EINVAL; } @@ -4952,7 +5022,7 @@ skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach, * update to free the scaler is done in plane/panel-fit programming. * For this purpose crtc/plane_state->scaler_id isn't reset here. */ - if (force_detach || !need_scaling) { + if (force_detach || !need_scaler) { if (*scaler_id >= 0) { scaler_state->scaler_users &= ~(1 << scaler_user); scaler_state->scalers[*scaler_id].in_use = 0; @@ -4966,7 +5036,7 @@ skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach, return 0; } - if (plane_scaler_check && pixel_format == DRM_FORMAT_NV12 && + if (format && format->format == DRM_FORMAT_NV12 && (src_h < SKL_MIN_YUV_420_SRC_H || src_w < SKL_MIN_YUV_420_SRC_W)) { DRM_DEBUG_KMS("NV12: src dimensions not met\n"); return -EINVAL; @@ -4975,10 +5045,10 @@ skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach, /* range checks */ if (src_w < SKL_MIN_SRC_W || src_h < SKL_MIN_SRC_H || dst_w < SKL_MIN_DST_W || dst_h < SKL_MIN_DST_H || - (IS_GEN11(dev_priv) && + (IS_GEN(dev_priv, 11) && (src_w > ICL_MAX_SRC_W || src_h > ICL_MAX_SRC_H || dst_w > ICL_MAX_DST_W || dst_h > ICL_MAX_DST_H)) || - (!IS_GEN11(dev_priv) && + (!IS_GEN(dev_priv, 11) && (src_w > SKL_MAX_SRC_W || src_h > SKL_MAX_SRC_H || dst_w > SKL_MAX_DST_W || dst_h > SKL_MAX_DST_H))) { DRM_DEBUG_KMS("scaler_user index %u.%u: src %ux%u dst %ux%u " @@ -5009,12 +5079,16 @@ skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach, int skl_update_scaler_crtc(struct intel_crtc_state *state) { const struct drm_display_mode *adjusted_mode = &state->base.adjusted_mode; + bool need_scaler = false; + + if (state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420) + need_scaler = true; return skl_update_scaler(state, !state->base.active, SKL_CRTC_INDEX, &state->scaler_state.scaler_id, state->pipe_src_w, state->pipe_src_h, adjusted_mode->crtc_hdisplay, - adjusted_mode->crtc_vdisplay, false, 0); + adjusted_mode->crtc_vdisplay, NULL, need_scaler); } /** @@ -5029,13 +5103,17 @@ int skl_update_scaler_crtc(struct intel_crtc_state *state) static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state, struct intel_plane_state *plane_state) { - struct intel_plane *intel_plane = to_intel_plane(plane_state->base.plane); struct drm_framebuffer *fb = plane_state->base.fb; int ret; - bool force_detach = !fb || !plane_state->base.visible; + bool need_scaler = false; + + /* Pre-gen11 and SDR planes always need a scaler for planar formats. */ + if (!icl_is_hdr_plane(intel_plane) && + fb && fb->format->format == DRM_FORMAT_NV12) + need_scaler = true; ret = skl_update_scaler(crtc_state, force_detach, drm_plane_index(&intel_plane->base), @@ -5044,7 +5122,7 @@ static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state, drm_rect_height(&plane_state->base.src) >> 16, drm_rect_width(&plane_state->base.dst), drm_rect_height(&plane_state->base.dst), - fb ? true : false, fb ? fb->format->format : 0); + fb ? fb->format : NULL, need_scaler); if (ret || plane_state->scaler_id < 0) return ret; @@ -5090,27 +5168,27 @@ static void skylake_scaler_disable(struct intel_crtc *crtc) skl_detach_scaler(crtc, i); } -static void skylake_pfit_enable(struct intel_crtc *crtc) +static void skylake_pfit_enable(const struct intel_crtc_state *crtc_state) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - int pipe = crtc->pipe; - struct intel_crtc_scaler_state *scaler_state = - &crtc->config->scaler_state; + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; + const struct intel_crtc_scaler_state *scaler_state = + &crtc_state->scaler_state; - if (crtc->config->pch_pfit.enabled) { + if (crtc_state->pch_pfit.enabled) { u16 uv_rgb_hphase, uv_rgb_vphase; int pfit_w, pfit_h, hscale, vscale; int id; - if (WARN_ON(crtc->config->scaler_state.scaler_id < 0)) + if (WARN_ON(crtc_state->scaler_state.scaler_id < 0)) return; - pfit_w = (crtc->config->pch_pfit.size >> 16) & 0xFFFF; - pfit_h = crtc->config->pch_pfit.size & 0xFFFF; + pfit_w = (crtc_state->pch_pfit.size >> 16) & 0xFFFF; + pfit_h = crtc_state->pch_pfit.size & 0xFFFF; - hscale = (crtc->config->pipe_src_w << 16) / pfit_w; - vscale = (crtc->config->pipe_src_h << 16) / pfit_h; + hscale = (crtc_state->pipe_src_w << 16) / pfit_w; + vscale = (crtc_state->pipe_src_h << 16) / pfit_h; uv_rgb_hphase = skl_scaler_calc_phase(1, hscale, false); uv_rgb_vphase = skl_scaler_calc_phase(1, vscale, false); @@ -5122,18 +5200,18 @@ static void skylake_pfit_enable(struct intel_crtc *crtc) PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_vphase)); I915_WRITE_FW(SKL_PS_HPHASE(pipe, id), PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_hphase)); - I915_WRITE(SKL_PS_WIN_POS(pipe, id), crtc->config->pch_pfit.pos); - I915_WRITE(SKL_PS_WIN_SZ(pipe, id), crtc->config->pch_pfit.size); + I915_WRITE(SKL_PS_WIN_POS(pipe, id), crtc_state->pch_pfit.pos); + I915_WRITE(SKL_PS_WIN_SZ(pipe, id), crtc_state->pch_pfit.size); } } -static void ironlake_pfit_enable(struct intel_crtc *crtc) +static void ironlake_pfit_enable(const struct intel_crtc_state *crtc_state) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); int pipe = crtc->pipe; - if (crtc->config->pch_pfit.enabled) { + if (crtc_state->pch_pfit.enabled) { /* Force use of hard-coded filter coefficients * as some pre-programmed values are broken, * e.g. x201. @@ -5143,8 +5221,8 @@ static void ironlake_pfit_enable(struct intel_crtc *crtc) PF_PIPE_SEL_IVB(pipe)); else I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3); - I915_WRITE(PF_WIN_POS(pipe), crtc->config->pch_pfit.pos); - I915_WRITE(PF_WIN_SZ(pipe), crtc->config->pch_pfit.size); + I915_WRITE(PF_WIN_POS(pipe), crtc_state->pch_pfit.pos); + I915_WRITE(PF_WIN_SZ(pipe), crtc_state->pch_pfit.size); } } @@ -5261,7 +5339,7 @@ intel_post_enable_primary(struct drm_crtc *crtc, * FIXME: Need to fix the logic to work when we turn off all planes * but leave the pipe running. */ - if (IS_GEN2(dev_priv)) + if (IS_GEN(dev_priv, 2)) intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); /* Underruns don't always raise interrupts, so check manually. */ @@ -5282,7 +5360,7 @@ intel_pre_disable_primary_noatomic(struct drm_crtc *crtc) * Gen2 reports pipe underruns whenever all planes are disabled. * So disable underrun reporting before all the planes get disabled. */ - if (IS_GEN2(dev_priv)) + if (IS_GEN(dev_priv, 2)) intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false); hsw_disable_ips(to_intel_crtc_state(crtc->state)); @@ -5296,7 +5374,7 @@ intel_pre_disable_primary_noatomic(struct drm_crtc *crtc) * event which is after the vblank start event, so we need to have a * wait-for-vblank between disabling the plane and the pipe. */ - if (HAS_GMCH_DISPLAY(dev_priv) && + if (HAS_GMCH(dev_priv) && intel_set_memory_cxsr(dev_priv, false)) intel_wait_for_vblank(dev_priv, pipe); } @@ -5304,18 +5382,36 @@ intel_pre_disable_primary_noatomic(struct drm_crtc *crtc) static bool hsw_pre_update_disable_ips(const struct intel_crtc_state *old_crtc_state, const struct intel_crtc_state *new_crtc_state) { + struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + if (!old_crtc_state->ips_enabled) return false; if (needs_modeset(&new_crtc_state->base)) return true; + /* + * Workaround : Do not read or write the pipe palette/gamma data while + * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled. + * + * Disable IPS before we program the LUT. + */ + if (IS_HASWELL(dev_priv) && + (new_crtc_state->base.color_mgmt_changed || + new_crtc_state->update_pipe) && + new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT) + return true; + return !new_crtc_state->ips_enabled; } static bool hsw_post_update_enable_ips(const struct intel_crtc_state *old_crtc_state, const struct intel_crtc_state *new_crtc_state) { + struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + if (!new_crtc_state->ips_enabled) return false; @@ -5323,6 +5419,18 @@ static bool hsw_post_update_enable_ips(const struct intel_crtc_state *old_crtc_s return true; /* + * Workaround : Do not read or write the pipe palette/gamma data while + * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled. + * + * Re-enable IPS after the LUT has been programmed. + */ + if (IS_HASWELL(dev_priv) && + (new_crtc_state->base.color_mgmt_changed || + new_crtc_state->update_pipe) && + new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT) + return true; + + /* * We can't read out IPS on broadwell, assume the worst and * forcibly enable IPS on the first fastset. */ @@ -5339,11 +5447,8 @@ static bool needs_nv12_wa(struct drm_i915_private *dev_priv, if (!crtc_state->nv12_planes) return false; - if (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv)) - return false; - - if ((INTEL_GEN(dev_priv) == 9 && !IS_GEMINILAKE(dev_priv)) || - IS_CANNONLAKE(dev_priv)) + /* WA Display #0827: Gen9:all */ + if (IS_GEN(dev_priv, 9) && !IS_GEMINILAKE(dev_priv)) return true; return false; @@ -5386,7 +5491,6 @@ static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state) if (needs_nv12_wa(dev_priv, old_crtc_state) && !needs_nv12_wa(dev_priv, pipe_config)) { skl_wa_clkgate(dev_priv, crtc->pipe, false); - skl_wa_528(dev_priv, crtc->pipe, false); } } @@ -5417,7 +5521,7 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state, * Gen2 reports pipe underruns whenever all planes are disabled. * So disable underrun reporting before all the planes get disabled. */ - if (IS_GEN2(dev_priv) && old_primary_state->visible && + if (IS_GEN(dev_priv, 2) && old_primary_state->visible && (modeset || !new_primary_state->base.visible)) intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false); } @@ -5426,7 +5530,6 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state, if (!needs_nv12_wa(dev_priv, old_crtc_state) && needs_nv12_wa(dev_priv, pipe_config)) { skl_wa_clkgate(dev_priv, crtc->pipe, true); - skl_wa_528(dev_priv, crtc->pipe, true); } /* @@ -5438,7 +5541,7 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state, * event which is after the vblank start event, so we need to have a * wait-for-vblank between disabling the plane and the pipe. */ - if (HAS_GMCH_DISPLAY(dev_priv) && old_crtc_state->base.active && + if (HAS_GMCH(dev_priv) && old_crtc_state->base.active && pipe_config->disable_cxsr && intel_set_memory_cxsr(dev_priv, false)) intel_wait_for_vblank(dev_priv, crtc->pipe); @@ -5449,7 +5552,8 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state, * * WaCxSRDisabledForSpriteScaling:ivb */ - if (pipe_config->disable_lp_wm && ilk_disable_lp_wm(dev)) + if (pipe_config->disable_lp_wm && ilk_disable_lp_wm(dev) && + old_crtc_state->base.active) intel_wait_for_vblank(dev_priv, crtc->pipe); /* @@ -5480,24 +5584,32 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state, intel_update_watermarks(crtc); } -static void intel_crtc_disable_planes(struct drm_crtc *crtc, unsigned plane_mask) +static void intel_crtc_disable_planes(struct intel_atomic_state *state, + struct intel_crtc *crtc) { - struct drm_device *dev = crtc->dev; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct drm_plane *p; - int pipe = intel_crtc->pipe; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + const struct intel_crtc_state *new_crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + unsigned int update_mask = new_crtc_state->update_planes; + const struct intel_plane_state *old_plane_state; + struct intel_plane *plane; + unsigned fb_bits = 0; + int i; - intel_crtc_dpms_overlay_disable(intel_crtc); + intel_crtc_dpms_overlay_disable(crtc); - drm_for_each_plane_mask(p, dev, plane_mask) - to_intel_plane(p)->disable_plane(to_intel_plane(p), intel_crtc); + for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) { + if (crtc->pipe != plane->pipe || + !(update_mask & BIT(plane->id))) + continue; - /* - * FIXME: Once we grow proper nuclear flip support out of this we need - * to compute the mask of flip planes precisely. For the time being - * consider this a flip to a NULL plane. - */ - intel_frontbuffer_flip(to_i915(dev), INTEL_FRONTBUFFER_ALL_MASK(pipe)); + plane->disable_plane(plane, new_crtc_state); + + if (old_plane_state->base.visible) + fb_bits |= plane->frontbuffer_bit; + } + + intel_frontbuffer_flip(dev_priv, fb_bits); } static void intel_encoders_pre_pll_enable(struct drm_crtc *crtc, @@ -5555,7 +5667,8 @@ static void intel_encoders_enable(struct drm_crtc *crtc, if (conn_state->crtc != crtc) continue; - encoder->enable(encoder, crtc_state, conn_state); + if (encoder->enable) + encoder->enable(encoder, crtc_state, conn_state); intel_opregion_notify_encoder(encoder, true); } } @@ -5576,7 +5689,8 @@ static void intel_encoders_disable(struct drm_crtc *crtc, continue; intel_opregion_notify_encoder(encoder, false); - encoder->disable(encoder, old_crtc_state, old_conn_state); + if (encoder->disable) + encoder->disable(encoder, old_crtc_state, old_conn_state); } } @@ -5620,6 +5734,26 @@ static void intel_encoders_post_pll_disable(struct drm_crtc *crtc, } } +static void intel_encoders_update_pipe(struct drm_crtc *crtc, + struct intel_crtc_state *crtc_state, + struct drm_atomic_state *old_state) +{ + struct drm_connector_state *conn_state; + struct drm_connector *conn; + int i; + + for_each_new_connector_in_state(old_state, conn, conn_state, i) { + struct intel_encoder *encoder = + to_intel_encoder(conn_state->best_encoder); + + if (conn_state->crtc != crtc) + continue; + + if (encoder->update_pipe) + encoder->update_pipe(encoder, crtc_state, conn_state); + } +} + static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config, struct drm_atomic_state *old_state) { @@ -5647,53 +5781,54 @@ static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config, intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false); intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false); - if (intel_crtc->config->has_pch_encoder) - intel_prepare_shared_dpll(intel_crtc); + if (pipe_config->has_pch_encoder) + intel_prepare_shared_dpll(pipe_config); - if (intel_crtc_has_dp_encoder(intel_crtc->config)) - intel_dp_set_m_n(intel_crtc, M1_N1); + if (intel_crtc_has_dp_encoder(pipe_config)) + intel_dp_set_m_n(pipe_config, M1_N1); - intel_set_pipe_timings(intel_crtc); - intel_set_pipe_src_size(intel_crtc); + intel_set_pipe_timings(pipe_config); + intel_set_pipe_src_size(pipe_config); - if (intel_crtc->config->has_pch_encoder) { - intel_cpu_transcoder_set_m_n(intel_crtc, - &intel_crtc->config->fdi_m_n, NULL); + if (pipe_config->has_pch_encoder) { + intel_cpu_transcoder_set_m_n(pipe_config, + &pipe_config->fdi_m_n, NULL); } - ironlake_set_pipeconf(crtc); + ironlake_set_pipeconf(pipe_config); intel_crtc->active = true; intel_encoders_pre_enable(crtc, pipe_config, old_state); - if (intel_crtc->config->has_pch_encoder) { + if (pipe_config->has_pch_encoder) { /* Note: FDI PLL enabling _must_ be done before we enable the * cpu pipes, hence this is separate from all the other fdi/pch * enabling. */ - ironlake_fdi_pll_enable(intel_crtc); + ironlake_fdi_pll_enable(pipe_config); } else { assert_fdi_tx_disabled(dev_priv, pipe); assert_fdi_rx_disabled(dev_priv, pipe); } - ironlake_pfit_enable(intel_crtc); + ironlake_pfit_enable(pipe_config); /* * On ILK+ LUT must be loaded before the pipe is running but with * clocks enabled */ - intel_color_load_luts(&pipe_config->base); + intel_color_load_luts(pipe_config); + intel_color_commit(pipe_config); if (dev_priv->display.initial_watermarks != NULL) - dev_priv->display.initial_watermarks(old_intel_state, intel_crtc->config); + dev_priv->display.initial_watermarks(old_intel_state, pipe_config); intel_enable_pipe(pipe_config); - if (intel_crtc->config->has_pch_encoder) + if (pipe_config->has_pch_encoder) ironlake_pch_enable(old_intel_state, pipe_config); assert_vblank_disabled(crtc); - drm_crtc_vblank_on(crtc); + intel_crtc_vblank_on(pipe_config); intel_encoders_enable(crtc, pipe_config, old_state); @@ -5706,7 +5841,7 @@ static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config, * some interlaced HDMI modes. Let's do the double wait always * in case there are more corner cases we don't know about. */ - if (intel_crtc->config->has_pch_encoder) { + if (pipe_config->has_pch_encoder) { intel_wait_for_vblank(dev_priv, pipe); intel_wait_for_vblank(dev_priv, pipe); } @@ -5738,12 +5873,11 @@ static void icl_pipe_mbus_enable(struct intel_crtc *crtc) { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; - uint32_t val; - - val = MBUS_DBOX_BW_CREDIT(1) | MBUS_DBOX_A_CREDIT(2); + u32 val; - /* Program B credit equally to all pipes */ - val |= MBUS_DBOX_B_CREDIT(24 / INTEL_INFO(dev_priv)->num_pipes); + val = MBUS_DBOX_A_CREDIT(2); + val |= MBUS_DBOX_BW_CREDIT(1); + val |= MBUS_DBOX_B_CREDIT(8); I915_WRITE(PIPE_MBUS_DBOX_CTL(pipe), val); } @@ -5755,80 +5889,67 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config, struct drm_i915_private *dev_priv = to_i915(crtc->dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe, hsw_workaround_pipe; - enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder; + enum transcoder cpu_transcoder = pipe_config->cpu_transcoder; struct intel_atomic_state *old_intel_state = to_intel_atomic_state(old_state); bool psl_clkgate_wa; - u32 pipe_chicken; if (WARN_ON(intel_crtc->active)) return; intel_encoders_pre_pll_enable(crtc, pipe_config, old_state); - if (intel_crtc->config->shared_dpll) - intel_enable_shared_dpll(intel_crtc); - - if (INTEL_GEN(dev_priv) >= 11) - icl_map_plls_to_ports(crtc, pipe_config, old_state); + if (pipe_config->shared_dpll) + intel_enable_shared_dpll(pipe_config); intel_encoders_pre_enable(crtc, pipe_config, old_state); - if (intel_crtc_has_dp_encoder(intel_crtc->config)) - intel_dp_set_m_n(intel_crtc, M1_N1); + if (intel_crtc_has_dp_encoder(pipe_config)) + intel_dp_set_m_n(pipe_config, M1_N1); if (!transcoder_is_dsi(cpu_transcoder)) - intel_set_pipe_timings(intel_crtc); + intel_set_pipe_timings(pipe_config); - intel_set_pipe_src_size(intel_crtc); + intel_set_pipe_src_size(pipe_config); if (cpu_transcoder != TRANSCODER_EDP && !transcoder_is_dsi(cpu_transcoder)) { I915_WRITE(PIPE_MULT(cpu_transcoder), - intel_crtc->config->pixel_multiplier - 1); + pipe_config->pixel_multiplier - 1); } - if (intel_crtc->config->has_pch_encoder) { - intel_cpu_transcoder_set_m_n(intel_crtc, - &intel_crtc->config->fdi_m_n, NULL); + if (pipe_config->has_pch_encoder) { + intel_cpu_transcoder_set_m_n(pipe_config, + &pipe_config->fdi_m_n, NULL); } if (!transcoder_is_dsi(cpu_transcoder)) - haswell_set_pipeconf(crtc); - - haswell_set_pipemisc(crtc); + haswell_set_pipeconf(pipe_config); - intel_color_set_csc(&pipe_config->base); + haswell_set_pipemisc(pipe_config); intel_crtc->active = true; /* Display WA #1180: WaDisableScalarClockGating: glk, cnl */ psl_clkgate_wa = (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) && - intel_crtc->config->pch_pfit.enabled; + pipe_config->pch_pfit.enabled; if (psl_clkgate_wa) glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, true); if (INTEL_GEN(dev_priv) >= 9) - skylake_pfit_enable(intel_crtc); + skylake_pfit_enable(pipe_config); else - ironlake_pfit_enable(intel_crtc); + ironlake_pfit_enable(pipe_config); /* * On ILK+ LUT must be loaded before the pipe is running but with * clocks enabled */ - intel_color_load_luts(&pipe_config->base); + intel_color_load_luts(pipe_config); + intel_color_commit(pipe_config); - /* - * Display WA #1153: enable hardware to bypass the alpha math - * and rounding for per-pixel values 00 and 0xff - */ - if (INTEL_GEN(dev_priv) >= 11) { - pipe_chicken = I915_READ(PIPE_CHICKEN(pipe)); - if (!(pipe_chicken & PER_PIXEL_ALPHA_BYPASS_EN)) - I915_WRITE_FW(PIPE_CHICKEN(pipe), - pipe_chicken | PER_PIXEL_ALPHA_BYPASS_EN); - } + if (INTEL_GEN(dev_priv) >= 11) + icl_set_pipe_chicken(intel_crtc); intel_ddi_set_pipe_settings(pipe_config); if (!transcoder_is_dsi(cpu_transcoder)) @@ -5844,14 +5965,14 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config, if (!transcoder_is_dsi(cpu_transcoder)) intel_enable_pipe(pipe_config); - if (intel_crtc->config->has_pch_encoder) + if (pipe_config->has_pch_encoder) lpt_pch_enable(old_intel_state, pipe_config); - if (intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_DP_MST)) + if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DP_MST)) intel_ddi_set_vc_payload_alloc(pipe_config, true); assert_vblank_disabled(crtc); - drm_crtc_vblank_on(crtc); + intel_crtc_vblank_on(pipe_config); intel_encoders_enable(crtc, pipe_config, old_state); @@ -5869,15 +5990,15 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config, } } -static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force) +static void ironlake_pfit_disable(const struct intel_crtc_state *old_crtc_state) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - int pipe = crtc->pipe; + struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; /* To avoid upsetting the power well on haswell only disable the pfit if * it's in use. The hw state code will make sure we get this right. */ - if (force || crtc->config->pch_pfit.enabled) { + if (old_crtc_state->pch_pfit.enabled) { I915_WRITE(PF_CTL(pipe), 0); I915_WRITE(PF_WIN_POS(pipe), 0); I915_WRITE(PF_WIN_SZ(pipe), 0); @@ -5908,14 +6029,14 @@ static void ironlake_crtc_disable(struct intel_crtc_state *old_crtc_state, intel_disable_pipe(old_crtc_state); - ironlake_pfit_disable(intel_crtc, false); + ironlake_pfit_disable(old_crtc_state); - if (intel_crtc->config->has_pch_encoder) + if (old_crtc_state->has_pch_encoder) ironlake_fdi_disable(crtc); intel_encoders_post_disable(crtc, old_crtc_state, old_state); - if (intel_crtc->config->has_pch_encoder) { + if (old_crtc_state->has_pch_encoder) { ironlake_disable_pch_transcoder(dev_priv, pipe); if (HAS_PCH_CPT(dev_priv)) { @@ -5966,24 +6087,24 @@ static void haswell_crtc_disable(struct intel_crtc_state *old_crtc_state, if (!transcoder_is_dsi(cpu_transcoder)) intel_ddi_disable_transcoder_func(old_crtc_state); + intel_dsc_disable(old_crtc_state); + if (INTEL_GEN(dev_priv) >= 9) skylake_scaler_disable(intel_crtc); else - ironlake_pfit_disable(intel_crtc, false); + ironlake_pfit_disable(old_crtc_state); intel_encoders_post_disable(crtc, old_crtc_state, old_state); - if (INTEL_GEN(dev_priv) >= 11) - icl_unmap_plls_to_ports(crtc, old_crtc_state, old_state); + intel_encoders_post_pll_disable(crtc, old_crtc_state, old_state); } -static void i9xx_pfit_enable(struct intel_crtc *crtc) +static void i9xx_pfit_enable(const struct intel_crtc_state *crtc_state) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_crtc_state *pipe_config = crtc->config; + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - if (!pipe_config->gmch_pfit.control) + if (!crtc_state->gmch_pfit.control) return; /* @@ -5993,8 +6114,8 @@ static void i9xx_pfit_enable(struct intel_crtc *crtc) WARN_ON(I915_READ(PFIT_CONTROL) & PFIT_ENABLE); assert_pipe_disabled(dev_priv, crtc->pipe); - I915_WRITE(PFIT_PGM_RATIOS, pipe_config->gmch_pfit.pgm_ratios); - I915_WRITE(PFIT_CONTROL, pipe_config->gmch_pfit.control); + I915_WRITE(PFIT_PGM_RATIOS, crtc_state->gmch_pfit.pgm_ratios); + I915_WRITE(PFIT_CONTROL, crtc_state->gmch_pfit.control); /* Border color in case we don't scale up to the full screen. Black by * default, change to something else for debugging. */ @@ -6049,6 +6170,28 @@ enum intel_display_power_domain intel_port_to_power_domain(enum port port) } } +enum intel_display_power_domain +intel_aux_power_domain(struct intel_digital_port *dig_port) +{ + switch (dig_port->aux_ch) { + case AUX_CH_A: + return POWER_DOMAIN_AUX_A; + case AUX_CH_B: + return POWER_DOMAIN_AUX_B; + case AUX_CH_C: + return POWER_DOMAIN_AUX_C; + case AUX_CH_D: + return POWER_DOMAIN_AUX_D; + case AUX_CH_E: + return POWER_DOMAIN_AUX_E; + case AUX_CH_F: + return POWER_DOMAIN_AUX_F; + default: + MISSING_CASE(dig_port->aux_ch); + return POWER_DOMAIN_AUX_A; + } +} + static u64 get_crtc_power_domains(struct drm_crtc *crtc, struct intel_crtc_state *crtc_state) { @@ -6111,7 +6254,7 @@ static void modeset_put_power_domains(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain; for_each_power_domain(domain, domains) - intel_display_power_put(dev_priv, domain); + intel_display_power_put_unchecked(dev_priv, domain); } static void valleyview_crtc_enable(struct intel_crtc_state *pipe_config, @@ -6128,22 +6271,18 @@ static void valleyview_crtc_enable(struct intel_crtc_state *pipe_config, if (WARN_ON(intel_crtc->active)) return; - if (intel_crtc_has_dp_encoder(intel_crtc->config)) - intel_dp_set_m_n(intel_crtc, M1_N1); + if (intel_crtc_has_dp_encoder(pipe_config)) + intel_dp_set_m_n(pipe_config, M1_N1); - intel_set_pipe_timings(intel_crtc); - intel_set_pipe_src_size(intel_crtc); + intel_set_pipe_timings(pipe_config); + intel_set_pipe_src_size(pipe_config); if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) { - struct drm_i915_private *dev_priv = to_i915(dev); - I915_WRITE(CHV_BLEND(pipe), CHV_BLEND_LEGACY); I915_WRITE(CHV_CANVAS(pipe), 0); } - i9xx_set_pipeconf(intel_crtc); - - intel_color_set_csc(&pipe_config->base); + i9xx_set_pipeconf(pipe_config); intel_crtc->active = true; @@ -6152,36 +6291,37 @@ static void valleyview_crtc_enable(struct intel_crtc_state *pipe_config, intel_encoders_pre_pll_enable(crtc, pipe_config, old_state); if (IS_CHERRYVIEW(dev_priv)) { - chv_prepare_pll(intel_crtc, intel_crtc->config); - chv_enable_pll(intel_crtc, intel_crtc->config); + chv_prepare_pll(intel_crtc, pipe_config); + chv_enable_pll(intel_crtc, pipe_config); } else { - vlv_prepare_pll(intel_crtc, intel_crtc->config); - vlv_enable_pll(intel_crtc, intel_crtc->config); + vlv_prepare_pll(intel_crtc, pipe_config); + vlv_enable_pll(intel_crtc, pipe_config); } intel_encoders_pre_enable(crtc, pipe_config, old_state); - i9xx_pfit_enable(intel_crtc); + i9xx_pfit_enable(pipe_config); - intel_color_load_luts(&pipe_config->base); + intel_color_load_luts(pipe_config); + intel_color_commit(pipe_config); dev_priv->display.initial_watermarks(old_intel_state, pipe_config); intel_enable_pipe(pipe_config); assert_vblank_disabled(crtc); - drm_crtc_vblank_on(crtc); + intel_crtc_vblank_on(pipe_config); intel_encoders_enable(crtc, pipe_config, old_state); } -static void i9xx_set_pll_dividers(struct intel_crtc *crtc) +static void i9xx_set_pll_dividers(const struct intel_crtc_state *crtc_state) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - I915_WRITE(FP0(crtc->pipe), crtc->config->dpll_hw_state.fp0); - I915_WRITE(FP1(crtc->pipe), crtc->config->dpll_hw_state.fp1); + I915_WRITE(FP0(crtc->pipe), crtc_state->dpll_hw_state.fp0); + I915_WRITE(FP1(crtc->pipe), crtc_state->dpll_hw_state.fp1); } static void i9xx_crtc_enable(struct intel_crtc_state *pipe_config, @@ -6198,48 +6338,49 @@ static void i9xx_crtc_enable(struct intel_crtc_state *pipe_config, if (WARN_ON(intel_crtc->active)) return; - i9xx_set_pll_dividers(intel_crtc); + i9xx_set_pll_dividers(pipe_config); - if (intel_crtc_has_dp_encoder(intel_crtc->config)) - intel_dp_set_m_n(intel_crtc, M1_N1); + if (intel_crtc_has_dp_encoder(pipe_config)) + intel_dp_set_m_n(pipe_config, M1_N1); - intel_set_pipe_timings(intel_crtc); - intel_set_pipe_src_size(intel_crtc); + intel_set_pipe_timings(pipe_config); + intel_set_pipe_src_size(pipe_config); - i9xx_set_pipeconf(intel_crtc); + i9xx_set_pipeconf(pipe_config); intel_crtc->active = true; - if (!IS_GEN2(dev_priv)) + if (!IS_GEN(dev_priv, 2)) intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); intel_encoders_pre_enable(crtc, pipe_config, old_state); i9xx_enable_pll(intel_crtc, pipe_config); - i9xx_pfit_enable(intel_crtc); + i9xx_pfit_enable(pipe_config); - intel_color_load_luts(&pipe_config->base); + intel_color_load_luts(pipe_config); + intel_color_commit(pipe_config); if (dev_priv->display.initial_watermarks != NULL) dev_priv->display.initial_watermarks(old_intel_state, - intel_crtc->config); + pipe_config); else intel_update_watermarks(intel_crtc); intel_enable_pipe(pipe_config); assert_vblank_disabled(crtc); - drm_crtc_vblank_on(crtc); + intel_crtc_vblank_on(pipe_config); intel_encoders_enable(crtc, pipe_config, old_state); } -static void i9xx_pfit_disable(struct intel_crtc *crtc) +static void i9xx_pfit_disable(const struct intel_crtc_state *old_crtc_state) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - if (!crtc->config->gmch_pfit.control) + if (!old_crtc_state->gmch_pfit.control) return; assert_pipe_disabled(dev_priv, crtc->pipe); @@ -6262,7 +6403,7 @@ static void i9xx_crtc_disable(struct intel_crtc_state *old_crtc_state, * On gen2 planes are double buffered but the pipe isn't, so we must * wait for planes to fully turn off before disabling the pipe. */ - if (IS_GEN2(dev_priv)) + if (IS_GEN(dev_priv, 2)) intel_wait_for_vblank(dev_priv, pipe); intel_encoders_disable(crtc, old_crtc_state, old_state); @@ -6272,22 +6413,22 @@ static void i9xx_crtc_disable(struct intel_crtc_state *old_crtc_state, intel_disable_pipe(old_crtc_state); - i9xx_pfit_disable(intel_crtc); + i9xx_pfit_disable(old_crtc_state); intel_encoders_post_disable(crtc, old_crtc_state, old_state); - if (!intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_DSI)) { + if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DSI)) { if (IS_CHERRYVIEW(dev_priv)) chv_disable_pll(dev_priv, pipe); else if (IS_VALLEYVIEW(dev_priv)) vlv_disable_pll(dev_priv, pipe); else - i9xx_disable_pll(intel_crtc); + i9xx_disable_pll(old_crtc_state); } intel_encoders_post_pll_disable(crtc, old_crtc_state, old_state); - if (!IS_GEN2(dev_priv)) + if (!IS_GEN(dev_priv, 2)) intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false); if (!dev_priv->display.initial_watermarks) @@ -6356,11 +6497,11 @@ static void intel_crtc_disable_noatomic(struct drm_crtc *crtc, intel_fbc_disable(intel_crtc); intel_update_watermarks(intel_crtc); - intel_disable_shared_dpll(intel_crtc); + intel_disable_shared_dpll(to_intel_crtc_state(crtc->state)); domains = intel_crtc->enabled_power_domains; for_each_power_domain(domain, domains) - intel_display_power_put(dev_priv, domain); + intel_display_power_put_unchecked(dev_priv, domain); intel_crtc->enabled_power_domains = 0; dev_priv->active_crtcs &= ~(1 << intel_crtc->pipe); @@ -6434,66 +6575,6 @@ static void intel_connector_verify_state(struct drm_crtc_state *crtc_state, } } -int intel_connector_init(struct intel_connector *connector) -{ - struct intel_digital_connector_state *conn_state; - - /* - * Allocate enough memory to hold intel_digital_connector_state, - * This might be a few bytes too many, but for connectors that don't - * need it we'll free the state and allocate a smaller one on the first - * succesful commit anyway. - */ - conn_state = kzalloc(sizeof(*conn_state), GFP_KERNEL); - if (!conn_state) - return -ENOMEM; - - __drm_atomic_helper_connector_reset(&connector->base, - &conn_state->base); - - return 0; -} - -struct intel_connector *intel_connector_alloc(void) -{ - struct intel_connector *connector; - - connector = kzalloc(sizeof *connector, GFP_KERNEL); - if (!connector) - return NULL; - - if (intel_connector_init(connector) < 0) { - kfree(connector); - return NULL; - } - - return connector; -} - -/* - * Free the bits allocated by intel_connector_alloc. - * This should only be used after intel_connector_alloc has returned - * successfully, and before drm_connector_init returns successfully. - * Otherwise the destroy callbacks for the connector and the state should - * take care of proper cleanup/free - */ -void intel_connector_free(struct intel_connector *connector) -{ - kfree(to_intel_digital_connector_state(connector->base.state)); - kfree(connector); -} - -/* Simple connector->get_hw_state implementation for encoders that support only - * one connector and no cloning and hence the encoder state determines the state - * of the connector. */ -bool intel_connector_get_hw_state(struct intel_connector *connector) -{ - enum pipe pipe = 0; - struct intel_encoder *encoder = connector->encoder; - - return encoder->get_hw_state(encoder, &pipe); -} - static int pipe_required_fdi_lanes(struct intel_crtc_state *crtc_state) { if (crtc_state->base.enable && crtc_state->has_pch_encoder) @@ -6604,6 +6685,9 @@ retry: link_bw, &pipe_config->fdi_m_n, false); ret = ironlake_check_fdi_lanes(dev, intel_crtc->pipe, pipe_config); + if (ret == -EDEADLK) + return ret; + if (ret == -EINVAL && pipe_config->pipe_bpp > 6*3) { pipe_config->pipe_bpp -= 2*3; DRM_DEBUG_KMS("fdi link bw constraint, reducing pipe bpp to %i\n", @@ -6683,9 +6767,9 @@ static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc) (crtc->pipe == PIPE_A || IS_I915G(dev_priv)); } -static uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config) +static u32 ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config) { - uint32_t pixel_rate; + u32 pixel_rate; pixel_rate = pipe_config->base.adjusted_mode.crtc_clock; @@ -6695,8 +6779,8 @@ static uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config) */ if (pipe_config->pch_pfit.enabled) { - uint64_t pipe_w, pipe_h, pfit_w, pfit_h; - uint32_t pfit_size = pipe_config->pch_pfit.size; + u64 pipe_w, pipe_h, pfit_w, pfit_h; + u32 pfit_size = pipe_config->pch_pfit.size; pipe_w = pipe_config->pipe_src_w; pipe_h = pipe_config->pipe_src_h; @@ -6711,7 +6795,7 @@ static uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config) if (WARN_ON(!pfit_w || !pfit_h)) return pixel_rate; - pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h, + pixel_rate = div_u64((u64)pixel_rate * pipe_w * pipe_h, pfit_w * pfit_h); } @@ -6722,7 +6806,7 @@ static void intel_crtc_compute_pixel_rate(struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); - if (HAS_GMCH_DISPLAY(dev_priv)) + if (HAS_GMCH(dev_priv)) /* FIXME calculate proper pipe pixel rate for GMCH pfit */ crtc_state->pixel_rate = crtc_state->base.adjusted_mode.crtc_clock; @@ -6760,7 +6844,9 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc, return -EINVAL; } - if (pipe_config->ycbcr420 && pipe_config->base.ctm) { + if ((pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 || + pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR444) && + pipe_config->base.ctm) { /* * There is only one pipe CSC unit per pipe, and we need that * for output conversion from RGB->YCBCR. So if CTM is already @@ -6805,7 +6891,7 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc, } static void -intel_reduce_m_n_ratio(uint32_t *num, uint32_t *den) +intel_reduce_m_n_ratio(u32 *num, u32 *den) { while (*num > DATA_LINK_M_N_MASK || *den > DATA_LINK_M_N_MASK) { @@ -6815,7 +6901,7 @@ intel_reduce_m_n_ratio(uint32_t *num, uint32_t *den) } static void compute_m_n(unsigned int m, unsigned int n, - uint32_t *ret_m, uint32_t *ret_n, + u32 *ret_m, u32 *ret_n, bool constant_n) { /* @@ -6830,12 +6916,12 @@ static void compute_m_n(unsigned int m, unsigned int n, else *ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX); - *ret_m = div_u64((uint64_t) m * *ret_n, n); + *ret_m = div_u64((u64)m * *ret_n, n); intel_reduce_m_n_ratio(ret_m, ret_n); } void -intel_link_compute_m_n(int bits_per_pixel, int nlanes, +intel_link_compute_m_n(u16 bits_per_pixel, int nlanes, int pixel_clock, int link_clock, struct intel_link_m_n *m_n, bool constant_n) @@ -6860,12 +6946,12 @@ static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv) && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE); } -static uint32_t pnv_dpll_compute_fp(struct dpll *dpll) +static u32 pnv_dpll_compute_fp(struct dpll *dpll) { return (1 << dpll->n) << 16 | dpll->m2; } -static uint32_t i9xx_dpll_compute_fp(struct dpll *dpll) +static u32 i9xx_dpll_compute_fp(struct dpll *dpll) { return dpll->n << 16 | dpll->m1 << 8 | dpll->m2; } @@ -6926,12 +7012,12 @@ static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv, enum pipe vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val); } -static void intel_pch_transcoder_set_m_n(struct intel_crtc *crtc, - struct intel_link_m_n *m_n) +static void intel_pch_transcoder_set_m_n(const struct intel_crtc_state *crtc_state, + const struct intel_link_m_n *m_n) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - int pipe = crtc->pipe; + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; I915_WRITE(PCH_TRANS_DATA_M1(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m); I915_WRITE(PCH_TRANS_DATA_N1(pipe), m_n->gmch_n); @@ -6939,25 +7025,39 @@ static void intel_pch_transcoder_set_m_n(struct intel_crtc *crtc, I915_WRITE(PCH_TRANS_LINK_N1(pipe), m_n->link_n); } -static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc, - struct intel_link_m_n *m_n, - struct intel_link_m_n *m2_n2) +static bool transcoder_has_m2_n2(struct drm_i915_private *dev_priv, + enum transcoder transcoder) +{ + if (IS_HASWELL(dev_priv)) + return transcoder == TRANSCODER_EDP; + + /* + * Strictly speaking some registers are available before + * gen7, but we only support DRRS on gen7+ + */ + return IS_GEN(dev_priv, 7) || IS_CHERRYVIEW(dev_priv); +} + +static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state, + const struct intel_link_m_n *m_n, + const struct intel_link_m_n *m2_n2) { + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - int pipe = crtc->pipe; - enum transcoder transcoder = crtc->config->cpu_transcoder; + enum pipe pipe = crtc->pipe; + enum transcoder transcoder = crtc_state->cpu_transcoder; if (INTEL_GEN(dev_priv) >= 5) { I915_WRITE(PIPE_DATA_M1(transcoder), TU_SIZE(m_n->tu) | m_n->gmch_m); I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n); I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m); I915_WRITE(PIPE_LINK_N1(transcoder), m_n->link_n); - /* M2_N2 registers to be set only for gen < 8 (M2_N2 available - * for gen < 8) and if DRRS is supported (to make sure the - * registers are not unnecessarily accessed). + /* + * M2_N2 registers are set only if DRRS is supported + * (to make sure the registers are not unnecessarily accessed). */ - if (m2_n2 && (IS_CHERRYVIEW(dev_priv) || - INTEL_GEN(dev_priv) < 8) && crtc->config->has_drrs) { + if (m2_n2 && crtc_state->has_drrs && + transcoder_has_m2_n2(dev_priv, transcoder)) { I915_WRITE(PIPE_DATA_M2(transcoder), TU_SIZE(m2_n2->tu) | m2_n2->gmch_m); I915_WRITE(PIPE_DATA_N2(transcoder), m2_n2->gmch_n); @@ -6972,29 +7072,29 @@ static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc, } } -void intel_dp_set_m_n(struct intel_crtc *crtc, enum link_m_n_set m_n) +void intel_dp_set_m_n(const struct intel_crtc_state *crtc_state, enum link_m_n_set m_n) { - struct intel_link_m_n *dp_m_n, *dp_m2_n2 = NULL; + const struct intel_link_m_n *dp_m_n, *dp_m2_n2 = NULL; if (m_n == M1_N1) { - dp_m_n = &crtc->config->dp_m_n; - dp_m2_n2 = &crtc->config->dp_m2_n2; + dp_m_n = &crtc_state->dp_m_n; + dp_m2_n2 = &crtc_state->dp_m2_n2; } else if (m_n == M2_N2) { /* * M2_N2 registers are not supported. Hence m2_n2 divider value * needs to be programmed into M1_N1. */ - dp_m_n = &crtc->config->dp_m2_n2; + dp_m_n = &crtc_state->dp_m2_n2; } else { DRM_ERROR("Unsupported divider value\n"); return; } - if (crtc->config->has_pch_encoder) - intel_pch_transcoder_set_m_n(crtc, &crtc->config->dp_m_n); + if (crtc_state->has_pch_encoder) + intel_pch_transcoder_set_m_n(crtc_state, &crtc_state->dp_m_n); else - intel_cpu_transcoder_set_m_n(crtc, dp_m_n, dp_m2_n2); + intel_cpu_transcoder_set_m_n(crtc_state, dp_m_n, dp_m2_n2); } static void vlv_compute_dpll(struct intel_crtc *crtc, @@ -7093,8 +7193,8 @@ static void vlv_prepare_pll(struct intel_crtc *crtc, /* Set HBR and RBR LPF coefficients */ if (pipe_config->port_clock == 162000 || - intel_crtc_has_type(crtc->config, INTEL_OUTPUT_ANALOG) || - intel_crtc_has_type(crtc->config, INTEL_OUTPUT_HDMI)) + intel_crtc_has_type(pipe_config, INTEL_OUTPUT_ANALOG) || + intel_crtc_has_type(pipe_config, INTEL_OUTPUT_HDMI)) vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe), 0x009f0003); else @@ -7121,7 +7221,7 @@ static void vlv_prepare_pll(struct intel_crtc *crtc, coreclk = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW7(pipe)); coreclk = (coreclk & 0x0000ff00) | 0x01c00000; - if (intel_crtc_has_dp_encoder(crtc->config)) + if (intel_crtc_has_dp_encoder(pipe_config)) coreclk |= 0x01000000; vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk); @@ -7400,13 +7500,14 @@ static void i8xx_compute_dpll(struct intel_crtc *crtc, crtc_state->dpll_hw_state.dpll = dpll; } -static void intel_set_pipe_timings(struct intel_crtc *intel_crtc) +static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev); - enum pipe pipe = intel_crtc->pipe; - enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder; - const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode; - uint32_t crtc_vtotal, crtc_vblank_end; + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; + enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; + const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode; + u32 crtc_vtotal, crtc_vblank_end; int vsyncshift = 0; /* We need to be careful not to changed the adjusted mode, for otherwise @@ -7419,7 +7520,7 @@ static void intel_set_pipe_timings(struct intel_crtc *intel_crtc) crtc_vtotal -= 1; crtc_vblank_end -= 1; - if (intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_SDVO)) + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO)) vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2; else vsyncshift = adjusted_mode->crtc_hsync_start - @@ -7461,18 +7562,18 @@ static void intel_set_pipe_timings(struct intel_crtc *intel_crtc) } -static void intel_set_pipe_src_size(struct intel_crtc *intel_crtc) +static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state) { - struct drm_device *dev = intel_crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - enum pipe pipe = intel_crtc->pipe; + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; /* pipesrc controls the size that is scaled from, which should * always be the user's requested size. */ I915_WRITE(PIPESRC(pipe), - ((intel_crtc->config->pipe_src_w - 1) << 16) | - (intel_crtc->config->pipe_src_h - 1)); + ((crtc_state->pipe_src_w - 1) << 16) | + (crtc_state->pipe_src_h - 1)); } static void intel_get_pipe_timings(struct intel_crtc *crtc, @@ -7481,7 +7582,7 @@ static void intel_get_pipe_timings(struct intel_crtc *crtc, struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = to_i915(dev); enum transcoder cpu_transcoder = pipe_config->cpu_transcoder; - uint32_t tmp; + u32 tmp; tmp = I915_READ(HTOTAL(cpu_transcoder)); pipe_config->base.adjusted_mode.crtc_hdisplay = (tmp & 0xffff) + 1; @@ -7548,29 +7649,30 @@ void intel_mode_from_pipe_config(struct drm_display_mode *mode, drm_mode_set_name(mode); } -static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc) +static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev); - uint32_t pipeconf; + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + u32 pipeconf; pipeconf = 0; /* we keep both pipes enabled on 830 */ if (IS_I830(dev_priv)) - pipeconf |= I915_READ(PIPECONF(intel_crtc->pipe)) & PIPECONF_ENABLE; + pipeconf |= I915_READ(PIPECONF(crtc->pipe)) & PIPECONF_ENABLE; - if (intel_crtc->config->double_wide) + if (crtc_state->double_wide) pipeconf |= PIPECONF_DOUBLE_WIDE; /* only g4x and later have fancy bpc/dither controls */ if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { /* Bspec claims that we can't use dithering for 30bpp pipes. */ - if (intel_crtc->config->dither && intel_crtc->config->pipe_bpp != 30) + if (crtc_state->dither && crtc_state->pipe_bpp != 30) pipeconf |= PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP; - switch (intel_crtc->config->pipe_bpp) { + switch (crtc_state->pipe_bpp) { case 18: pipeconf |= PIPECONF_6BPC; break; @@ -7586,9 +7688,9 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc) } } - if (intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) { + if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) { if (INTEL_GEN(dev_priv) < 4 || - intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_SDVO)) + intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO)) pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION; else pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT; @@ -7596,11 +7698,11 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc) pipeconf |= PIPECONF_PROGRESSIVE; if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) && - intel_crtc->config->limited_color_range) + crtc_state->limited_color_range) pipeconf |= PIPECONF_COLOR_RANGE_SELECT; - I915_WRITE(PIPECONF(intel_crtc->pipe), pipeconf); - POSTING_READ(PIPECONF(intel_crtc->pipe)); + I915_WRITE(PIPECONF(crtc->pipe), pipeconf); + POSTING_READ(PIPECONF(crtc->pipe)); } static int i8xx_crtc_compute_clock(struct intel_crtc *crtc, @@ -7796,7 +7898,7 @@ static void i9xx_get_pfit_config(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - uint32_t tmp; + u32 tmp; if (INTEL_GEN(dev_priv) <= 3 && (IS_I830(dev_priv) || !IS_MOBILE(dev_priv))) @@ -7963,18 +8065,64 @@ static void chv_crtc_clock_get(struct intel_crtc *crtc, pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock); } +static void intel_get_crtc_ycbcr_config(struct intel_crtc *crtc, + struct intel_crtc_state *pipe_config) +{ + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum intel_output_format output = INTEL_OUTPUT_FORMAT_RGB; + + pipe_config->lspcon_downsampling = false; + + if (IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) >= 9) { + u32 tmp = I915_READ(PIPEMISC(crtc->pipe)); + + if (tmp & PIPEMISC_OUTPUT_COLORSPACE_YUV) { + bool ycbcr420_enabled = tmp & PIPEMISC_YUV420_ENABLE; + bool blend = tmp & PIPEMISC_YUV420_MODE_FULL_BLEND; + + if (ycbcr420_enabled) { + /* We support 4:2:0 in full blend mode only */ + if (!blend) + output = INTEL_OUTPUT_FORMAT_INVALID; + else if (!(IS_GEMINILAKE(dev_priv) || + INTEL_GEN(dev_priv) >= 10)) + output = INTEL_OUTPUT_FORMAT_INVALID; + else + output = INTEL_OUTPUT_FORMAT_YCBCR420; + } else { + /* + * Currently there is no interface defined to + * check user preference between RGB/YCBCR444 + * or YCBCR420. So the only possible case for + * YCBCR444 usage is driving YCBCR420 output + * with LSPCON, when pipe is configured for + * YCBCR444 output and LSPCON takes care of + * downsampling it. + */ + pipe_config->lspcon_downsampling = true; + output = INTEL_OUTPUT_FORMAT_YCBCR444; + } + } + } + + pipe_config->output_format = output; +} + static bool i9xx_get_pipe_config(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum intel_display_power_domain power_domain; - uint32_t tmp; + intel_wakeref_t wakeref; + u32 tmp; bool ret; power_domain = POWER_DOMAIN_PIPE(crtc->pipe); - if (!intel_display_power_get_if_enabled(dev_priv, power_domain)) + wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); + if (!wakeref) return false; + pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe; pipe_config->shared_dpll = NULL; @@ -8072,7 +8220,7 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc, ret = true; out: - intel_display_power_put(dev_priv, power_domain); + intel_display_power_put(dev_priv, power_domain, wakeref); return ret; } @@ -8246,7 +8394,7 @@ static void ironlake_init_pch_refclk(struct drm_i915_private *dev_priv) static void lpt_reset_fdi_mphy(struct drm_i915_private *dev_priv) { - uint32_t tmp; + u32 tmp; tmp = I915_READ(SOUTH_CHICKEN2); tmp |= FDI_MPHY_IOSFSB_RESET_CTL; @@ -8268,7 +8416,7 @@ static void lpt_reset_fdi_mphy(struct drm_i915_private *dev_priv) /* WaMPhyProgramming:hsw */ static void lpt_program_fdi_mphy(struct drm_i915_private *dev_priv) { - uint32_t tmp; + u32 tmp; tmp = intel_sbi_read(dev_priv, 0x8008, SBI_MPHY); tmp &= ~(0xFF << 24); @@ -8349,7 +8497,7 @@ static void lpt_program_fdi_mphy(struct drm_i915_private *dev_priv) static void lpt_enable_clkout_dp(struct drm_i915_private *dev_priv, bool with_spread, bool with_fdi) { - uint32_t reg, tmp; + u32 reg, tmp; if (WARN(with_fdi && !with_spread, "FDI requires downspread\n")) with_spread = true; @@ -8388,7 +8536,7 @@ static void lpt_enable_clkout_dp(struct drm_i915_private *dev_priv, /* Sequence to disable CLKOUT_DP */ static void lpt_disable_clkout_dp(struct drm_i915_private *dev_priv) { - uint32_t reg, tmp; + u32 reg, tmp; mutex_lock(&dev_priv->sb_lock); @@ -8413,7 +8561,7 @@ static void lpt_disable_clkout_dp(struct drm_i915_private *dev_priv) #define BEND_IDX(steps) ((50 + (steps)) / 5) -static const uint16_t sscdivintphase[] = { +static const u16 sscdivintphase[] = { [BEND_IDX( 50)] = 0x3B23, [BEND_IDX( 45)] = 0x3B23, [BEND_IDX( 40)] = 0x3C23, @@ -8445,7 +8593,7 @@ static const uint16_t sscdivintphase[] = { */ static void lpt_bend_clkout_dp(struct drm_i915_private *dev_priv, int steps) { - uint32_t tmp; + u32 tmp; int idx = BEND_IDX(steps); if (WARN_ON(steps % 5 != 0)) @@ -8506,16 +8654,16 @@ void intel_init_pch_refclk(struct drm_i915_private *dev_priv) lpt_init_pch_refclk(dev_priv); } -static void ironlake_set_pipeconf(struct drm_crtc *crtc) +static void ironlake_set_pipeconf(const struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = to_i915(crtc->dev); - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - int pipe = intel_crtc->pipe; - uint32_t val; + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; + u32 val; val = 0; - switch (intel_crtc->config->pipe_bpp) { + switch (crtc_state->pipe_bpp) { case 18: val |= PIPECONF_6BPC; break; @@ -8533,32 +8681,32 @@ static void ironlake_set_pipeconf(struct drm_crtc *crtc) BUG(); } - if (intel_crtc->config->dither) + if (crtc_state->dither) val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP); - if (intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) + if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) val |= PIPECONF_INTERLACED_ILK; else val |= PIPECONF_PROGRESSIVE; - if (intel_crtc->config->limited_color_range) + if (crtc_state->limited_color_range) val |= PIPECONF_COLOR_RANGE_SELECT; I915_WRITE(PIPECONF(pipe), val); POSTING_READ(PIPECONF(pipe)); } -static void haswell_set_pipeconf(struct drm_crtc *crtc) +static void haswell_set_pipeconf(const struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = to_i915(crtc->dev); - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder; + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; u32 val = 0; - if (IS_HASWELL(dev_priv) && intel_crtc->config->dither) + if (IS_HASWELL(dev_priv) && crtc_state->dither) val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP); - if (intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) + if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) val |= PIPECONF_INTERLACED_ILK; else val |= PIPECONF_PROGRESSIVE; @@ -8567,16 +8715,15 @@ static void haswell_set_pipeconf(struct drm_crtc *crtc) POSTING_READ(PIPECONF(cpu_transcoder)); } -static void haswell_set_pipemisc(struct drm_crtc *crtc) +static void haswell_set_pipemisc(const struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = to_i915(crtc->dev); - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct intel_crtc_state *config = intel_crtc->config; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev); if (IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) >= 9) { u32 val = 0; - switch (intel_crtc->config->pipe_bpp) { + switch (crtc_state->pipe_bpp) { case 18: val |= PIPEMISC_DITHER_6_BPC; break; @@ -8594,14 +8741,16 @@ static void haswell_set_pipemisc(struct drm_crtc *crtc) BUG(); } - if (intel_crtc->config->dither) + if (crtc_state->dither) val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP; - if (config->ycbcr420) { - val |= PIPEMISC_OUTPUT_COLORSPACE_YUV | - PIPEMISC_YUV420_ENABLE | + if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 || + crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444) + val |= PIPEMISC_OUTPUT_COLORSPACE_YUV; + + if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420) + val |= PIPEMISC_YUV420_ENABLE | PIPEMISC_YUV420_MODE_FULL_BLEND; - } I915_WRITE(PIPEMISC(intel_crtc->pipe), val); } @@ -8812,12 +8961,8 @@ static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc, m_n->gmch_n = I915_READ(PIPE_DATA_N1(transcoder)); m_n->tu = ((I915_READ(PIPE_DATA_M1(transcoder)) & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1; - /* Read M2_N2 registers only for gen < 8 (M2_N2 available for - * gen < 8) and if DRRS is supported (to make sure the - * registers are not unnecessarily read). - */ - if (m2_n2 && INTEL_GEN(dev_priv) < 8 && - crtc->config->has_drrs) { + + if (m2_n2 && transcoder_has_m2_n2(dev_priv, transcoder)) { m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder)); m2_n2->link_n = I915_READ(PIPE_LINK_N2(transcoder)); m2_n2->gmch_m = I915_READ(PIPE_DATA_M2(transcoder)) @@ -8861,7 +9006,7 @@ static void skylake_get_pfit_config(struct intel_crtc *crtc, struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc_scaler_state *scaler_state = &pipe_config->scaler_state; - uint32_t ps_ctrl = 0; + u32 ps_ctrl = 0; int id = -1; int i; @@ -8873,6 +9018,7 @@ static void skylake_get_pfit_config(struct intel_crtc *crtc, pipe_config->pch_pfit.enabled = true; pipe_config->pch_pfit.pos = I915_READ(SKL_PS_WIN_POS(crtc->pipe, i)); pipe_config->pch_pfit.size = I915_READ(SKL_PS_WIN_SZ(crtc->pipe, i)); + scaler_state->scalers[i].in_use = true; break; } } @@ -8993,7 +9139,7 @@ skylake_get_initial_plane_config(struct intel_crtc *crtc, fb->width = ((val >> 0) & 0x1fff) + 1; val = I915_READ(PLANE_STRIDE(pipe, plane_id)); - stride_mult = intel_fb_stride_alignment(fb, 0); + stride_mult = skl_plane_stride_mult(fb, 0, DRM_MODE_ROTATE_0); fb->pitches[0] = (val & 0x3ff) * stride_mult; aligned_height = intel_fb_align_height(fb, 0, fb->height); @@ -9017,7 +9163,7 @@ static void ironlake_get_pfit_config(struct intel_crtc *crtc, { struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = to_i915(dev); - uint32_t tmp; + u32 tmp; tmp = I915_READ(PF_CTL(crtc->pipe)); @@ -9029,7 +9175,7 @@ static void ironlake_get_pfit_config(struct intel_crtc *crtc, /* We currently do not free assignements of panel fitters on * ivb/hsw (since we don't use the higher upscaling modes which * differentiates them) so just WARN about this case for now. */ - if (IS_GEN7(dev_priv)) { + if (IS_GEN(dev_priv, 7)) { WARN_ON((tmp & PF_PIPE_SEL_MASK_IVB) != PF_PIPE_SEL_IVB(crtc->pipe)); } @@ -9042,13 +9188,16 @@ static bool ironlake_get_pipe_config(struct intel_crtc *crtc, struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = to_i915(dev); enum intel_display_power_domain power_domain; - uint32_t tmp; + intel_wakeref_t wakeref; + u32 tmp; bool ret; power_domain = POWER_DOMAIN_PIPE(crtc->pipe); - if (!intel_display_power_get_if_enabled(dev_priv, power_domain)) + wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); + if (!wakeref) return false; + pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe; pipe_config->shared_dpll = NULL; @@ -9128,7 +9277,7 @@ static bool ironlake_get_pipe_config(struct intel_crtc *crtc, ret = true; out: - intel_display_power_put(dev_priv, power_domain); + intel_display_power_put(dev_priv, power_domain, wakeref); return ret; } @@ -9168,7 +9317,7 @@ static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv) I915_STATE_WARN(intel_irqs_enabled(dev_priv), "IRQs enabled\n"); } -static uint32_t hsw_read_dcomp(struct drm_i915_private *dev_priv) +static u32 hsw_read_dcomp(struct drm_i915_private *dev_priv) { if (IS_HASWELL(dev_priv)) return I915_READ(D_COMP_HSW); @@ -9176,7 +9325,7 @@ static uint32_t hsw_read_dcomp(struct drm_i915_private *dev_priv) return I915_READ(D_COMP_BDW); } -static void hsw_write_dcomp(struct drm_i915_private *dev_priv, uint32_t val) +static void hsw_write_dcomp(struct drm_i915_private *dev_priv, u32 val) { if (IS_HASWELL(dev_priv)) { mutex_lock(&dev_priv->pcu_lock); @@ -9201,7 +9350,7 @@ static void hsw_write_dcomp(struct drm_i915_private *dev_priv, uint32_t val) static void hsw_disable_lcpll(struct drm_i915_private *dev_priv, bool switch_to_fclk, bool allow_power_down) { - uint32_t val; + u32 val; assert_can_disable_lcpll(dev_priv); @@ -9248,7 +9397,7 @@ static void hsw_disable_lcpll(struct drm_i915_private *dev_priv, */ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv) { - uint32_t val; + u32 val; val = I915_READ(LCPLL_CTL); @@ -9323,7 +9472,7 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv) */ void hsw_enable_pc8(struct drm_i915_private *dev_priv) { - uint32_t val; + u32 val; DRM_DEBUG_KMS("Enabling package C8+\n"); @@ -9339,7 +9488,7 @@ void hsw_enable_pc8(struct drm_i915_private *dev_priv) void hsw_disable_pc8(struct drm_i915_private *dev_priv) { - uint32_t val; + u32 val; DRM_DEBUG_KMS("Disabling package C8+\n"); @@ -9356,10 +9505,12 @@ void hsw_disable_pc8(struct drm_i915_private *dev_priv) static int haswell_crtc_compute_clock(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state) { + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->base.state); - if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI)) { + if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) || + IS_ICELAKE(dev_priv)) { struct intel_encoder *encoder = intel_get_crtc_new_encoder(state, crtc_state); @@ -9397,30 +9548,17 @@ static void icelake_get_ddi_pll(struct drm_i915_private *dev_priv, u32 temp; /* TODO: TBT pll not implemented. */ - switch (port) { - case PORT_A: - case PORT_B: + if (intel_port_is_combophy(dev_priv, port)) { temp = I915_READ(DPCLKA_CFGCR0_ICL) & DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port); id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port); - if (WARN_ON(id != DPLL_ID_ICL_DPLL0 && id != DPLL_ID_ICL_DPLL1)) + if (WARN_ON(!intel_dpll_is_combophy(id))) return; - break; - case PORT_C: - id = DPLL_ID_ICL_MGPLL1; - break; - case PORT_D: - id = DPLL_ID_ICL_MGPLL2; - break; - case PORT_E: - id = DPLL_ID_ICL_MGPLL3; - break; - case PORT_F: - id = DPLL_ID_ICL_MGPLL4; - break; - default: - MISSING_CASE(port); + } else if (intel_port_is_tc(dev_priv, port)) { + id = icl_tc_port_to_pll_id(intel_port_to_tc(dev_priv, port)); + } else { + WARN(1, "Invalid port %x\n", port); return; } @@ -9472,7 +9610,7 @@ static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv, struct intel_crtc_state *pipe_config) { enum intel_dpll_id id; - uint32_t ddi_pll_sel = I915_READ(PORT_CLK_SEL(port)); + u32 ddi_pll_sel = I915_READ(PORT_CLK_SEL(port)); switch (ddi_pll_sel) { case PORT_CLK_SEL_WRPLL1: @@ -9510,11 +9648,18 @@ static bool hsw_get_transcoder_state(struct intel_crtc *crtc, struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = to_i915(dev); enum intel_display_power_domain power_domain; + unsigned long panel_transcoder_mask = BIT(TRANSCODER_EDP); + unsigned long enabled_panel_transcoders = 0; + enum transcoder panel_transcoder; u32 tmp; + if (IS_ICELAKE(dev_priv)) + panel_transcoder_mask |= + BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1); + /* * The pipe->transcoder mapping is fixed with the exception of the eDP - * transcoder handled below. + * and DSI transcoders handled below. */ pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe; @@ -9522,32 +9667,56 @@ static bool hsw_get_transcoder_state(struct intel_crtc *crtc, * XXX: Do intel_display_power_get_if_enabled before reading this (for * consistency and less surprising code; it's in always on power). */ - tmp = I915_READ(TRANS_DDI_FUNC_CTL(TRANSCODER_EDP)); - if (tmp & TRANS_DDI_FUNC_ENABLE) { - enum pipe trans_edp_pipe; + for_each_set_bit(panel_transcoder, + &panel_transcoder_mask, + ARRAY_SIZE(INTEL_INFO(dev_priv)->trans_offsets)) { + enum pipe trans_pipe; + + tmp = I915_READ(TRANS_DDI_FUNC_CTL(panel_transcoder)); + if (!(tmp & TRANS_DDI_FUNC_ENABLE)) + continue; + + /* + * Log all enabled ones, only use the first one. + * + * FIXME: This won't work for two separate DSI displays. + */ + enabled_panel_transcoders |= BIT(panel_transcoder); + if (enabled_panel_transcoders != BIT(panel_transcoder)) + continue; + switch (tmp & TRANS_DDI_EDP_INPUT_MASK) { default: - WARN(1, "unknown pipe linked to edp transcoder\n"); + WARN(1, "unknown pipe linked to transcoder %s\n", + transcoder_name(panel_transcoder)); /* fall through */ case TRANS_DDI_EDP_INPUT_A_ONOFF: case TRANS_DDI_EDP_INPUT_A_ON: - trans_edp_pipe = PIPE_A; + trans_pipe = PIPE_A; break; case TRANS_DDI_EDP_INPUT_B_ONOFF: - trans_edp_pipe = PIPE_B; + trans_pipe = PIPE_B; break; case TRANS_DDI_EDP_INPUT_C_ONOFF: - trans_edp_pipe = PIPE_C; + trans_pipe = PIPE_C; break; } - if (trans_edp_pipe == crtc->pipe) - pipe_config->cpu_transcoder = TRANSCODER_EDP; + if (trans_pipe == crtc->pipe) + pipe_config->cpu_transcoder = panel_transcoder; } + /* + * Valid combos: none, eDP, DSI0, DSI1, DSI0+DSI1 + */ + WARN_ON((enabled_panel_transcoders & BIT(TRANSCODER_EDP)) && + enabled_panel_transcoders != BIT(TRANSCODER_EDP)); + power_domain = POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder); if (!intel_display_power_get_if_enabled(dev_priv, power_domain)) return false; + + WARN_ON(*power_domain_mask & BIT_ULL(power_domain)); *power_domain_mask |= BIT_ULL(power_domain); tmp = I915_READ(PIPECONF(pipe_config->cpu_transcoder)); @@ -9575,6 +9744,8 @@ static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc, power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder); if (!intel_display_power_get_if_enabled(dev_priv, power_domain)) continue; + + WARN_ON(*power_domain_mask & BIT_ULL(power_domain)); *power_domain_mask |= BIT_ULL(power_domain); /* @@ -9609,7 +9780,7 @@ static void haswell_get_ddi_port_state(struct intel_crtc *crtc, struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_shared_dpll *pll; enum port port; - uint32_t tmp; + u32 tmp; tmp = I915_READ(TRANS_DDI_FUNC_CTL(pipe_config->cpu_transcoder)); @@ -9677,36 +9848,23 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc, if (!active) goto out; - if (!transcoder_is_dsi(pipe_config->cpu_transcoder)) { + if (!transcoder_is_dsi(pipe_config->cpu_transcoder) || + IS_ICELAKE(dev_priv)) { haswell_get_ddi_port_state(crtc, pipe_config); intel_get_pipe_timings(crtc, pipe_config); } intel_get_pipe_src_size(crtc, pipe_config); + intel_get_crtc_ycbcr_config(crtc, pipe_config); pipe_config->gamma_mode = I915_READ(GAMMA_MODE(crtc->pipe)) & GAMMA_MODE_MODE_MASK; - if (IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) >= 9) { - u32 tmp = I915_READ(PIPEMISC(crtc->pipe)); - bool clrspace_yuv = tmp & PIPEMISC_OUTPUT_COLORSPACE_YUV; - - if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10) { - bool blend_mode_420 = tmp & - PIPEMISC_YUV420_MODE_FULL_BLEND; - - pipe_config->ycbcr420 = tmp & PIPEMISC_YUV420_ENABLE; - if (pipe_config->ycbcr420 != clrspace_yuv || - pipe_config->ycbcr420 != blend_mode_420) - DRM_DEBUG_KMS("Bad 4:2:0 mode (%08x)\n", tmp); - } else if (clrspace_yuv) { - DRM_DEBUG_KMS("YCbCr 4:2:0 Unsupported\n"); - } - } - power_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe); if (intel_display_power_get_if_enabled(dev_priv, power_domain)) { + WARN_ON(power_domain_mask & BIT_ULL(power_domain)); power_domain_mask |= BIT_ULL(power_domain); + if (INTEL_GEN(dev_priv) >= 9) skylake_get_pfit_config(crtc, pipe_config); else @@ -9736,7 +9894,7 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc, out: for_each_power_domain(power_domain, power_domain_mask) - intel_display_power_put(dev_priv, power_domain); + intel_display_power_put_unchecked(dev_priv, power_domain); return active; } @@ -9749,7 +9907,7 @@ static u32 intel_cursor_base(const struct intel_plane_state *plane_state) const struct drm_i915_gem_object *obj = intel_fb_obj(fb); u32 base; - if (INTEL_INFO(dev_priv)->cursor_needs_physical) + if (INTEL_INFO(dev_priv)->display.cursor_needs_physical) base = obj->phys_handle->busaddr; else base = intel_plane_ggtt_offset(plane_state); @@ -9757,7 +9915,7 @@ static u32 intel_cursor_base(const struct intel_plane_state *plane_state) base += plane_state->color_plane[0].offset; /* ILK+ do this automagically */ - if (HAS_GMCH_DISPLAY(dev_priv) && + if (HAS_GMCH(dev_priv) && plane_state->base.rotation & DRM_MODE_ROTATE_180) base += (plane_state->base.crtc_h * plane_state->base.crtc_w - 1) * fb->format->cpp[0]; @@ -9870,11 +10028,15 @@ i845_cursor_max_stride(struct intel_plane *plane, return 2048; } +static u32 i845_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state) +{ + return CURSOR_GAMMA_ENABLE; +} + static u32 i845_cursor_ctl(const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state) { return CURSOR_ENABLE | - CURSOR_GAMMA_ENABLE | CURSOR_FORMAT_ARGB | CURSOR_STRIDE(plane_state->color_plane[0].stride); } @@ -9944,7 +10106,9 @@ static void i845_update_cursor(struct intel_plane *plane, unsigned int width = plane_state->base.crtc_w; unsigned int height = plane_state->base.crtc_h; - cntl = plane_state->ctl; + cntl = plane_state->ctl | + i845_cursor_ctl_crtc(crtc_state); + size = (height << 12) | width; base = intel_cursor_base(plane_state); @@ -9972,15 +10136,13 @@ static void i845_update_cursor(struct intel_plane *plane, I915_WRITE_FW(CURPOS(PIPE_A), pos); } - POSTING_READ_FW(CURCNTR(PIPE_A)); - spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); } static void i845_disable_cursor(struct intel_plane *plane, - struct intel_crtc *crtc) + const struct intel_crtc_state *crtc_state) { - i845_update_cursor(plane, NULL, NULL); + i845_update_cursor(plane, crtc_state, NULL); } static bool i845_cursor_get_hw_state(struct intel_plane *plane, @@ -9988,17 +10150,19 @@ static bool i845_cursor_get_hw_state(struct intel_plane *plane, { struct drm_i915_private *dev_priv = to_i915(plane->base.dev); enum intel_display_power_domain power_domain; + intel_wakeref_t wakeref; bool ret; power_domain = POWER_DOMAIN_PIPE(PIPE_A); - if (!intel_display_power_get_if_enabled(dev_priv, power_domain)) + wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); + if (!wakeref) return false; ret = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE; *pipe = PIPE_A; - intel_display_power_put(dev_priv, power_domain); + intel_display_power_put(dev_priv, power_domain, wakeref); return ret; } @@ -10011,27 +10175,36 @@ i9xx_cursor_max_stride(struct intel_plane *plane, return plane->base.dev->mode_config.cursor_width * 4; } -static u32 i9xx_cursor_ctl(const struct intel_crtc_state *crtc_state, - const struct intel_plane_state *plane_state) +static u32 i9xx_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = - to_i915(plane_state->base.plane->dev); struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); u32 cntl = 0; - if (IS_GEN6(dev_priv) || IS_IVYBRIDGE(dev_priv)) - cntl |= MCURSOR_TRICKLE_FEED_DISABLE; + if (INTEL_GEN(dev_priv) >= 11) + return cntl; - if (INTEL_GEN(dev_priv) <= 10) { - cntl |= MCURSOR_GAMMA_ENABLE; + cntl |= MCURSOR_GAMMA_ENABLE; - if (HAS_DDI(dev_priv)) - cntl |= MCURSOR_PIPE_CSC_ENABLE; - } + if (HAS_DDI(dev_priv)) + cntl |= MCURSOR_PIPE_CSC_ENABLE; if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) cntl |= MCURSOR_PIPE_SELECT(crtc->pipe); + return cntl; +} + +static u32 i9xx_cursor_ctl(const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state) +{ + struct drm_i915_private *dev_priv = + to_i915(plane_state->base.plane->dev); + u32 cntl = 0; + + if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv)) + cntl |= MCURSOR_TRICKLE_FEED_DISABLE; + switch (plane_state->base.crtc_w) { case 64: cntl |= MCURSOR_MODE_64_ARGB_AX; @@ -10156,7 +10329,8 @@ static void i9xx_update_cursor(struct intel_plane *plane, unsigned long irqflags; if (plane_state && plane_state->base.visible) { - cntl = plane_state->ctl; + cntl = plane_state->ctl | + i9xx_cursor_ctl_crtc(crtc_state); if (plane_state->base.crtc_h != plane_state->base.crtc_w) fbc_ctl = CUR_FBC_CTL_EN | (plane_state->base.crtc_h - 1); @@ -10171,8 +10345,8 @@ static void i9xx_update_cursor(struct intel_plane *plane, * On some platforms writing CURCNTR first will also * cause CURPOS to be armed by the CURBASE write. * Without the CURCNTR write the CURPOS write would - * arm itself. Thus we always start the full update - * with a CURCNTR write. + * arm itself. Thus we always update CURCNTR before + * CURPOS. * * On other platforms CURPOS always requires the * CURBASE write to arm the update. Additonally @@ -10182,15 +10356,20 @@ static void i9xx_update_cursor(struct intel_plane *plane, * cursor that doesn't appear to move, or even change * shape. Thus we always write CURBASE. * - * CURCNTR and CUR_FBC_CTL are always - * armed by the CURBASE write only. + * The other registers are armed by by the CURBASE write + * except when the plane is getting enabled at which time + * the CURCNTR write arms the update. */ + + if (INTEL_GEN(dev_priv) >= 9) + skl_write_cursor_wm(plane, crtc_state); + if (plane->cursor.base != base || plane->cursor.size != fbc_ctl || plane->cursor.cntl != cntl) { - I915_WRITE_FW(CURCNTR(pipe), cntl); if (HAS_CUR_FBC(dev_priv)) I915_WRITE_FW(CUR_FBC_CTL(pipe), fbc_ctl); + I915_WRITE_FW(CURCNTR(pipe), cntl); I915_WRITE_FW(CURPOS(pipe), pos); I915_WRITE_FW(CURBASE(pipe), base); @@ -10202,15 +10381,13 @@ static void i9xx_update_cursor(struct intel_plane *plane, I915_WRITE_FW(CURBASE(pipe), base); } - POSTING_READ_FW(CURBASE(pipe)); - spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); } static void i9xx_disable_cursor(struct intel_plane *plane, - struct intel_crtc *crtc) + const struct intel_crtc_state *crtc_state) { - i9xx_update_cursor(plane, NULL, NULL); + i9xx_update_cursor(plane, crtc_state, NULL); } static bool i9xx_cursor_get_hw_state(struct intel_plane *plane, @@ -10218,6 +10395,7 @@ static bool i9xx_cursor_get_hw_state(struct intel_plane *plane, { struct drm_i915_private *dev_priv = to_i915(plane->base.dev); enum intel_display_power_domain power_domain; + intel_wakeref_t wakeref; bool ret; u32 val; @@ -10227,7 +10405,8 @@ static bool i9xx_cursor_get_hw_state(struct intel_plane *plane, * display power wells. */ power_domain = POWER_DOMAIN_PIPE(plane->pipe); - if (!intel_display_power_get_if_enabled(dev_priv, power_domain)) + wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); + if (!wakeref) return false; val = I915_READ(CURCNTR(plane->pipe)); @@ -10240,7 +10419,7 @@ static bool i9xx_cursor_get_hw_state(struct intel_plane *plane, *pipe = (val & MCURSOR_PIPE_SELECT_MASK) >> MCURSOR_PIPE_SELECT_SHIFT; - intel_display_power_put(dev_priv, power_domain); + intel_display_power_put(dev_priv, power_domain, wakeref); return ret; } @@ -10489,7 +10668,7 @@ static int i9xx_pll_refclk(struct drm_device *dev, return dev_priv->vbt.lvds_ssc_freq; else if (HAS_PCH_SPLIT(dev_priv)) return 120000; - else if (!IS_GEN2(dev_priv)) + else if (!IS_GEN(dev_priv, 2)) return 96000; else return 48000; @@ -10522,7 +10701,7 @@ static void i9xx_crtc_clock_get(struct intel_crtc *crtc, clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT; } - if (!IS_GEN2(dev_priv)) { + if (!IS_GEN(dev_priv, 2)) { if (IS_PINEVIEW(dev_priv)) clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >> DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW); @@ -10674,20 +10853,17 @@ static void intel_crtc_destroy(struct drm_crtc *crtc) /** * intel_wm_need_update - Check whether watermarks need updating - * @plane: drm plane - * @state: new plane state + * @cur: current plane state + * @new: new plane state * * Check current plane state versus the new one to determine whether * watermarks need to be recalculated. * * Returns true or false. */ -static bool intel_wm_need_update(struct drm_plane *plane, - struct drm_plane_state *state) +static bool intel_wm_need_update(struct intel_plane_state *cur, + struct intel_plane_state *new) { - struct intel_plane_state *new = to_intel_plane_state(state); - struct intel_plane_state *cur = to_intel_plane_state(plane->state); - /* Update watermarks on tiling or size changes. */ if (new->base.visible != cur->base.visible) return true; @@ -10796,7 +10972,8 @@ int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_stat /* must disable cxsr around plane enable/disable */ if (plane->id != PLANE_CURSOR) pipe_config->disable_cxsr = true; - } else if (intel_wm_need_update(&plane->base, plane_state)) { + } else if (intel_wm_need_update(to_intel_plane_state(plane->base.state), + to_intel_plane_state(plane_state))) { if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) { /* FIXME bollocks */ pipe_config->update_wm_pre = true; @@ -10808,14 +10985,43 @@ int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_stat pipe_config->fb_bits |= plane->frontbuffer_bit; /* + * ILK/SNB DVSACNTR/Sprite Enable + * IVB SPR_CTL/Sprite Enable + * "When in Self Refresh Big FIFO mode, a write to enable the + * plane will be internally buffered and delayed while Big FIFO + * mode is exiting." + * + * Which means that enabling the sprite can take an extra frame + * when we start in big FIFO mode (LP1+). Thus we need to drop + * down to LP0 and wait for vblank in order to make sure the + * sprite gets enabled on the next vblank after the register write. + * Doing otherwise would risk enabling the sprite one frame after + * we've already signalled flip completion. We can resume LP1+ + * once the sprite has been enabled. + * + * * WaCxSRDisabledForSpriteScaling:ivb + * IVB SPR_SCALE/Scaling Enable + * "Low Power watermarks must be disabled for at least one + * frame before enabling sprite scaling, and kept disabled + * until sprite scaling is disabled." + * + * ILK/SNB DVSASCALE/Scaling Enable + * "When in Self Refresh Big FIFO mode, scaling enable will be + * masked off while Big FIFO mode is exiting." + * + * Despite the w/a only being listed for IVB we assume that + * the ILK/SNB note has similar ramifications, hence we apply + * the w/a on all three platforms. * - * cstate->update_wm was already set above, so this flag will - * take effect when we commit and program watermarks. + * With experimental results seems this is needed also for primary + * plane, not only sprite plane. */ - if (plane->id == PLANE_SPRITE0 && IS_IVYBRIDGE(dev_priv) && - needs_scaling(to_intel_plane_state(plane_state)) && - !needs_scaling(old_plane_state)) + if (plane->id != PLANE_CURSOR && + (IS_GEN_RANGE(dev_priv, 5, 6) || + IS_IVYBRIDGE(dev_priv)) && + (turn_on || (!needs_scaling(old_plane_state) && + needs_scaling(to_intel_plane_state(plane_state))))) pipe_config->disable_lp_wm = true; return 0; @@ -10851,19 +11057,113 @@ static bool check_single_encoder_cloning(struct drm_atomic_state *state, return true; } +static int icl_add_linked_planes(struct intel_atomic_state *state) +{ + struct intel_plane *plane, *linked; + struct intel_plane_state *plane_state, *linked_plane_state; + int i; + + for_each_new_intel_plane_in_state(state, plane, plane_state, i) { + linked = plane_state->linked_plane; + + if (!linked) + continue; + + linked_plane_state = intel_atomic_get_plane_state(state, linked); + if (IS_ERR(linked_plane_state)) + return PTR_ERR(linked_plane_state); + + WARN_ON(linked_plane_state->linked_plane != plane); + WARN_ON(linked_plane_state->slave == plane_state->slave); + } + + return 0; +} + +static int icl_check_nv12_planes(struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->base.state); + struct intel_plane *plane, *linked; + struct intel_plane_state *plane_state; + int i; + + if (INTEL_GEN(dev_priv) < 11) + return 0; + + /* + * Destroy all old plane links and make the slave plane invisible + * in the crtc_state->active_planes mask. + */ + for_each_new_intel_plane_in_state(state, plane, plane_state, i) { + if (plane->pipe != crtc->pipe || !plane_state->linked_plane) + continue; + + plane_state->linked_plane = NULL; + if (plane_state->slave && !plane_state->base.visible) { + crtc_state->active_planes &= ~BIT(plane->id); + crtc_state->update_planes |= BIT(plane->id); + } + + plane_state->slave = false; + } + + if (!crtc_state->nv12_planes) + return 0; + + for_each_new_intel_plane_in_state(state, plane, plane_state, i) { + struct intel_plane_state *linked_state = NULL; + + if (plane->pipe != crtc->pipe || + !(crtc_state->nv12_planes & BIT(plane->id))) + continue; + + for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, linked) { + if (!icl_is_nv12_y_plane(linked->id)) + continue; + + if (crtc_state->active_planes & BIT(linked->id)) + continue; + + linked_state = intel_atomic_get_plane_state(state, linked); + if (IS_ERR(linked_state)) + return PTR_ERR(linked_state); + + break; + } + + if (!linked_state) { + DRM_DEBUG_KMS("Need %d free Y planes for NV12\n", + hweight8(crtc_state->nv12_planes)); + + return -EINVAL; + } + + plane_state->linked_plane = linked; + + linked_state->slave = true; + linked_state->linked_plane = plane; + crtc_state->active_planes |= BIT(linked->id); + crtc_state->update_planes |= BIT(linked->id); + DRM_DEBUG_KMS("Using %s as Y plane for %s\n", linked->base.name, plane->base.name); + } + + return 0; +} + static int intel_crtc_atomic_check(struct drm_crtc *crtc, struct drm_crtc_state *crtc_state) { - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(crtc->dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_crtc_state *pipe_config = to_intel_crtc_state(crtc_state); - struct drm_atomic_state *state = crtc_state->state; int ret; bool mode_changed = needs_modeset(crtc_state); - if (mode_changed && !crtc_state->active) + if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv) && + mode_changed && !crtc_state->active) pipe_config->update_wm_post = true; if (mode_changed && crtc_state->enable && @@ -10875,8 +11175,8 @@ static int intel_crtc_atomic_check(struct drm_crtc *crtc, return ret; } - if (crtc_state->color_mgmt_changed) { - ret = intel_color_check(crtc, crtc_state); + if (mode_changed || crtc_state->color_mgmt_changed) { + ret = intel_color_check(pipe_config); if (ret) return ret; @@ -10896,8 +11196,7 @@ static int intel_crtc_atomic_check(struct drm_crtc *crtc, } } - if (dev_priv->display.compute_intermediate_wm && - !to_intel_atomic_state(state)->skip_intermediate_wm) { + if (dev_priv->display.compute_intermediate_wm) { if (WARN_ON(!dev_priv->display.compute_pipe_wm)) return 0; @@ -10906,23 +11205,20 @@ static int intel_crtc_atomic_check(struct drm_crtc *crtc, * old state and the new state. We can program these * immediately. */ - ret = dev_priv->display.compute_intermediate_wm(dev, - intel_crtc, - pipe_config); + ret = dev_priv->display.compute_intermediate_wm(pipe_config); if (ret) { DRM_DEBUG_KMS("No valid intermediate pipe watermarks are possible\n"); return ret; } - } else if (dev_priv->display.compute_intermediate_wm) { - if (HAS_PCH_SPLIT(dev_priv) && INTEL_GEN(dev_priv) < 9) - pipe_config->wm.ilk.intermediate = pipe_config->wm.ilk.optimal; } if (INTEL_GEN(dev_priv) >= 9) { - if (mode_changed) + if (mode_changed || pipe_config->update_pipe) ret = skl_update_scaler_crtc(pipe_config); if (!ret) + ret = icl_check_nv12_planes(pipe_config); + if (!ret) ret = skl_check_pipe_max_pixel_rate(intel_crtc, pipe_config); if (!ret) @@ -10937,8 +11233,6 @@ static int intel_crtc_atomic_check(struct drm_crtc *crtc, } static const struct drm_crtc_helper_funcs intel_helper_funcs = { - .atomic_begin = intel_begin_crtc_commit, - .atomic_flush = intel_finish_crtc_commit, .atomic_check = intel_crtc_atomic_check, }; @@ -10967,30 +11261,42 @@ static void intel_modeset_update_connector_atomic_state(struct drm_device *dev) drm_connector_list_iter_end(&conn_iter); } -static void -connected_sink_compute_bpp(struct intel_connector *connector, - struct intel_crtc_state *pipe_config) +static int +compute_sink_pipe_bpp(const struct drm_connector_state *conn_state, + struct intel_crtc_state *pipe_config) { - const struct drm_display_info *info = &connector->base.display_info; - int bpp = pipe_config->pipe_bpp; - - DRM_DEBUG_KMS("[CONNECTOR:%d:%s] checking for sink bpp constrains\n", - connector->base.base.id, - connector->base.name); + struct drm_connector *connector = conn_state->connector; + const struct drm_display_info *info = &connector->display_info; + int bpp; - /* Don't use an invalid EDID bpc value */ - if (info->bpc != 0 && info->bpc * 3 < bpp) { - DRM_DEBUG_KMS("clamping display bpp (was %d) to EDID reported max of %d\n", - bpp, info->bpc * 3); - pipe_config->pipe_bpp = info->bpc * 3; + switch (conn_state->max_bpc) { + case 6 ... 7: + bpp = 6 * 3; + break; + case 8 ... 9: + bpp = 8 * 3; + break; + case 10 ... 11: + bpp = 10 * 3; + break; + case 12: + bpp = 12 * 3; + break; + default: + return -EINVAL; } - /* Clamp bpp to 8 on screens without EDID 1.4 */ - if (info->bpc == 0 && bpp > 24) { - DRM_DEBUG_KMS("clamping display bpp (was %d) to default limit of 24\n", - bpp); - pipe_config->pipe_bpp = 24; + if (bpp < pipe_config->pipe_bpp) { + DRM_DEBUG_KMS("[CONNECTOR:%d:%s] Limiting display bpp to %d instead of " + "EDID bpp %d, requested bpp %d, max platform bpp %d\n", + connector->base.id, connector->name, + bpp, 3 * info->bpc, 3 * conn_state->max_requested_bpc, + pipe_config->pipe_bpp); + + pipe_config->pipe_bpp = bpp; } + + return 0; } static int @@ -10998,7 +11304,7 @@ compute_baseline_pipe_bpp(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - struct drm_atomic_state *state; + struct drm_atomic_state *state = pipe_config->base.state; struct drm_connector *connector; struct drm_connector_state *connector_state; int bpp, i; @@ -11011,21 +11317,21 @@ compute_baseline_pipe_bpp(struct intel_crtc *crtc, else bpp = 8*3; - pipe_config->pipe_bpp = bpp; - state = pipe_config->base.state; - - /* Clamp display bpp to EDID value */ + /* Clamp display bpp to connector max bpp */ for_each_new_connector_in_state(state, connector, connector_state, i) { + int ret; + if (connector_state->crtc != &crtc->base) continue; - connected_sink_compute_bpp(to_intel_connector(connector), - pipe_config); + ret = compute_sink_pipe_bpp(connector_state, pipe_config); + if (ret) + return ret; } - return bpp; + return 0; } static void intel_dump_crtc_timings(const struct drm_display_mode *mode) @@ -11095,6 +11401,20 @@ static void snprintf_output_types(char *buf, size_t len, WARN_ON_ONCE(output_types != 0); } +static const char * const output_format_str[] = { + [INTEL_OUTPUT_FORMAT_INVALID] = "Invalid", + [INTEL_OUTPUT_FORMAT_RGB] = "RGB", + [INTEL_OUTPUT_FORMAT_YCBCR420] = "YCBCR4:2:0", + [INTEL_OUTPUT_FORMAT_YCBCR444] = "YCBCR4:4:4", +}; + +static const char *output_formats(enum intel_output_format format) +{ + if (format >= ARRAY_SIZE(output_format_str)) + format = INTEL_OUTPUT_FORMAT_INVALID; + return output_format_str[format]; +} + static void intel_dump_pipe_config(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config, const char *context) @@ -11114,6 +11434,9 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc, DRM_DEBUG_KMS("output_types: %s (0x%x)\n", buf, pipe_config->output_types); + DRM_DEBUG_KMS("output format: %s\n", + output_formats(pipe_config->output_format)); + DRM_DEBUG_KMS("cpu_transcoder: %s, pipe bpp: %i, dithering: %i\n", transcoder_name(pipe_config->cpu_transcoder), pipe_config->pipe_bpp, pipe_config->dither); @@ -11123,9 +11446,6 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc, pipe_config->fdi_lanes, &pipe_config->fdi_m_n); - if (pipe_config->ycbcr420) - DRM_DEBUG_KMS("YCbCr 4:2:0 output enabled\n"); - if (intel_crtc_has_dp_encoder(pipe_config)) { intel_dump_m_n_config(pipe_config, "dp m_n", pipe_config->lane_count, &pipe_config->dp_m_n); @@ -11154,7 +11474,7 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc, pipe_config->scaler_state.scaler_users, pipe_config->scaler_state.scaler_id); - if (HAS_GMCH_DISPLAY(dev_priv)) + if (HAS_GMCH(dev_priv)) DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n", pipe_config->gmch_pfit.control, pipe_config->gmch_pfit.pgm_ratios, @@ -11266,44 +11586,38 @@ static bool check_digital_port_conflicts(struct drm_atomic_state *state) return ret; } -static void +static int clear_intel_crtc_state(struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); - struct intel_crtc_scaler_state scaler_state; - struct intel_dpll_hw_state dpll_hw_state; - struct intel_shared_dpll *shared_dpll; - struct intel_crtc_wm_state wm_state; - bool force_thru, ips_force_disable; + struct intel_crtc_state *saved_state; + + saved_state = kzalloc(sizeof(*saved_state), GFP_KERNEL); + if (!saved_state) + return -ENOMEM; /* FIXME: before the switch to atomic started, a new pipe_config was * kzalloc'd. Code that depends on any field being zero should be * fixed, so that the crtc_state can be safely duplicated. For now, * only fields that are know to not cause problems are preserved. */ - scaler_state = crtc_state->scaler_state; - shared_dpll = crtc_state->shared_dpll; - dpll_hw_state = crtc_state->dpll_hw_state; - force_thru = crtc_state->pch_pfit.force_thru; - ips_force_disable = crtc_state->ips_force_disable; + saved_state->scaler_state = crtc_state->scaler_state; + saved_state->shared_dpll = crtc_state->shared_dpll; + saved_state->dpll_hw_state = crtc_state->dpll_hw_state; + saved_state->pch_pfit.force_thru = crtc_state->pch_pfit.force_thru; + saved_state->ips_force_disable = crtc_state->ips_force_disable; if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) - wm_state = crtc_state->wm; + saved_state->wm = crtc_state->wm; /* Keep base drm_crtc_state intact, only clear our extended struct */ BUILD_BUG_ON(offsetof(struct intel_crtc_state, base)); - memset(&crtc_state->base + 1, 0, + memcpy(&crtc_state->base + 1, &saved_state->base + 1, sizeof(*crtc_state) - sizeof(crtc_state->base)); - crtc_state->scaler_state = scaler_state; - crtc_state->shared_dpll = shared_dpll; - crtc_state->dpll_hw_state = dpll_hw_state; - crtc_state->pch_pfit.force_thru = force_thru; - crtc_state->ips_force_disable = ips_force_disable; - if (IS_G4X(dev_priv) || - IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) - crtc_state->wm = wm_state; + kfree(saved_state); + return 0; } static int @@ -11314,11 +11628,13 @@ intel_modeset_pipe_config(struct drm_crtc *crtc, struct intel_encoder *encoder; struct drm_connector *connector; struct drm_connector_state *connector_state; - int base_bpp, ret = -EINVAL; + int base_bpp, ret; int i; bool retry = true; - clear_intel_crtc_state(pipe_config); + ret = clear_intel_crtc_state(pipe_config); + if (ret) + return ret; pipe_config->cpu_transcoder = (enum transcoder) to_intel_crtc(crtc)->pipe; @@ -11336,10 +11652,12 @@ intel_modeset_pipe_config(struct drm_crtc *crtc, (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC))) pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC; - base_bpp = compute_baseline_pipe_bpp(to_intel_crtc(crtc), - pipe_config); - if (base_bpp < 0) - goto fail; + ret = compute_baseline_pipe_bpp(to_intel_crtc(crtc), + pipe_config); + if (ret) + return ret; + + base_bpp = pipe_config->pipe_bpp; /* * Determine the real pipe dimensions. Note that stereo modes can @@ -11361,7 +11679,7 @@ intel_modeset_pipe_config(struct drm_crtc *crtc, if (!check_single_encoder_cloning(state, to_intel_crtc(crtc), encoder)) { DRM_DEBUG_KMS("rejecting invalid cloning configuration\n"); - goto fail; + return -EINVAL; } /* @@ -11394,10 +11712,13 @@ encoder_retry: continue; encoder = to_intel_encoder(connector_state->best_encoder); - - if (!(encoder->compute_config(encoder, pipe_config, connector_state))) { - DRM_DEBUG_KMS("Encoder config failure\n"); - goto fail; + ret = encoder->compute_config(encoder, pipe_config, + connector_state); + if (ret < 0) { + if (ret != -EDEADLK) + DRM_DEBUG_KMS("Encoder config failure: %d\n", + ret); + return ret; } } @@ -11408,16 +11729,16 @@ encoder_retry: * pipe_config->pixel_multiplier; ret = intel_crtc_compute_config(to_intel_crtc(crtc), pipe_config); + if (ret == -EDEADLK) + return ret; if (ret < 0) { DRM_DEBUG_KMS("CRTC fixup failed\n"); - goto fail; + return ret; } if (ret == RETRY) { - if (WARN(!retry, "loop in pipe configuration computation\n")) { - ret = -EINVAL; - goto fail; - } + if (WARN(!retry, "loop in pipe configuration computation\n")) + return -EINVAL; DRM_DEBUG_KMS("CRTC bw constrained, retrying\n"); retry = false; @@ -11433,8 +11754,7 @@ encoder_retry: DRM_DEBUG_KMS("hw max bpp: %i, pipe bpp: %i, dithering: %i\n", base_bpp, pipe_config->pipe_bpp, pipe_config->dither); -fail: - return ret; + return 0; } static bool intel_fuzzy_clock_check(int clock1, int clock2) @@ -11523,6 +11843,23 @@ pipe_config_err(bool adjust, const char *name, const char *format, ...) va_end(args); } +static bool fastboot_enabled(struct drm_i915_private *dev_priv) +{ + if (i915_modparams.fastboot != -1) + return i915_modparams.fastboot; + + /* Enable fastboot by default on Skylake and newer */ + if (INTEL_GEN(dev_priv) >= 9) + return true; + + /* Enable fastboot by default on VLV and CHV */ + if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) + return true; + + /* Disabled by default on all others */ + return false; +} + static bool intel_pipe_config_compare(struct drm_i915_private *dev_priv, struct intel_crtc_state *current_config, @@ -11534,6 +11871,11 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv, (current_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED) && !(pipe_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED); + if (fixup_inherited && !fastboot_enabled(dev_priv)) { + DRM_DEBUG_KMS("initial modeset and fastboot not set\n"); + ret = false; + } + #define PIPE_CONF_CHECK_X(name) do { \ if (current_config->name != pipe_config->name) { \ pipe_config_err(adjust, __stringify(name), \ @@ -11703,6 +12045,7 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv, PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_end); PIPE_CONF_CHECK_I(pixel_multiplier); + PIPE_CONF_CHECK_I(output_format); PIPE_CONF_CHECK_BOOL(has_hdmi_sink); if ((INTEL_GEN(dev_priv) < 8 && !IS_HASWELL(dev_priv)) || IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) @@ -11711,7 +12054,6 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv, PIPE_CONF_CHECK_BOOL(hdmi_scrambling); PIPE_CONF_CHECK_BOOL(hdmi_high_tmds_clock_ratio); PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_infoframe); - PIPE_CONF_CHECK_BOOL(ycbcr420); PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_audio); @@ -11833,6 +12175,8 @@ static void verify_wm_state(struct drm_crtc *crtc, struct skl_pipe_wm hw_wm, *sw_wm; struct skl_plane_wm *hw_plane_wm, *sw_plane_wm; struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry; + struct skl_ddb_entry hw_ddb_y[I915_MAX_PLANES]; + struct skl_ddb_entry hw_ddb_uv[I915_MAX_PLANES]; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); const enum pipe pipe = intel_crtc->pipe; int plane, level, max_level = ilk_wm_max_level(dev_priv); @@ -11840,9 +12184,11 @@ static void verify_wm_state(struct drm_crtc *crtc, if (INTEL_GEN(dev_priv) < 9 || !new_state->active) return; - skl_pipe_wm_get_hw_state(crtc, &hw_wm); + skl_pipe_wm_get_hw_state(intel_crtc, &hw_wm); sw_wm = &to_intel_crtc_state(new_state)->wm.skl.optimal; + skl_pipe_ddb_get_hw_state(intel_crtc, hw_ddb_y, hw_ddb_uv); + skl_ddb_get_hw_state(dev_priv, &hw_ddb); sw_ddb = &dev_priv->wm.skl_hw.ddb; @@ -11885,8 +12231,8 @@ static void verify_wm_state(struct drm_crtc *crtc, } /* DDB */ - hw_ddb_entry = &hw_ddb.plane[pipe][plane]; - sw_ddb_entry = &sw_ddb->plane[pipe][plane]; + hw_ddb_entry = &hw_ddb_y[plane]; + sw_ddb_entry = &to_intel_crtc_state(new_state)->wm.skl.plane_ddb_y[plane]; if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) { DRM_ERROR("mismatch in DDB state pipe %c plane %d (expected (%u,%u), found (%u,%u))\n", @@ -11935,8 +12281,8 @@ static void verify_wm_state(struct drm_crtc *crtc, } /* DDB */ - hw_ddb_entry = &hw_ddb.plane[pipe][PLANE_CURSOR]; - sw_ddb_entry = &sw_ddb->plane[pipe][PLANE_CURSOR]; + hw_ddb_entry = &hw_ddb_y[PLANE_CURSOR]; + sw_ddb_entry = &to_intel_crtc_state(new_state)->wm.skl.plane_ddb_y[PLANE_CURSOR]; if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) { DRM_ERROR("mismatch in DDB state pipe %c cursor (expected (%u,%u), found (%u,%u))\n", @@ -12220,8 +12566,9 @@ intel_modeset_verify_disabled(struct drm_device *dev, verify_disabled_dpll_state(dev); } -static void update_scanline_offset(struct intel_crtc *crtc) +static void update_scanline_offset(const struct intel_crtc_state *crtc_state) { + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); /* @@ -12251,8 +12598,8 @@ static void update_scanline_offset(struct intel_crtc *crtc) * However if queried just before the start of vblank we'll get an * answer that's slightly in the future. */ - if (IS_GEN2(dev_priv)) { - const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; + if (IS_GEN(dev_priv, 2)) { + const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode; int vtotal; vtotal = adjusted_mode->crtc_vtotal; @@ -12261,7 +12608,7 @@ static void update_scanline_offset(struct intel_crtc *crtc) crtc->scanline_offset = vtotal - 1; } else if (HAS_DDI(dev_priv) && - intel_crtc_has_type(crtc->config, INTEL_OUTPUT_HDMI)) { + intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) { crtc->scanline_offset = 2; } else crtc->scanline_offset = 1; @@ -12492,9 +12839,9 @@ static int intel_modeset_checks(struct drm_atomic_state *state) * phase. The code here should be run after the per-crtc and per-plane 'check' * handlers to ensure that all derived state has been updated. */ -static int calc_watermark_data(struct drm_atomic_state *state) +static int calc_watermark_data(struct intel_atomic_state *state) { - struct drm_device *dev = state->dev; + struct drm_device *dev = state->base.dev; struct drm_i915_private *dev_priv = to_i915(dev); /* Is there platform-specific watermark information to calculate? */ @@ -12544,14 +12891,15 @@ static int intel_atomic_check(struct drm_device *dev, } ret = intel_modeset_pipe_config(crtc, pipe_config); + if (ret == -EDEADLK) + return ret; if (ret) { intel_dump_pipe_config(to_intel_crtc(crtc), pipe_config, "[failed]"); return ret; } - if (i915_modparams.fastboot && - intel_pipe_config_compare(dev_priv, + if (intel_pipe_config_compare(dev_priv, to_intel_crtc_state(old_crtc_state), pipe_config, true)) { crtc_state->mode_changed = false; @@ -12566,6 +12914,10 @@ static int intel_atomic_check(struct drm_device *dev, "[modeset]" : "[fastset]"); } + ret = drm_dp_mst_atomic_check(state); + if (ret) + return ret; + if (any_ms) { ret = intel_modeset_checks(state); @@ -12575,12 +12927,16 @@ static int intel_atomic_check(struct drm_device *dev, intel_state->cdclk.logical = dev_priv->cdclk.logical; } + ret = icl_add_linked_planes(intel_state); + if (ret) + return ret; + ret = drm_atomic_helper_check_planes(dev, state); if (ret) return ret; intel_fbc_choose_crtc(dev_priv, intel_state); - return calc_watermark_data(state); + return calc_watermark_data(intel_state); } static int intel_atomic_prepare_commit(struct drm_device *dev, @@ -12592,8 +12948,9 @@ static int intel_atomic_prepare_commit(struct drm_device *dev, u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; + struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(&crtc->base)]; - if (!dev->max_vblank_count) + if (!vblank->max_vblank_count) return (u32)drm_crtc_accurate_vblank_count(&crtc->base); return dev->driver->get_vblank_counter(dev, crtc->pipe); @@ -12614,7 +12971,7 @@ static void intel_update_crtc(struct drm_crtc *crtc, to_intel_plane(crtc->primary)); if (modeset) { - update_scanline_offset(intel_crtc); + update_scanline_offset(pipe_config); dev_priv->display.crtc_enable(pipe_config, state); /* vblanks work again, re-enable pipe CRC. */ @@ -12622,12 +12979,24 @@ static void intel_update_crtc(struct drm_crtc *crtc, } else { intel_pre_plane_update(to_intel_crtc_state(old_crtc_state), pipe_config); + + if (pipe_config->update_pipe) + intel_encoders_update_pipe(crtc, pipe_config, state); } - if (new_plane_state) + if (pipe_config->update_pipe && !pipe_config->enable_fbc) + intel_fbc_disable(intel_crtc); + else if (new_plane_state) intel_fbc_enable(intel_crtc, pipe_config, new_plane_state); - drm_atomic_helper_commit_planes_on_crtc(old_crtc_state); + intel_begin_crtc_commit(crtc, old_crtc_state); + + if (INTEL_GEN(dev_priv) >= 9) + skl_update_planes_on_crtc(to_intel_atomic_state(state), intel_crtc); + else + i9xx_update_planes_on_crtc(to_intel_atomic_state(state), intel_crtc); + + intel_finish_crtc_commit(crtc, old_crtc_state); } static void intel_update_crtcs(struct drm_atomic_state *state) @@ -12659,13 +13028,12 @@ static void skl_update_crtcs(struct drm_atomic_state *state) int i; u8 hw_enabled_slices = dev_priv->wm.skl_hw.ddb.enabled_slices; u8 required_slices = intel_state->wm_results.ddb.enabled_slices; - - const struct skl_ddb_entry *entries[I915_MAX_PIPES] = {}; + struct skl_ddb_entry entries[I915_MAX_PIPES] = {}; for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) /* ignore allocations for crtc's that have been turned off. */ if (new_crtc_state->active) - entries[i] = &to_intel_crtc_state(old_crtc_state)->wm.skl.ddb; + entries[i] = to_intel_crtc_state(old_crtc_state)->wm.skl.ddb; /* If 2nd DBuf slice required, enable it here */ if (INTEL_GEN(dev_priv) >= 11 && required_slices > hw_enabled_slices) @@ -12691,14 +13059,13 @@ static void skl_update_crtcs(struct drm_atomic_state *state) if (updated & cmask || !cstate->base.active) continue; - if (skl_ddb_allocation_overlaps(dev_priv, + if (skl_ddb_allocation_overlaps(&cstate->wm.skl.ddb, entries, - &cstate->wm.skl.ddb, - i)) + INTEL_INFO(dev_priv)->num_pipes, i)) continue; updated |= cmask; - entries[i] = &cstate->wm.skl.ddb; + entries[i] = cstate->wm.skl.ddb; /* * If this is an already active pipe, it's DDB changed, @@ -12788,9 +13155,11 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state) struct intel_atomic_state *intel_state = to_intel_atomic_state(state); struct drm_i915_private *dev_priv = to_i915(dev); struct drm_crtc_state *old_crtc_state, *new_crtc_state; + struct intel_crtc_state *new_intel_crtc_state, *old_intel_crtc_state; struct drm_crtc *crtc; - struct intel_crtc_state *intel_cstate; + struct intel_crtc *intel_crtc; u64 put_domains[I915_MAX_PIPES] = {}; + intel_wakeref_t wakeref = 0; int i; intel_atomic_commit_fence_wait(intel_state); @@ -12798,27 +13167,28 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state) drm_atomic_helper_wait_for_dependencies(state); if (intel_state->modeset) - intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET); + wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET); for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + old_intel_crtc_state = to_intel_crtc_state(old_crtc_state); + new_intel_crtc_state = to_intel_crtc_state(new_crtc_state); + intel_crtc = to_intel_crtc(crtc); if (needs_modeset(new_crtc_state) || to_intel_crtc_state(new_crtc_state)->update_pipe) { - put_domains[to_intel_crtc(crtc)->pipe] = + put_domains[intel_crtc->pipe] = modeset_get_crtc_power_domains(crtc, - to_intel_crtc_state(new_crtc_state)); + new_intel_crtc_state); } if (!needs_modeset(new_crtc_state)) continue; - intel_pre_plane_update(to_intel_crtc_state(old_crtc_state), - to_intel_crtc_state(new_crtc_state)); + intel_pre_plane_update(old_intel_crtc_state, new_intel_crtc_state); if (old_crtc_state->active) { - intel_crtc_disable_planes(crtc, old_crtc_state->plane_mask); + intel_crtc_disable_planes(intel_state, intel_crtc); /* * We need to disable pipe CRC before disabling the pipe, @@ -12826,10 +13196,10 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state) */ intel_crtc_disable_pipe_crc(intel_crtc); - dev_priv->display.crtc_disable(to_intel_crtc_state(old_crtc_state), state); + dev_priv->display.crtc_disable(old_intel_crtc_state, state); intel_crtc->active = false; intel_fbc_disable(intel_crtc); - intel_disable_shared_dpll(intel_crtc); + intel_disable_shared_dpll(old_intel_crtc_state); /* * Underruns don't always raise @@ -12840,10 +13210,10 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state) /* FIXME unify this for all platforms */ if (!new_crtc_state->active && - !HAS_GMCH_DISPLAY(dev_priv) && + !HAS_GMCH(dev_priv) && dev_priv->display.initial_watermarks) dev_priv->display.initial_watermarks(intel_state, - to_intel_crtc_state(new_crtc_state)); + new_intel_crtc_state); } } @@ -12894,6 +13264,16 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state) */ drm_atomic_helper_wait_for_flip_done(dev, state); + for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) { + new_intel_crtc_state = to_intel_crtc_state(new_crtc_state); + + if (new_crtc_state->active && + !needs_modeset(new_crtc_state) && + (new_intel_crtc_state->base.color_mgmt_changed || + new_intel_crtc_state->update_pipe)) + intel_color_load_luts(new_intel_crtc_state); + } + /* * Now that the vblank has passed, we can go ahead and program the * optimal watermarks on platforms that need two-step watermark @@ -12902,11 +13282,11 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state) * TODO: Move this (and other cleanup) to an async worker eventually. */ for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) { - intel_cstate = to_intel_crtc_state(new_crtc_state); + new_intel_crtc_state = to_intel_crtc_state(new_crtc_state); if (dev_priv->display.optimize_watermarks) dev_priv->display.optimize_watermarks(intel_state, - intel_cstate); + new_intel_crtc_state); } for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { @@ -12934,7 +13314,7 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state) * the culprit. */ intel_uncore_arm_unclaimed_mmio_detection(dev_priv); - intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET); + intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET, wakeref); } /* @@ -13173,7 +13553,7 @@ static int intel_plane_pin_fb(struct intel_plane_state *plane_state) struct i915_vma *vma; if (plane->id == PLANE_CURSOR && - INTEL_INFO(dev_priv)->cursor_needs_physical) { + INTEL_INFO(dev_priv)->display.cursor_needs_physical) { struct drm_i915_gem_object *obj = intel_fb_obj(fb); const int align = intel_cursor_alignment(dev_priv); int err; @@ -13289,13 +13669,12 @@ intel_prepare_plane_fb(struct drm_plane *plane, ret = intel_plane_pin_fb(to_intel_plane_state(new_state)); - fb_obj_bump_render_priority(obj); - mutex_unlock(&dev_priv->drm.struct_mutex); i915_gem_object_unpin_pages(obj); if (ret) return ret; + fb_obj_bump_render_priority(obj); intel_fb_obj_flush(obj, ORIGIN_DIRTYFB); if (!new_state->fence) { /* implicit fencing */ @@ -13410,23 +13789,20 @@ static void intel_begin_crtc_commit(struct drm_crtc *crtc, intel_atomic_get_new_crtc_state(old_intel_state, intel_crtc); bool modeset = needs_modeset(&intel_cstate->base); - if (!modeset && - (intel_cstate->base.color_mgmt_changed || - intel_cstate->update_pipe)) { - intel_color_set_csc(&intel_cstate->base); - intel_color_load_luts(&intel_cstate->base); - } - /* Perform vblank evasion around commit operation */ intel_pipe_update_start(intel_cstate); if (modeset) goto out; + if (intel_cstate->base.color_mgmt_changed || + intel_cstate->update_pipe) + intel_color_commit(intel_cstate); + if (intel_cstate->update_pipe) intel_update_pipe_config(old_intel_cstate, intel_cstate); else if (INTEL_GEN(dev_priv) >= 9) - skl_detach_scalers(intel_crtc); + skl_detach_scalers(intel_cstate); out: if (dev_priv->display.atomic_update_watermarks) @@ -13439,7 +13815,7 @@ void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc, { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - if (!IS_GEN2(dev_priv)) + if (!IS_GEN(dev_priv, 2)) intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true); if (crtc_state->has_pch_encoder) { @@ -13528,56 +13904,6 @@ static bool i965_plane_format_mod_supported(struct drm_plane *_plane, } } -static bool skl_plane_format_mod_supported(struct drm_plane *_plane, - u32 format, u64 modifier) -{ - struct intel_plane *plane = to_intel_plane(_plane); - - switch (modifier) { - case DRM_FORMAT_MOD_LINEAR: - case I915_FORMAT_MOD_X_TILED: - case I915_FORMAT_MOD_Y_TILED: - case I915_FORMAT_MOD_Yf_TILED: - break; - case I915_FORMAT_MOD_Y_TILED_CCS: - case I915_FORMAT_MOD_Yf_TILED_CCS: - if (!plane->has_ccs) - return false; - break; - default: - return false; - } - - switch (format) { - case DRM_FORMAT_XRGB8888: - case DRM_FORMAT_XBGR8888: - case DRM_FORMAT_ARGB8888: - case DRM_FORMAT_ABGR8888: - if (is_ccs_modifier(modifier)) - return true; - /* fall through */ - case DRM_FORMAT_RGB565: - case DRM_FORMAT_XRGB2101010: - case DRM_FORMAT_XBGR2101010: - case DRM_FORMAT_YUYV: - case DRM_FORMAT_YVYU: - case DRM_FORMAT_UYVY: - case DRM_FORMAT_VYUY: - case DRM_FORMAT_NV12: - if (modifier == I915_FORMAT_MOD_Yf_TILED) - return true; - /* fall through */ - case DRM_FORMAT_C8: - if (modifier == DRM_FORMAT_MOD_LINEAR || - modifier == I915_FORMAT_MOD_X_TILED || - modifier == I915_FORMAT_MOD_Y_TILED) - return true; - /* fall through */ - default: - return false; - } -} - static bool intel_cursor_format_mod_supported(struct drm_plane *_plane, u32 format, u64 modifier) { @@ -13585,18 +13911,7 @@ static bool intel_cursor_format_mod_supported(struct drm_plane *_plane, format == DRM_FORMAT_ARGB8888; } -static struct drm_plane_funcs skl_plane_funcs = { - .update_plane = drm_atomic_helper_update_plane, - .disable_plane = drm_atomic_helper_disable_plane, - .destroy = intel_plane_destroy, - .atomic_get_property = intel_plane_atomic_get_property, - .atomic_set_property = intel_plane_atomic_set_property, - .atomic_duplicate_state = intel_plane_duplicate_state, - .atomic_destroy_state = intel_plane_destroy_state, - .format_mod_supported = skl_plane_format_mod_supported, -}; - -static struct drm_plane_funcs i965_plane_funcs = { +static const struct drm_plane_funcs i965_plane_funcs = { .update_plane = drm_atomic_helper_update_plane, .disable_plane = drm_atomic_helper_disable_plane, .destroy = intel_plane_destroy, @@ -13607,7 +13922,7 @@ static struct drm_plane_funcs i965_plane_funcs = { .format_mod_supported = i965_plane_format_mod_supported, }; -static struct drm_plane_funcs i8xx_plane_funcs = { +static const struct drm_plane_funcs i8xx_plane_funcs = { .update_plane = drm_atomic_helper_update_plane, .disable_plane = drm_atomic_helper_disable_plane, .destroy = intel_plane_destroy, @@ -13624,8 +13939,8 @@ intel_legacy_cursor_update(struct drm_plane *plane, struct drm_framebuffer *fb, int crtc_x, int crtc_y, unsigned int crtc_w, unsigned int crtc_h, - uint32_t src_x, uint32_t src_y, - uint32_t src_w, uint32_t src_h, + u32 src_x, u32 src_y, + u32 src_w, u32 src_h, struct drm_modeset_acquire_ctx *ctx) { struct drm_i915_private *dev_priv = to_i915(crtc->dev); @@ -13633,14 +13948,16 @@ intel_legacy_cursor_update(struct drm_plane *plane, struct drm_plane_state *old_plane_state, *new_plane_state; struct intel_plane *intel_plane = to_intel_plane(plane); struct drm_framebuffer *old_fb; - struct drm_crtc_state *crtc_state = crtc->state; + struct intel_crtc_state *crtc_state = + to_intel_crtc_state(crtc->state); + struct intel_crtc_state *new_crtc_state; /* * When crtc is inactive or there is a modeset pending, * wait for it to complete in the slowpath */ - if (!crtc_state->active || needs_modeset(crtc_state) || - to_intel_crtc_state(crtc_state)->update_pipe) + if (!crtc_state->base.active || needs_modeset(&crtc_state->base) || + crtc_state->update_pipe) goto slow; old_plane_state = plane->state; @@ -13670,6 +13987,12 @@ intel_legacy_cursor_update(struct drm_plane *plane, if (!new_plane_state) return -ENOMEM; + new_crtc_state = to_intel_crtc_state(intel_crtc_duplicate_state(crtc)); + if (!new_crtc_state) { + ret = -ENOMEM; + goto out_free; + } + drm_atomic_set_fb_for_plane(new_plane_state, fb); new_plane_state->src_x = src_x; @@ -13681,9 +14004,8 @@ intel_legacy_cursor_update(struct drm_plane *plane, new_plane_state->crtc_w = crtc_w; new_plane_state->crtc_h = crtc_h; - ret = intel_plane_atomic_check_with_state(to_intel_crtc_state(crtc->state), - to_intel_crtc_state(crtc->state), /* FIXME need a new crtc state? */ - to_intel_plane_state(plane->state), + ret = intel_plane_atomic_check_with_state(crtc_state, new_crtc_state, + to_intel_plane_state(old_plane_state), to_intel_plane_state(new_plane_state)); if (ret) goto out_free; @@ -13705,14 +14027,25 @@ intel_legacy_cursor_update(struct drm_plane *plane, /* Swap plane state */ plane->state = new_plane_state; + /* + * We cannot swap crtc_state as it may be in use by an atomic commit or + * page flip that's running simultaneously. If we swap crtc_state and + * destroy the old state, we will cause a use-after-free there. + * + * Only update active_planes, which is needed for our internal + * bookkeeping. Either value will do the right thing when updating + * planes atomically. If the cursor was part of the atomic update then + * we would have taken the slowpath. + */ + crtc_state->active_planes = new_crtc_state->active_planes; + if (plane->state->visible) { trace_intel_update_plane(plane, to_intel_crtc(crtc)); - intel_plane->update_plane(intel_plane, - to_intel_crtc_state(crtc->state), + intel_plane->update_plane(intel_plane, crtc_state, to_intel_plane_state(plane->state)); } else { trace_intel_disable_plane(plane, to_intel_crtc(crtc)); - intel_plane->disable_plane(intel_plane, to_intel_crtc(crtc)); + intel_plane->disable_plane(intel_plane, crtc_state); } intel_plane_unpin_fb(to_intel_plane_state(old_plane_state)); @@ -13720,6 +14053,8 @@ intel_legacy_cursor_update(struct drm_plane *plane, out_unlock: mutex_unlock(&dev_priv->drm.struct_mutex); out_free: + if (new_crtc_state) + intel_crtc_destroy_state(crtc, &new_crtc_state->base); if (ret) intel_plane_destroy_state(plane, new_plane_state); else @@ -13760,176 +14095,90 @@ static bool i9xx_plane_has_fbc(struct drm_i915_private *dev_priv, return i9xx_plane == PLANE_A; } -static bool skl_plane_has_fbc(struct drm_i915_private *dev_priv, - enum pipe pipe, enum plane_id plane_id) -{ - if (!HAS_FBC(dev_priv)) - return false; - - return pipe == PIPE_A && plane_id == PLANE_PRIMARY; -} - -bool skl_plane_has_planar(struct drm_i915_private *dev_priv, - enum pipe pipe, enum plane_id plane_id) -{ - /* - * FIXME: ICL requires two hardware planes for scanning out NV12 - * framebuffers. Do not advertize support until this is implemented. - */ - if (INTEL_GEN(dev_priv) >= 11) - return false; - - if (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv)) - return false; - - if (INTEL_GEN(dev_priv) == 9 && !IS_GEMINILAKE(dev_priv) && pipe == PIPE_C) - return false; - - if (plane_id != PLANE_PRIMARY && plane_id != PLANE_SPRITE0) - return false; - - return true; -} - static struct intel_plane * intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct intel_plane *primary = NULL; - struct intel_plane_state *state = NULL; + struct intel_plane *plane; const struct drm_plane_funcs *plane_funcs; - const uint32_t *intel_primary_formats; unsigned int supported_rotations; - unsigned int num_formats; - const uint64_t *modifiers; + unsigned int possible_crtcs; + const u64 *modifiers; + const u32 *formats; + int num_formats; int ret; - primary = kzalloc(sizeof(*primary), GFP_KERNEL); - if (!primary) { - ret = -ENOMEM; - goto fail; - } - - state = intel_create_plane_state(&primary->base); - if (!state) { - ret = -ENOMEM; - goto fail; - } + if (INTEL_GEN(dev_priv) >= 9) + return skl_universal_plane_create(dev_priv, pipe, + PLANE_PRIMARY); - primary->base.state = &state->base; + plane = intel_plane_alloc(); + if (IS_ERR(plane)) + return plane; - if (INTEL_GEN(dev_priv) >= 9) - state->scaler_id = -1; - primary->pipe = pipe; + plane->pipe = pipe; /* * On gen2/3 only plane A can do FBC, but the panel fitter and LVDS * port is hooked to pipe B. Hence we want plane A feeding pipe B. */ if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) < 4) - primary->i9xx_plane = (enum i9xx_plane_id) !pipe; - else - primary->i9xx_plane = (enum i9xx_plane_id) pipe; - primary->id = PLANE_PRIMARY; - primary->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, primary->id); - - if (INTEL_GEN(dev_priv) >= 9) - primary->has_fbc = skl_plane_has_fbc(dev_priv, - primary->pipe, - primary->id); + plane->i9xx_plane = (enum i9xx_plane_id) !pipe; else - primary->has_fbc = i9xx_plane_has_fbc(dev_priv, - primary->i9xx_plane); + plane->i9xx_plane = (enum i9xx_plane_id) pipe; + plane->id = PLANE_PRIMARY; + plane->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, plane->id); - if (primary->has_fbc) { + plane->has_fbc = i9xx_plane_has_fbc(dev_priv, plane->i9xx_plane); + if (plane->has_fbc) { struct intel_fbc *fbc = &dev_priv->fbc; - fbc->possible_framebuffer_bits |= primary->frontbuffer_bit; + fbc->possible_framebuffer_bits |= plane->frontbuffer_bit; } - if (INTEL_GEN(dev_priv) >= 9) { - primary->has_ccs = skl_plane_has_ccs(dev_priv, pipe, - PLANE_PRIMARY); - - if (skl_plane_has_planar(dev_priv, pipe, PLANE_PRIMARY)) { - intel_primary_formats = skl_pri_planar_formats; - num_formats = ARRAY_SIZE(skl_pri_planar_formats); - } else { - intel_primary_formats = skl_primary_formats; - num_formats = ARRAY_SIZE(skl_primary_formats); - } - - if (primary->has_ccs) - modifiers = skl_format_modifiers_ccs; - else - modifiers = skl_format_modifiers_noccs; - - primary->max_stride = skl_plane_max_stride; - primary->update_plane = skl_update_plane; - primary->disable_plane = skl_disable_plane; - primary->get_hw_state = skl_plane_get_hw_state; - primary->check_plane = skl_plane_check; - - plane_funcs = &skl_plane_funcs; - } else if (INTEL_GEN(dev_priv) >= 4) { - intel_primary_formats = i965_primary_formats; + if (INTEL_GEN(dev_priv) >= 4) { + formats = i965_primary_formats; num_formats = ARRAY_SIZE(i965_primary_formats); modifiers = i9xx_format_modifiers; - primary->max_stride = i9xx_plane_max_stride; - primary->update_plane = i9xx_update_plane; - primary->disable_plane = i9xx_disable_plane; - primary->get_hw_state = i9xx_plane_get_hw_state; - primary->check_plane = i9xx_plane_check; + plane->max_stride = i9xx_plane_max_stride; + plane->update_plane = i9xx_update_plane; + plane->disable_plane = i9xx_disable_plane; + plane->get_hw_state = i9xx_plane_get_hw_state; + plane->check_plane = i9xx_plane_check; plane_funcs = &i965_plane_funcs; } else { - intel_primary_formats = i8xx_primary_formats; + formats = i8xx_primary_formats; num_formats = ARRAY_SIZE(i8xx_primary_formats); modifiers = i9xx_format_modifiers; - primary->max_stride = i9xx_plane_max_stride; - primary->update_plane = i9xx_update_plane; - primary->disable_plane = i9xx_disable_plane; - primary->get_hw_state = i9xx_plane_get_hw_state; - primary->check_plane = i9xx_plane_check; + plane->max_stride = i9xx_plane_max_stride; + plane->update_plane = i9xx_update_plane; + plane->disable_plane = i9xx_disable_plane; + plane->get_hw_state = i9xx_plane_get_hw_state; + plane->check_plane = i9xx_plane_check; plane_funcs = &i8xx_plane_funcs; } - if (INTEL_GEN(dev_priv) >= 9) - ret = drm_universal_plane_init(&dev_priv->drm, &primary->base, - 0, plane_funcs, - intel_primary_formats, num_formats, - modifiers, - DRM_PLANE_TYPE_PRIMARY, - "plane 1%c", pipe_name(pipe)); - else if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv)) - ret = drm_universal_plane_init(&dev_priv->drm, &primary->base, - 0, plane_funcs, - intel_primary_formats, num_formats, - modifiers, + possible_crtcs = BIT(pipe); + + if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv)) + ret = drm_universal_plane_init(&dev_priv->drm, &plane->base, + possible_crtcs, plane_funcs, + formats, num_formats, modifiers, DRM_PLANE_TYPE_PRIMARY, "primary %c", pipe_name(pipe)); else - ret = drm_universal_plane_init(&dev_priv->drm, &primary->base, - 0, plane_funcs, - intel_primary_formats, num_formats, - modifiers, + ret = drm_universal_plane_init(&dev_priv->drm, &plane->base, + possible_crtcs, plane_funcs, + formats, num_formats, modifiers, DRM_PLANE_TYPE_PRIMARY, "plane %c", - plane_name(primary->i9xx_plane)); + plane_name(plane->i9xx_plane)); if (ret) goto fail; - if (INTEL_GEN(dev_priv) >= 10) { - supported_rotations = - DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_90 | - DRM_MODE_ROTATE_180 | DRM_MODE_ROTATE_270 | - DRM_MODE_REFLECT_X; - } else if (INTEL_GEN(dev_priv) >= 9) { - supported_rotations = - DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_90 | - DRM_MODE_ROTATE_180 | DRM_MODE_ROTATE_270; - } else if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) { + if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) { supported_rotations = DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180 | DRM_MODE_REFLECT_X; @@ -13941,26 +14190,16 @@ intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe) } if (INTEL_GEN(dev_priv) >= 4) - drm_plane_create_rotation_property(&primary->base, + drm_plane_create_rotation_property(&plane->base, DRM_MODE_ROTATE_0, supported_rotations); - if (INTEL_GEN(dev_priv) >= 9) - drm_plane_create_color_properties(&primary->base, - BIT(DRM_COLOR_YCBCR_BT601) | - BIT(DRM_COLOR_YCBCR_BT709), - BIT(DRM_COLOR_YCBCR_LIMITED_RANGE) | - BIT(DRM_COLOR_YCBCR_FULL_RANGE), - DRM_COLOR_YCBCR_BT709, - DRM_COLOR_YCBCR_LIMITED_RANGE); - - drm_plane_helper_add(&primary->base, &intel_plane_helper_funcs); + drm_plane_helper_add(&plane->base, &intel_plane_helper_funcs); - return primary; + return plane; fail: - kfree(state); - kfree(primary); + intel_plane_free(plane); return ERR_PTR(ret); } @@ -13969,23 +14208,13 @@ static struct intel_plane * intel_cursor_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct intel_plane *cursor = NULL; - struct intel_plane_state *state = NULL; + unsigned int possible_crtcs; + struct intel_plane *cursor; int ret; - cursor = kzalloc(sizeof(*cursor), GFP_KERNEL); - if (!cursor) { - ret = -ENOMEM; - goto fail; - } - - state = intel_create_plane_state(&cursor->base); - if (!state) { - ret = -ENOMEM; - goto fail; - } - - cursor->base.state = &state->base; + cursor = intel_plane_alloc(); + if (IS_ERR(cursor)) + return cursor; cursor->pipe = pipe; cursor->i9xx_plane = (enum i9xx_plane_id) pipe; @@ -14012,8 +14241,10 @@ intel_cursor_plane_create(struct drm_i915_private *dev_priv, if (IS_I845G(dev_priv) || IS_I865G(dev_priv) || HAS_CUR_FBC(dev_priv)) cursor->cursor.size = ~0; + possible_crtcs = BIT(pipe); + ret = drm_universal_plane_init(&dev_priv->drm, &cursor->base, - 0, &intel_cursor_plane_funcs, + possible_crtcs, &intel_cursor_plane_funcs, intel_cursor_formats, ARRAY_SIZE(intel_cursor_formats), cursor_format_modifiers, @@ -14028,16 +14259,12 @@ intel_cursor_plane_create(struct drm_i915_private *dev_priv, DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180); - if (INTEL_GEN(dev_priv) >= 9) - state->scaler_id = -1; - drm_plane_helper_add(&cursor->base, &intel_plane_helper_funcs); return cursor; fail: - kfree(state); - kfree(cursor); + intel_plane_free(cursor); return ERR_PTR(ret); } @@ -14050,7 +14277,7 @@ static void intel_crtc_init_scalers(struct intel_crtc *crtc, struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); int i; - crtc->num_scalers = dev_priv->info.num_scalers[crtc->pipe]; + crtc->num_scalers = RUNTIME_INFO(dev_priv)->num_scalers[crtc->pipe]; if (!crtc->num_scalers) return; @@ -14058,7 +14285,7 @@ static void intel_crtc_init_scalers(struct intel_crtc *crtc, struct intel_scaler *scaler = &scaler_state->scalers[i]; scaler->in_use = 0; - scaler->mode = PS_SCALER_MODE_DYN; + scaler->mode = 0; } scaler_state->scaler_id = -1; @@ -14136,7 +14363,7 @@ static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe) drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs); - intel_color_init(&intel_crtc->base); + intel_color_init(intel_crtc); WARN_ON(drm_crtc_index(&intel_crtc->base) != intel_crtc->pipe); @@ -14153,18 +14380,6 @@ fail: return ret; } -enum pipe intel_get_pipe_from_connector(struct intel_connector *connector) -{ - struct drm_device *dev = connector->base.dev; - - WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex)); - - if (!connector->base.state->crtc) - return INVALID_PIPE; - - return to_intel_crtc(connector->base.state->crtc)->pipe; -} - int intel_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { @@ -14199,7 +14414,7 @@ static int intel_encoder_clones(struct intel_encoder *encoder) return index_mask; } -static bool has_edp_a(struct drm_i915_private *dev_priv) +static bool ilk_has_edp_a(struct drm_i915_private *dev_priv) { if (!IS_MOBILE(dev_priv)) return false; @@ -14207,13 +14422,13 @@ static bool has_edp_a(struct drm_i915_private *dev_priv) if ((I915_READ(DP_A) & DP_DETECTED) == 0) return false; - if (IS_GEN5(dev_priv) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE)) + if (IS_GEN(dev_priv, 5) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE)) return false; return true; } -static bool intel_crt_present(struct drm_i915_private *dev_priv) +static bool intel_ddi_crt_present(struct drm_i915_private *dev_priv) { if (INTEL_GEN(dev_priv) >= 9) return false; @@ -14221,15 +14436,12 @@ static bool intel_crt_present(struct drm_i915_private *dev_priv) if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv)) return false; - if (IS_CHERRYVIEW(dev_priv)) - return false; - if (HAS_PCH_LPT_H(dev_priv) && I915_READ(SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED) return false; /* DDI E can't be used if DDI A requires 4 lanes */ - if (HAS_DDI(dev_priv) && I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES) + if (I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES) return false; if (!dev_priv->vbt.int_crt_support) @@ -14281,26 +14493,25 @@ static void intel_setup_outputs(struct drm_i915_private *dev_priv) intel_pps_init(dev_priv); - if (INTEL_INFO(dev_priv)->num_pipes == 0) + if (!HAS_DISPLAY(dev_priv)) return; - /* - * intel_edp_init_connector() depends on this completing first, to - * prevent the registeration of both eDP and LVDS and the incorrect - * sharing of the PPS. - */ - intel_lvds_init(dev_priv); - - if (intel_crt_present(dev_priv)) - intel_crt_init(dev_priv); - if (IS_ICELAKE(dev_priv)) { intel_ddi_init(dev_priv, PORT_A); intel_ddi_init(dev_priv, PORT_B); intel_ddi_init(dev_priv, PORT_C); intel_ddi_init(dev_priv, PORT_D); intel_ddi_init(dev_priv, PORT_E); - intel_ddi_init(dev_priv, PORT_F); + /* + * On some ICL SKUs port F is not present. No strap bits for + * this, so rely on VBT. + * Work around broken VBTs on SKUs known to have no port F. + */ + if (IS_ICL_WITH_PORT_F(dev_priv) && + intel_bios_is_port_present(dev_priv, PORT_F)) + intel_ddi_init(dev_priv, PORT_F); + + icl_dsi_init(dev_priv); } else if (IS_GEN9_LP(dev_priv)) { /* * FIXME: Broxton doesn't support port detection via the @@ -14315,6 +14526,9 @@ static void intel_setup_outputs(struct drm_i915_private *dev_priv) } else if (HAS_DDI(dev_priv)) { int found; + if (intel_ddi_crt_present(dev_priv)) + intel_crt_init(dev_priv); + /* * Haswell uses DDI functions to detect digital outputs. * On SKL pre-D0 the strap isn't connected, so we assume @@ -14341,16 +14555,23 @@ static void intel_setup_outputs(struct drm_i915_private *dev_priv) * On SKL we don't have a way to detect DDI-E so we rely on VBT. */ if (IS_GEN9_BC(dev_priv) && - (dev_priv->vbt.ddi_port_info[PORT_E].supports_dp || - dev_priv->vbt.ddi_port_info[PORT_E].supports_dvi || - dev_priv->vbt.ddi_port_info[PORT_E].supports_hdmi)) + intel_bios_is_port_present(dev_priv, PORT_E)) intel_ddi_init(dev_priv, PORT_E); } else if (HAS_PCH_SPLIT(dev_priv)) { int found; + + /* + * intel_edp_init_connector() depends on this completing first, + * to prevent the registration of both eDP and LVDS and the + * incorrect sharing of the PPS. + */ + intel_lvds_init(dev_priv); + intel_crt_init(dev_priv); + dpd_is_edp = intel_dp_is_port_edp(dev_priv, PORT_D); - if (has_edp_a(dev_priv)) + if (ilk_has_edp_a(dev_priv)) intel_dp_init(dev_priv, DP_A, PORT_A); if (I915_READ(PCH_HDMIB) & SDVO_DETECTED) { @@ -14376,6 +14597,9 @@ static void intel_setup_outputs(struct drm_i915_private *dev_priv) } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { bool has_edp, has_port; + if (IS_VALLEYVIEW(dev_priv) && dev_priv->vbt.int_crt_support) + intel_crt_init(dev_priv); + /* * The DP_DETECTED bit is the latched state of the DDC * SDA pin at boot. However since eDP doesn't require DDC @@ -14418,9 +14642,17 @@ static void intel_setup_outputs(struct drm_i915_private *dev_priv) } vlv_dsi_init(dev_priv); - } else if (!IS_GEN2(dev_priv) && !IS_PINEVIEW(dev_priv)) { + } else if (IS_PINEVIEW(dev_priv)) { + intel_lvds_init(dev_priv); + intel_crt_init(dev_priv); + } else if (IS_GEN_RANGE(dev_priv, 3, 4)) { bool found = false; + if (IS_MOBILE(dev_priv)) + intel_lvds_init(dev_priv); + + intel_crt_init(dev_priv); + if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) { DRM_DEBUG_KMS("probing SDVOB\n"); found = intel_sdvo_init(dev_priv, GEN3_SDVOB, PORT_B); @@ -14452,11 +14684,16 @@ static void intel_setup_outputs(struct drm_i915_private *dev_priv) if (IS_G4X(dev_priv) && (I915_READ(DP_D) & DP_DETECTED)) intel_dp_init(dev_priv, DP_D, PORT_D); - } else if (IS_GEN2(dev_priv)) - intel_dvo_init(dev_priv); - if (SUPPORTS_TV(dev_priv)) - intel_tv_init(dev_priv); + if (SUPPORTS_TV(dev_priv)) + intel_tv_init(dev_priv); + } else if (IS_GEN(dev_priv, 2)) { + if (IS_I85X(dev_priv)) + intel_lvds_init(dev_priv); + + intel_crt_init(dev_priv); + intel_dvo_init(dev_priv); + } intel_psr_init(dev_priv); @@ -14523,7 +14760,7 @@ static const struct drm_framebuffer_funcs intel_fb_funcs = { static u32 intel_fb_pitch_limit(struct drm_i915_private *dev_priv, - uint64_t fb_modifier, uint32_t pixel_format) + u32 pixel_format, u64 fb_modifier) { struct intel_crtc *crtc; struct intel_plane *plane; @@ -14545,7 +14782,6 @@ static int intel_framebuffer_init(struct intel_framebuffer *intel_fb, { struct drm_i915_private *dev_priv = to_i915(obj->base.dev); struct drm_framebuffer *fb = &intel_fb->base; - struct drm_format_name_buf format_name; u32 pitch_limit; unsigned int tiling, stride; int ret = -EINVAL; @@ -14576,33 +14812,14 @@ static int intel_framebuffer_init(struct intel_framebuffer *intel_fb, } } - /* Passed in modifier sanity checking. */ - switch (mode_cmd->modifier[0]) { - case I915_FORMAT_MOD_Y_TILED_CCS: - case I915_FORMAT_MOD_Yf_TILED_CCS: - switch (mode_cmd->pixel_format) { - case DRM_FORMAT_XBGR8888: - case DRM_FORMAT_ABGR8888: - case DRM_FORMAT_XRGB8888: - case DRM_FORMAT_ARGB8888: - break; - default: - DRM_DEBUG_KMS("RC supported only with RGB8888 formats\n"); - goto err; - } - /* fall through */ - case I915_FORMAT_MOD_Y_TILED: - case I915_FORMAT_MOD_Yf_TILED: - if (INTEL_GEN(dev_priv) < 9) { - DRM_DEBUG_KMS("Unsupported tiling 0x%llx!\n", - mode_cmd->modifier[0]); - goto err; - } - case DRM_FORMAT_MOD_LINEAR: - case I915_FORMAT_MOD_X_TILED: - break; - default: - DRM_DEBUG_KMS("Unsupported fb modifier 0x%llx!\n", + if (!drm_any_plane_has_format(&dev_priv->drm, + mode_cmd->pixel_format, + mode_cmd->modifier[0])) { + struct drm_format_name_buf format_name; + + DRM_DEBUG_KMS("unsupported pixel format %s / modifier 0x%llx\n", + drm_get_format_name(mode_cmd->pixel_format, + &format_name), mode_cmd->modifier[0]); goto err; } @@ -14617,8 +14834,8 @@ static int intel_framebuffer_init(struct intel_framebuffer *intel_fb, goto err; } - pitch_limit = intel_fb_pitch_limit(dev_priv, mode_cmd->modifier[0], - mode_cmd->pixel_format); + pitch_limit = intel_fb_pitch_limit(dev_priv, mode_cmd->pixel_format, + mode_cmd->modifier[0]); if (mode_cmd->pitches[0] > pitch_limit) { DRM_DEBUG_KMS("%s pitch (%u) must be at most %d\n", mode_cmd->modifier[0] != DRM_FORMAT_MOD_LINEAR ? @@ -14637,83 +14854,12 @@ static int intel_framebuffer_init(struct intel_framebuffer *intel_fb, goto err; } - /* Reject formats not supported by any plane early. */ - switch (mode_cmd->pixel_format) { - case DRM_FORMAT_C8: - case DRM_FORMAT_RGB565: - case DRM_FORMAT_XRGB8888: - case DRM_FORMAT_ARGB8888: - break; - case DRM_FORMAT_XRGB1555: - if (INTEL_GEN(dev_priv) > 3) { - DRM_DEBUG_KMS("unsupported pixel format: %s\n", - drm_get_format_name(mode_cmd->pixel_format, &format_name)); - goto err; - } - break; - case DRM_FORMAT_ABGR8888: - if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) && - INTEL_GEN(dev_priv) < 9) { - DRM_DEBUG_KMS("unsupported pixel format: %s\n", - drm_get_format_name(mode_cmd->pixel_format, &format_name)); - goto err; - } - break; - case DRM_FORMAT_XBGR8888: - case DRM_FORMAT_XRGB2101010: - case DRM_FORMAT_XBGR2101010: - if (INTEL_GEN(dev_priv) < 4) { - DRM_DEBUG_KMS("unsupported pixel format: %s\n", - drm_get_format_name(mode_cmd->pixel_format, &format_name)); - goto err; - } - break; - case DRM_FORMAT_ABGR2101010: - if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) { - DRM_DEBUG_KMS("unsupported pixel format: %s\n", - drm_get_format_name(mode_cmd->pixel_format, &format_name)); - goto err; - } - break; - case DRM_FORMAT_YUYV: - case DRM_FORMAT_UYVY: - case DRM_FORMAT_YVYU: - case DRM_FORMAT_VYUY: - if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) { - DRM_DEBUG_KMS("unsupported pixel format: %s\n", - drm_get_format_name(mode_cmd->pixel_format, &format_name)); - goto err; - } - break; - case DRM_FORMAT_NV12: - if (INTEL_GEN(dev_priv) < 9 || IS_SKYLAKE(dev_priv) || - IS_BROXTON(dev_priv) || INTEL_GEN(dev_priv) >= 11) { - DRM_DEBUG_KMS("unsupported pixel format: %s\n", - drm_get_format_name(mode_cmd->pixel_format, - &format_name)); - goto err; - } - break; - default: - DRM_DEBUG_KMS("unsupported pixel format: %s\n", - drm_get_format_name(mode_cmd->pixel_format, &format_name)); - goto err; - } - /* FIXME need to adjust LINOFF/TILEOFF accordingly. */ if (mode_cmd->offsets[0] != 0) goto err; drm_helper_mode_fill_fb_struct(&dev_priv->drm, fb, mode_cmd); - if (fb->format->format == DRM_FORMAT_NV12 && - (fb->width < SKL_MIN_YUV_420_SRC_W || - fb->height < SKL_MIN_YUV_420_SRC_H || - (fb->width % 4) != 0 || (fb->height % 4) != 0)) { - DRM_DEBUG_KMS("src dimensions not correct for NV12\n"); - goto err; - } - for (i = 0; i < fb->format->num_planes; i++) { u32 stride_alignment; @@ -14733,7 +14879,7 @@ static int intel_framebuffer_init(struct intel_framebuffer *intel_fb, * require the entire fb to accommodate that to avoid * potential runtime errors at plane configuration time. */ - if (IS_GEN9(dev_priv) && i == 0 && fb->width > 3840 && + if (IS_GEN(dev_priv, 9) && i == 0 && fb->width > 3840 && is_ccs_modifier(fb->modifier)) stride_alignment *= 4; @@ -14938,7 +15084,7 @@ void intel_init_display_hooks(struct drm_i915_private *dev_priv) dev_priv->display.crtc_compute_clock = pnv_crtc_compute_clock; dev_priv->display.crtc_enable = i9xx_crtc_enable; dev_priv->display.crtc_disable = i9xx_crtc_disable; - } else if (!IS_GEN2(dev_priv)) { + } else if (!IS_GEN(dev_priv, 2)) { dev_priv->display.get_pipe_config = i9xx_get_pipe_config; dev_priv->display.get_initial_plane_config = i9xx_get_initial_plane_config; @@ -14954,9 +15100,9 @@ void intel_init_display_hooks(struct drm_i915_private *dev_priv) dev_priv->display.crtc_disable = i9xx_crtc_disable; } - if (IS_GEN5(dev_priv)) { + if (IS_GEN(dev_priv, 5)) { dev_priv->display.fdi_link_train = ironlake_fdi_link_train; - } else if (IS_GEN6(dev_priv)) { + } else if (IS_GEN(dev_priv, 6)) { dev_priv->display.fdi_link_train = gen6_fdi_link_train; } else if (IS_IVYBRIDGE(dev_priv)) { /* FIXME: detect B0+ stepping and use auto training */ @@ -14971,174 +15117,6 @@ void intel_init_display_hooks(struct drm_i915_private *dev_priv) dev_priv->display.update_crtcs = intel_update_crtcs; } -/* - * Some machines (Lenovo U160) do not work with SSC on LVDS for some reason - */ -static void quirk_ssc_force_disable(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = to_i915(dev); - dev_priv->quirks |= QUIRK_LVDS_SSC_DISABLE; - DRM_INFO("applying lvds SSC disable quirk\n"); -} - -/* - * A machine (e.g. Acer Aspire 5734Z) may need to invert the panel backlight - * brightness value - */ -static void quirk_invert_brightness(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = to_i915(dev); - dev_priv->quirks |= QUIRK_INVERT_BRIGHTNESS; - DRM_INFO("applying inverted panel brightness quirk\n"); -} - -/* Some VBT's incorrectly indicate no backlight is present */ -static void quirk_backlight_present(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = to_i915(dev); - dev_priv->quirks |= QUIRK_BACKLIGHT_PRESENT; - DRM_INFO("applying backlight present quirk\n"); -} - -/* Toshiba Satellite P50-C-18C requires T12 delay to be min 800ms - * which is 300 ms greater than eDP spec T12 min. - */ -static void quirk_increase_t12_delay(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = to_i915(dev); - - dev_priv->quirks |= QUIRK_INCREASE_T12_DELAY; - DRM_INFO("Applying T12 delay quirk\n"); -} - -/* - * GeminiLake NUC HDMI outputs require additional off time - * this allows the onboard retimer to correctly sync to signal - */ -static void quirk_increase_ddi_disabled_time(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = to_i915(dev); - - dev_priv->quirks |= QUIRK_INCREASE_DDI_DISABLED_TIME; - DRM_INFO("Applying Increase DDI Disabled quirk\n"); -} - -struct intel_quirk { - int device; - int subsystem_vendor; - int subsystem_device; - void (*hook)(struct drm_device *dev); -}; - -/* For systems that don't have a meaningful PCI subdevice/subvendor ID */ -struct intel_dmi_quirk { - void (*hook)(struct drm_device *dev); - const struct dmi_system_id (*dmi_id_list)[]; -}; - -static int intel_dmi_reverse_brightness(const struct dmi_system_id *id) -{ - DRM_INFO("Backlight polarity reversed on %s\n", id->ident); - return 1; -} - -static const struct intel_dmi_quirk intel_dmi_quirks[] = { - { - .dmi_id_list = &(const struct dmi_system_id[]) { - { - .callback = intel_dmi_reverse_brightness, - .ident = "NCR Corporation", - .matches = {DMI_MATCH(DMI_SYS_VENDOR, "NCR Corporation"), - DMI_MATCH(DMI_PRODUCT_NAME, ""), - }, - }, - { } /* terminating entry */ - }, - .hook = quirk_invert_brightness, - }, -}; - -static struct intel_quirk intel_quirks[] = { - /* Lenovo U160 cannot use SSC on LVDS */ - { 0x0046, 0x17aa, 0x3920, quirk_ssc_force_disable }, - - /* Sony Vaio Y cannot use SSC on LVDS */ - { 0x0046, 0x104d, 0x9076, quirk_ssc_force_disable }, - - /* Acer Aspire 5734Z must invert backlight brightness */ - { 0x2a42, 0x1025, 0x0459, quirk_invert_brightness }, - - /* Acer/eMachines G725 */ - { 0x2a42, 0x1025, 0x0210, quirk_invert_brightness }, - - /* Acer/eMachines e725 */ - { 0x2a42, 0x1025, 0x0212, quirk_invert_brightness }, - - /* Acer/Packard Bell NCL20 */ - { 0x2a42, 0x1025, 0x034b, quirk_invert_brightness }, - - /* Acer Aspire 4736Z */ - { 0x2a42, 0x1025, 0x0260, quirk_invert_brightness }, - - /* Acer Aspire 5336 */ - { 0x2a42, 0x1025, 0x048a, quirk_invert_brightness }, - - /* Acer C720 and C720P Chromebooks (Celeron 2955U) have backlights */ - { 0x0a06, 0x1025, 0x0a11, quirk_backlight_present }, - - /* Acer C720 Chromebook (Core i3 4005U) */ - { 0x0a16, 0x1025, 0x0a11, quirk_backlight_present }, - - /* Apple Macbook 2,1 (Core 2 T7400) */ - { 0x27a2, 0x8086, 0x7270, quirk_backlight_present }, - - /* Apple Macbook 4,1 */ - { 0x2a02, 0x106b, 0x00a1, quirk_backlight_present }, - - /* Toshiba CB35 Chromebook (Celeron 2955U) */ - { 0x0a06, 0x1179, 0x0a88, quirk_backlight_present }, - - /* HP Chromebook 14 (Celeron 2955U) */ - { 0x0a06, 0x103c, 0x21ed, quirk_backlight_present }, - - /* Dell Chromebook 11 */ - { 0x0a06, 0x1028, 0x0a35, quirk_backlight_present }, - - /* Dell Chromebook 11 (2015 version) */ - { 0x0a16, 0x1028, 0x0a35, quirk_backlight_present }, - - /* Toshiba Satellite P50-C-18C */ - { 0x191B, 0x1179, 0xF840, quirk_increase_t12_delay }, - - /* GeminiLake NUC */ - { 0x3185, 0x8086, 0x2072, quirk_increase_ddi_disabled_time }, - { 0x3184, 0x8086, 0x2072, quirk_increase_ddi_disabled_time }, - /* ASRock ITX*/ - { 0x3185, 0x1849, 0x2212, quirk_increase_ddi_disabled_time }, - { 0x3184, 0x1849, 0x2212, quirk_increase_ddi_disabled_time }, -}; - -static void intel_init_quirks(struct drm_device *dev) -{ - struct pci_dev *d = dev->pdev; - int i; - - for (i = 0; i < ARRAY_SIZE(intel_quirks); i++) { - struct intel_quirk *q = &intel_quirks[i]; - - if (d->device == q->device && - (d->subsystem_vendor == q->subsystem_vendor || - q->subsystem_vendor == PCI_ANY_ID) && - (d->subsystem_device == q->subsystem_device || - q->subsystem_device == PCI_ANY_ID)) - q->hook(dev); - } - for (i = 0; i < ARRAY_SIZE(intel_dmi_quirks); i++) { - if (dmi_check_system(*intel_dmi_quirks[i].dmi_id_list) != 0) - intel_dmi_quirks[i].hook(dev); - } -} - /* Disable the VGA plane that we never use */ static void i915_disable_vga(struct drm_i915_private *dev_priv) { @@ -15217,7 +15195,7 @@ retry: * intermediate watermarks (since we don't trust the current * watermarks). */ - if (!HAS_GMCH_DISPLAY(dev_priv)) + if (!HAS_GMCH(dev_priv)) intel_state->skip_intermediate_wm = true; ret = intel_atomic_check(dev, state); @@ -15256,12 +15234,12 @@ fail: static void intel_update_fdi_pll_freq(struct drm_i915_private *dev_priv) { - if (IS_GEN5(dev_priv)) { + if (IS_GEN(dev_priv, 5)) { u32 fdi_pll_clk = I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK; dev_priv->fdi_pll_freq = (fdi_pll_clk + 2) * 10000; - } else if (IS_GEN6(dev_priv) || IS_IVYBRIDGE(dev_priv)) { + } else if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv)) { dev_priv->fdi_pll_freq = 270000; } else { return; @@ -15352,7 +15330,9 @@ int intel_modeset_init(struct drm_device *dev) INIT_WORK(&dev_priv->atomic_helper.free_work, intel_atomic_helper_free_state_worker); - intel_init_quirks(dev); + intel_init_quirks(dev_priv); + + intel_fbc_init(dev_priv); intel_init_pm(dev_priv); @@ -15375,10 +15355,10 @@ int intel_modeset_init(struct drm_device *dev) } /* maximum framebuffer dimensions */ - if (IS_GEN2(dev_priv)) { + if (IS_GEN(dev_priv, 2)) { dev->mode_config.max_width = 2048; dev->mode_config.max_height = 2048; - } else if (IS_GEN3(dev_priv)) { + } else if (IS_GEN(dev_priv, 3)) { dev->mode_config.max_width = 4096; dev->mode_config.max_height = 4096; } else { @@ -15389,7 +15369,7 @@ int intel_modeset_init(struct drm_device *dev) if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) { dev->mode_config.cursor_width = IS_I845G(dev_priv) ? 64 : 512; dev->mode_config.cursor_height = 1023; - } else if (IS_GEN2(dev_priv)) { + } else if (IS_GEN(dev_priv, 2)) { dev->mode_config.cursor_width = 64; dev->mode_config.cursor_height = 64; } else { @@ -15456,7 +15436,7 @@ int intel_modeset_init(struct drm_device *dev) * Note that we need to do this after reconstructing the BIOS fb's * since the watermark calculation done here will use pstate->fb. */ - if (!HAS_GMCH_DISPLAY(dev_priv)) + if (!HAS_GMCH(dev_priv)) sanitize_watermarks(dev); /* @@ -15584,8 +15564,8 @@ intel_sanitize_plane_mapping(struct drm_i915_private *dev_priv) if (pipe == crtc->pipe) continue; - DRM_DEBUG_KMS("%s attached to the wrong pipe, disabling plane\n", - plane->base.name); + DRM_DEBUG_KMS("[PLANE:%d:%s] attached to the wrong pipe, disabling plane\n", + plane->base.base.id, plane->base.name); plane_crtc = intel_get_crtc_for_pipe(dev_priv, pipe); intel_plane_disable_noatomic(plane_crtc, plane); @@ -15626,7 +15606,8 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc, { struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = to_i915(dev); - enum transcoder cpu_transcoder = crtc->config->cpu_transcoder; + struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state); + enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; /* Clear any frame start delays used for debugging left by the BIOS */ if (crtc->active && !transcoder_is_dsi(cpu_transcoder)) { @@ -15636,7 +15617,7 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc, I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK); } - if (crtc->active) { + if (crtc_state->base.active) { struct intel_plane *plane; /* Disable everything but the primary plane */ @@ -15648,14 +15629,23 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc, plane->base.type != DRM_PLANE_TYPE_PRIMARY) intel_plane_disable_noatomic(crtc, plane); } + + /* + * Disable any background color set by the BIOS, but enable the + * gamma and CSC to match how we program our planes. + */ + if (INTEL_GEN(dev_priv) >= 9) + I915_WRITE(SKL_BOTTOM_COLOR(crtc->pipe), + SKL_BOTTOM_COLOR_GAMMA_ENABLE | + SKL_BOTTOM_COLOR_CSC_ENABLE); } /* Adjust the state of the output pipe according to whether we * have active connectors/encoders. */ - if (crtc->active && !intel_crtc_has_encoders(crtc)) + if (crtc_state->base.active && !intel_crtc_has_encoders(crtc)) intel_crtc_disable_noatomic(&crtc->base, ctx); - if (crtc->active || HAS_GMCH_DISPLAY(dev_priv)) { + if (crtc_state->base.active || HAS_GMCH(dev_priv)) { /* * We start out with underrun reporting disabled to avoid races. * For correct bookkeeping mark this on active crtcs. @@ -15684,15 +15674,45 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc, } } +static bool has_bogus_dpll_config(const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); + + /* + * Some SNB BIOSen (eg. ASUS K53SV) are known to misprogram + * the hardware when a high res displays plugged in. DPLL P + * divider is zero, and the pipe timings are bonkers. We'll + * try to disable everything in that case. + * + * FIXME would be nice to be able to sanitize this state + * without several WARNs, but for now let's take the easy + * road. + */ + return IS_GEN(dev_priv, 6) && + crtc_state->base.active && + crtc_state->shared_dpll && + crtc_state->port_clock == 0; +} + static void intel_sanitize_encoder(struct intel_encoder *encoder) { + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_connector *connector; + struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); + struct intel_crtc_state *crtc_state = crtc ? + to_intel_crtc_state(crtc->base.state) : NULL; /* We need to check both for a crtc link (meaning that the * encoder is active and trying to read from a pipe) and the * pipe itself being active. */ - bool has_active_crtc = encoder->base.crtc && - to_intel_crtc(encoder->base.crtc)->active; + bool has_active_crtc = crtc_state && + crtc_state->base.active; + + if (crtc_state && has_bogus_dpll_config(crtc_state)) { + DRM_DEBUG_KMS("BIOS has misprogrammed the hardware. Disabling pipe %c\n", + pipe_name(crtc->pipe)); + has_active_crtc = false; + } connector = intel_encoder_find_connector(encoder); if (connector && !has_active_crtc) { @@ -15703,15 +15723,25 @@ static void intel_sanitize_encoder(struct intel_encoder *encoder) /* Connector is active, but has no active pipe. This is * fallout from our resume register restoring. Disable * the encoder manually again. */ - if (encoder->base.crtc) { - struct drm_crtc_state *crtc_state = encoder->base.crtc->state; + if (crtc_state) { + struct drm_encoder *best_encoder; DRM_DEBUG_KMS("[ENCODER:%d:%s] manually disabled\n", encoder->base.base.id, encoder->base.name); - encoder->disable(encoder, to_intel_crtc_state(crtc_state), connector->base.state); + + /* avoid oopsing in case the hooks consult best_encoder */ + best_encoder = connector->base.state->best_encoder; + connector->base.state->best_encoder = &encoder->base; + + if (encoder->disable) + encoder->disable(encoder, crtc_state, + connector->base.state); if (encoder->post_disable) - encoder->post_disable(encoder, to_intel_crtc_state(crtc_state), connector->base.state); + encoder->post_disable(encoder, crtc_state, + connector->base.state); + + connector->base.state->best_encoder = best_encoder; } encoder->base.crtc = NULL; @@ -15726,6 +15756,9 @@ static void intel_sanitize_encoder(struct intel_encoder *encoder) /* notify opregion of the sanitized encoder state */ intel_opregion_notify_encoder(encoder, connector && has_active_crtc); + + if (INTEL_GEN(dev_priv) >= 11) + icl_sanitize_encoder_pll_mapping(encoder); } void i915_redisable_vga_power_on(struct drm_i915_private *dev_priv) @@ -15740,19 +15773,25 @@ void i915_redisable_vga_power_on(struct drm_i915_private *dev_priv) void i915_redisable_vga(struct drm_i915_private *dev_priv) { - /* This function can be called both from intel_modeset_setup_hw_state or + intel_wakeref_t wakeref; + + /* + * This function can be called both from intel_modeset_setup_hw_state or * at a very early point in our resume sequence, where the power well * structures are not yet restored. Since this function is at a very * paranoid "someone might have enabled VGA while we were not looking" * level, just check if the power well is enabled instead of trying to * follow the "don't touch the power well if we don't need it" policy - * the rest of the driver uses. */ - if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_VGA)) + * the rest of the driver uses. + */ + wakeref = intel_display_power_get_if_enabled(dev_priv, + POWER_DOMAIN_VGA); + if (!wakeref) return; i915_redisable_vga_power_on(dev_priv); - intel_display_power_put(dev_priv, POWER_DOMAIN_VGA); + intel_display_power_put(dev_priv, POWER_DOMAIN_VGA, wakeref); } /* FIXME read out full plane state for all planes */ @@ -15774,6 +15813,10 @@ static void readout_plane_state(struct drm_i915_private *dev_priv) crtc_state = to_intel_crtc_state(crtc->base.state); intel_set_plane_visible(crtc_state, plane_state, visible); + + DRM_DEBUG_KMS("[PLANE:%d:%s] hw state readout: %s, pipe %c\n", + plane->base.base.id, plane->base.name, + enableddisabled(visible), pipe_name(pipe)); } for_each_intel_crtc(&dev_priv->drm, crtc) { @@ -15926,7 +15969,7 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) drm_calc_timestamping_constants(&crtc->base, &crtc_state->base.adjusted_mode); - update_scanline_offset(crtc); + update_scanline_offset(crtc_state); } dev_priv->min_cdclk[crtc->pipe] = min_cdclk; @@ -15943,8 +15986,6 @@ get_encoder_power_domains(struct drm_i915_private *dev_priv) struct intel_encoder *encoder; for_each_intel_encoder(&dev_priv->drm, encoder) { - u64 get_domains; - enum intel_display_power_domain domain; struct intel_crtc_state *crtc_state; if (!encoder->get_power_domains) @@ -15958,9 +15999,7 @@ get_encoder_power_domains(struct drm_i915_private *dev_priv) continue; crtc_state = to_intel_crtc_state(encoder->base.crtc->state); - get_domains = encoder->get_power_domains(encoder, crtc_state); - for_each_power_domain(domain, get_domains) - intel_display_power_get(dev_priv, domain); + encoder->get_power_domains(encoder, crtc_state); } } @@ -15981,6 +16020,65 @@ static void intel_early_display_was(struct drm_i915_private *dev_priv) } } +static void ibx_sanitize_pch_hdmi_port(struct drm_i915_private *dev_priv, + enum port port, i915_reg_t hdmi_reg) +{ + u32 val = I915_READ(hdmi_reg); + + if (val & SDVO_ENABLE || + (val & SDVO_PIPE_SEL_MASK) == SDVO_PIPE_SEL(PIPE_A)) + return; + + DRM_DEBUG_KMS("Sanitizing transcoder select for HDMI %c\n", + port_name(port)); + + val &= ~SDVO_PIPE_SEL_MASK; + val |= SDVO_PIPE_SEL(PIPE_A); + + I915_WRITE(hdmi_reg, val); +} + +static void ibx_sanitize_pch_dp_port(struct drm_i915_private *dev_priv, + enum port port, i915_reg_t dp_reg) +{ + u32 val = I915_READ(dp_reg); + + if (val & DP_PORT_EN || + (val & DP_PIPE_SEL_MASK) == DP_PIPE_SEL(PIPE_A)) + return; + + DRM_DEBUG_KMS("Sanitizing transcoder select for DP %c\n", + port_name(port)); + + val &= ~DP_PIPE_SEL_MASK; + val |= DP_PIPE_SEL(PIPE_A); + + I915_WRITE(dp_reg, val); +} + +static void ibx_sanitize_pch_ports(struct drm_i915_private *dev_priv) +{ + /* + * The BIOS may select transcoder B on some of the PCH + * ports even it doesn't enable the port. This would trip + * assert_pch_dp_disabled() and assert_pch_hdmi_disabled(). + * Sanitize the transcoder select bits to prevent that. We + * assume that the BIOS never actually enabled the port, + * because if it did we'd actually have to toggle the port + * on and back off to make the transcoder A select stick + * (see. intel_dp_link_down(), intel_disable_hdmi(), + * intel_disable_sdvo()). + */ + ibx_sanitize_pch_dp_port(dev_priv, PORT_B, PCH_DP_B); + ibx_sanitize_pch_dp_port(dev_priv, PORT_C, PCH_DP_C); + ibx_sanitize_pch_dp_port(dev_priv, PORT_D, PCH_DP_D); + + /* PCH SDVOB multiplex with HDMIB */ + ibx_sanitize_pch_hdmi_port(dev_priv, PORT_B, PCH_HDMIB); + ibx_sanitize_pch_hdmi_port(dev_priv, PORT_C, PCH_HDMIC); + ibx_sanitize_pch_hdmi_port(dev_priv, PORT_D, PCH_HDMID); +} + /* Scan out the current hw modeset state, * and sanitizes it to the current state */ @@ -15989,11 +16087,13 @@ intel_modeset_setup_hw_state(struct drm_device *dev, struct drm_modeset_acquire_ctx *ctx) { struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_crtc *crtc; + struct intel_crtc_state *crtc_state; struct intel_encoder *encoder; + struct intel_crtc *crtc; + intel_wakeref_t wakeref; int i; - intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); + wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); intel_early_display_was(dev_priv); intel_modeset_readout_hw_state(dev); @@ -16001,15 +16101,20 @@ intel_modeset_setup_hw_state(struct drm_device *dev, /* HW state is read out, now we need to sanitize this mess. */ get_encoder_power_domains(dev_priv); + if (HAS_PCH_IBX(dev_priv)) + ibx_sanitize_pch_ports(dev_priv); + /* * intel_sanitize_plane_mapping() may need to do vblank * waits, so we need vblank interrupts restored beforehand. */ for_each_intel_crtc(&dev_priv->drm, crtc) { + crtc_state = to_intel_crtc_state(crtc->base.state); + drm_crtc_vblank_reset(&crtc->base); - if (crtc->active) - drm_crtc_vblank_on(&crtc->base); + if (crtc_state->base.active) + intel_crtc_vblank_on(crtc_state); } intel_sanitize_plane_mapping(dev_priv); @@ -16018,8 +16123,9 @@ intel_modeset_setup_hw_state(struct drm_device *dev, intel_sanitize_encoder(encoder); for_each_intel_crtc(&dev_priv->drm, crtc) { + crtc_state = to_intel_crtc_state(crtc->base.state); intel_sanitize_crtc(crtc, ctx); - intel_dump_pipe_config(crtc, crtc->config, + intel_dump_pipe_config(crtc, crtc_state, "[setup_hw_state]"); } @@ -16039,26 +16145,27 @@ intel_modeset_setup_hw_state(struct drm_device *dev, } if (IS_G4X(dev_priv)) { - g4x_wm_get_hw_state(dev); + g4x_wm_get_hw_state(dev_priv); g4x_wm_sanitize(dev_priv); } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { - vlv_wm_get_hw_state(dev); + vlv_wm_get_hw_state(dev_priv); vlv_wm_sanitize(dev_priv); } else if (INTEL_GEN(dev_priv) >= 9) { - skl_wm_get_hw_state(dev); + skl_wm_get_hw_state(dev_priv); } else if (HAS_PCH_SPLIT(dev_priv)) { - ilk_wm_get_hw_state(dev); + ilk_wm_get_hw_state(dev_priv); } for_each_intel_crtc(dev, crtc) { u64 put_domains; - put_domains = modeset_get_crtc_power_domains(&crtc->base, crtc->config); + crtc_state = to_intel_crtc_state(crtc->base.state); + put_domains = modeset_get_crtc_power_domains(&crtc->base, crtc_state); if (WARN_ON(put_domains)) modeset_put_power_domains(dev_priv, put_domains); } - intel_display_power_put(dev_priv, POWER_DOMAIN_INIT); + intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, wakeref); intel_fbc_init_pipe_state(dev_priv); } @@ -16097,29 +16204,6 @@ void intel_display_resume(struct drm_device *dev) drm_atomic_state_put(state); } -int intel_connector_register(struct drm_connector *connector) -{ - struct intel_connector *intel_connector = to_intel_connector(connector); - int ret; - - ret = intel_backlight_device_register(intel_connector); - if (ret) - goto err; - - return 0; - -err: - return ret; -} - -void intel_connector_unregister(struct drm_connector *connector) -{ - struct intel_connector *intel_connector = to_intel_connector(connector); - - intel_backlight_device_unregister(intel_connector); - intel_panel_destroy_backlight(connector); -} - static void intel_hpd_poll_fini(struct drm_device *dev) { struct intel_connector *connector; @@ -16130,9 +16214,9 @@ static void intel_hpd_poll_fini(struct drm_device *dev) for_each_intel_connector_iter(connector, &conn_iter) { if (connector->modeset_retry_work.func) cancel_work_sync(&connector->modeset_retry_work); - if (connector->hdcp_shim) { - cancel_delayed_work_sync(&connector->hdcp_check_work); - cancel_work_sync(&connector->hdcp_prop_work); + if (connector->hdcp.shim) { + cancel_delayed_work_sync(&connector->hdcp.check_work); + cancel_work_sync(&connector->hdcp.prop_work); } } drm_connector_list_iter_end(&conn_iter); @@ -16172,18 +16256,13 @@ void intel_modeset_cleanup(struct drm_device *dev) drm_mode_config_cleanup(dev); - intel_cleanup_overlay(dev_priv); + intel_overlay_cleanup(dev_priv); intel_teardown_gmbus(dev_priv); destroy_workqueue(dev_priv->modeset_wq); -} -void intel_connector_attach_encoder(struct intel_connector *connector, - struct intel_encoder *encoder) -{ - connector->encoder = encoder; - drm_connector_attach_encoder(&connector->base, &encoder->base); + intel_fbc_cleanup_cfb(dev_priv); } /* @@ -16273,7 +16352,7 @@ intel_display_capture_error_state(struct drm_i915_private *dev_priv) }; int i; - if (INTEL_INFO(dev_priv)->num_pipes == 0) + if (!HAS_DISPLAY(dev_priv)) return NULL; error = kzalloc(sizeof(*error), GFP_ATOMIC); @@ -16309,7 +16388,7 @@ intel_display_capture_error_state(struct drm_i915_private *dev_priv) error->pipe[i].source = I915_READ(PIPESRC(i)); - if (HAS_GMCH_DISPLAY(dev_priv)) + if (HAS_GMCH(dev_priv)) error->pipe[i].stat = I915_READ(PIPESTAT(i)); } diff --git a/drivers/gpu/drm/i915/intel_display.h b/drivers/gpu/drm/i915/intel_display.h index 9fac67e31205..2220588e86ac 100644 --- a/drivers/gpu/drm/i915/intel_display.h +++ b/drivers/gpu/drm/i915/intel_display.h @@ -26,6 +26,7 @@ #define _INTEL_DISPLAY_H_ #include <drm/drm_util.h> +#include <drm/i915_drm.h> enum i915_gpio { GPIOA, @@ -43,6 +44,11 @@ enum i915_gpio { GPIOM, }; +/* + * Keep the pipe enum values fixed: the code assumes that PIPE_A=0, the + * rest have consecutive values and match the enum values of transcoders + * with a 1:1 transcoder -> pipe mapping. + */ enum pipe { INVALID_PIPE = -1, @@ -57,12 +63,25 @@ enum pipe { #define pipe_name(p) ((p) + 'A') enum transcoder { - TRANSCODER_A = 0, - TRANSCODER_B, - TRANSCODER_C, + /* + * The following transcoders have a 1:1 transcoder -> pipe mapping, + * keep their values fixed: the code assumes that TRANSCODER_A=0, the + * rest have consecutive values and match the enum values of the pipes + * they map to. + */ + TRANSCODER_A = PIPE_A, + TRANSCODER_B = PIPE_B, + TRANSCODER_C = PIPE_C, + + /* + * The following transcoders can map to any pipe, their enum value + * doesn't need to stay fixed. + */ TRANSCODER_EDP, - TRANSCODER_DSI_A, - TRANSCODER_DSI_C, + TRANSCODER_DSI_0, + TRANSCODER_DSI_1, + TRANSCODER_DSI_A = TRANSCODER_DSI_0, /* legacy DSI */ + TRANSCODER_DSI_C = TRANSCODER_DSI_1, /* legacy DSI */ I915_MAX_TRANSCODERS }; @@ -103,7 +122,7 @@ enum i9xx_plane_id { }; #define plane_name(p) ((p) + 'A') -#define sprite_name(p, s) ((p) * INTEL_INFO(dev_priv)->num_sprites[(p)] + (s) + 'A') +#define sprite_name(p, s) ((p) * RUNTIME_INFO(dev_priv)->num_sprites[(p)] + (s) + 'A') /* * Per-pipe plane identifier. @@ -120,6 +139,9 @@ enum plane_id { PLANE_SPRITE0, PLANE_SPRITE1, PLANE_SPRITE2, + PLANE_SPRITE3, + PLANE_SPRITE4, + PLANE_SPRITE5, PLANE_CURSOR, I915_MAX_PLANES, @@ -129,21 +151,6 @@ enum plane_id { for ((__p) = PLANE_PRIMARY; (__p) < I915_MAX_PLANES; (__p)++) \ for_each_if((__crtc)->plane_ids_mask & BIT(__p)) -enum port { - PORT_NONE = -1, - - PORT_A = 0, - PORT_B, - PORT_C, - PORT_D, - PORT_E, - PORT_F, - - I915_MAX_PORTS -}; - -#define port_name(p) ((p) + 'A') - /* * Ports identifier referenced from other drivers. * Expected to remain stable over time @@ -221,6 +228,7 @@ enum intel_display_power_domain { POWER_DOMAIN_TRANSCODER_B, POWER_DOMAIN_TRANSCODER_C, POWER_DOMAIN_TRANSCODER_EDP, + POWER_DOMAIN_TRANSCODER_EDP_VDSC, POWER_DOMAIN_TRANSCODER_DSI_A, POWER_DOMAIN_TRANSCODER_DSI_C, POWER_DOMAIN_PORT_DDI_A_LANES, @@ -289,12 +297,12 @@ struct intel_link_m_n { #define for_each_universal_plane(__dev_priv, __pipe, __p) \ for ((__p) = 0; \ - (__p) < INTEL_INFO(__dev_priv)->num_sprites[(__pipe)] + 1; \ + (__p) < RUNTIME_INFO(__dev_priv)->num_sprites[(__pipe)] + 1; \ (__p)++) #define for_each_sprite(__dev_priv, __p, __s) \ for ((__s) = 0; \ - (__s) < INTEL_INFO(__dev_priv)->num_sprites[(__p)]; \ + (__s) < RUNTIME_INFO(__dev_priv)->num_sprites[(__p)]; \ (__s)++) #define for_each_port_masked(__port, __ports_mask) \ @@ -363,7 +371,7 @@ struct intel_link_m_n { (__dev_priv)->power_domains.power_well_count; \ (__power_well)++) -#define for_each_power_well_rev(__dev_priv, __power_well) \ +#define for_each_power_well_reverse(__dev_priv, __power_well) \ for ((__power_well) = (__dev_priv)->power_domains.power_wells + \ (__dev_priv)->power_domains.power_well_count - 1; \ (__power_well) - (__dev_priv)->power_domains.power_wells >= 0; \ @@ -373,10 +381,18 @@ struct intel_link_m_n { for_each_power_well(__dev_priv, __power_well) \ for_each_if((__power_well)->desc->domains & (__domain_mask)) -#define for_each_power_domain_well_rev(__dev_priv, __power_well, __domain_mask) \ - for_each_power_well_rev(__dev_priv, __power_well) \ +#define for_each_power_domain_well_reverse(__dev_priv, __power_well, __domain_mask) \ + for_each_power_well_reverse(__dev_priv, __power_well) \ for_each_if((__power_well)->desc->domains & (__domain_mask)) +#define for_each_old_intel_plane_in_state(__state, plane, old_plane_state, __i) \ + for ((__i) = 0; \ + (__i) < (__state)->base.dev->mode_config.num_total_plane && \ + ((plane) = to_intel_plane((__state)->base.planes[__i].ptr), \ + (old_plane_state) = to_intel_plane_state((__state)->base.planes[__i].old_state), 1); \ + (__i)++) \ + for_each_if(plane) + #define for_each_new_intel_plane_in_state(__state, plane, new_plane_state, __i) \ for ((__i) = 0; \ (__i) < (__state)->base.dev->mode_config.num_total_plane && \ @@ -402,10 +418,18 @@ struct intel_link_m_n { (__i)++) \ for_each_if(plane) -void intel_link_compute_m_n(int bpp, int nlanes, +#define for_each_oldnew_intel_crtc_in_state(__state, crtc, old_crtc_state, new_crtc_state, __i) \ + for ((__i) = 0; \ + (__i) < (__state)->base.dev->mode_config.num_crtc && \ + ((crtc) = to_intel_crtc((__state)->base.crtcs[__i].ptr), \ + (old_crtc_state) = to_intel_crtc_state((__state)->base.crtcs[__i].old_state), \ + (new_crtc_state) = to_intel_crtc_state((__state)->base.crtcs[__i].new_state), 1); \ + (__i)++) \ + for_each_if(crtc) + +void intel_link_compute_m_n(u16 bpp, int nlanes, int pixel_clock, int link_clock, struct intel_link_m_n *m_n, bool constant_n); - bool is_ccs_modifier(u64 modifier); #endif diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c index 13f9b56a9ce7..48da4a969a0a 100644 --- a/drivers/gpu/drm/i915/intel_dp.c +++ b/drivers/gpu/drm/i915/intel_dp.c @@ -32,19 +32,31 @@ #include <linux/notifier.h> #include <linux/reboot.h> #include <asm/byteorder.h> -#include <drm/drmP.h> #include <drm/drm_atomic_helper.h> #include <drm/drm_crtc.h> -#include <drm/drm_crtc_helper.h> #include <drm/drm_dp_helper.h> #include <drm/drm_edid.h> #include <drm/drm_hdcp.h> +#include <drm/drm_probe_helper.h> #include "intel_drv.h" #include <drm/i915_drm.h> #include "i915_drv.h" #define DP_DPRX_ESI_LEN 14 +/* DP DSC small joiner has 2 FIFOs each of 640 x 6 bytes */ +#define DP_DSC_MAX_SMALL_JOINER_RAM_BUFFER 61440 +#define DP_DSC_MIN_SUPPORTED_BPC 8 +#define DP_DSC_MAX_SUPPORTED_BPC 10 + +/* DP DSC throughput values used for slice count calculations KPixels/s */ +#define DP_DSC_PEAK_PIXEL_RATE 2720000 +#define DP_DSC_MAX_ENC_THROUGHPUT_0 340000 +#define DP_DSC_MAX_ENC_THROUGHPUT_1 400000 + +/* DP DSC FEC Overhead factor = (100 - 2.4)/100 */ +#define DP_DSC_FEC_OVERHEAD_FACTOR 976 + /* Compliance test status bits */ #define INTEL_DP_RESOLUTION_SHIFT_MASK 0 #define INTEL_DP_RESOLUTION_PREFERRED (1 << INTEL_DP_RESOLUTION_SHIFT_MASK) @@ -93,6 +105,14 @@ static const struct dp_link_dpll chv_dpll[] = { { .p1 = 4, .p2 = 1, .n = 1, .m1 = 2, .m2 = 0x6c00000 } }, }; +/* Constants for DP DSC configurations */ +static const u8 valid_dsc_bpp[] = {6, 8, 10, 12, 15}; + +/* With Single pipe configuration, HW is capable of supporting maximum + * of 4 slices per line. + */ +static const u8 valid_dsc_slicecount[] = {1, 2, 4}; + /** * intel_dp_is_edp - is the given port attached to an eDP panel (either CPU or PCH) * @intel_dp: DP struct @@ -222,138 +242,6 @@ intel_dp_link_required(int pixel_clock, int bpp) return DIV_ROUND_UP(pixel_clock * bpp, 8); } -void icl_program_mg_dp_mode(struct intel_dp *intel_dp) -{ - struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - enum port port = intel_dig_port->base.port; - enum tc_port tc_port = intel_port_to_tc(dev_priv, port); - u32 ln0, ln1, lane_info; - - if (tc_port == PORT_TC_NONE || intel_dig_port->tc_type == TC_PORT_TBT) - return; - - ln0 = I915_READ(MG_DP_MODE(port, 0)); - ln1 = I915_READ(MG_DP_MODE(port, 1)); - - switch (intel_dig_port->tc_type) { - case TC_PORT_TYPEC: - ln0 &= ~(MG_DP_MODE_CFG_DP_X1_MODE | MG_DP_MODE_CFG_DP_X2_MODE); - ln1 &= ~(MG_DP_MODE_CFG_DP_X1_MODE | MG_DP_MODE_CFG_DP_X2_MODE); - - lane_info = (I915_READ(PORT_TX_DFLEXDPSP) & - DP_LANE_ASSIGNMENT_MASK(tc_port)) >> - DP_LANE_ASSIGNMENT_SHIFT(tc_port); - - switch (lane_info) { - case 0x1: - case 0x4: - break; - case 0x2: - ln0 |= MG_DP_MODE_CFG_DP_X1_MODE; - break; - case 0x3: - ln0 |= MG_DP_MODE_CFG_DP_X1_MODE | - MG_DP_MODE_CFG_DP_X2_MODE; - break; - case 0x8: - ln1 |= MG_DP_MODE_CFG_DP_X1_MODE; - break; - case 0xC: - ln1 |= MG_DP_MODE_CFG_DP_X1_MODE | - MG_DP_MODE_CFG_DP_X2_MODE; - break; - case 0xF: - ln0 |= MG_DP_MODE_CFG_DP_X1_MODE | - MG_DP_MODE_CFG_DP_X2_MODE; - ln1 |= MG_DP_MODE_CFG_DP_X1_MODE | - MG_DP_MODE_CFG_DP_X2_MODE; - break; - default: - MISSING_CASE(lane_info); - } - break; - - case TC_PORT_LEGACY: - ln0 |= MG_DP_MODE_CFG_DP_X1_MODE | MG_DP_MODE_CFG_DP_X2_MODE; - ln1 |= MG_DP_MODE_CFG_DP_X1_MODE | MG_DP_MODE_CFG_DP_X2_MODE; - break; - - default: - MISSING_CASE(intel_dig_port->tc_type); - return; - } - - I915_WRITE(MG_DP_MODE(port, 0), ln0); - I915_WRITE(MG_DP_MODE(port, 1), ln1); -} - -void icl_enable_phy_clock_gating(struct intel_digital_port *dig_port) -{ - struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); - enum port port = dig_port->base.port; - enum tc_port tc_port = intel_port_to_tc(dev_priv, port); - i915_reg_t mg_regs[2] = { MG_DP_MODE(port, 0), MG_DP_MODE(port, 1) }; - u32 val; - int i; - - if (tc_port == PORT_TC_NONE) - return; - - for (i = 0; i < ARRAY_SIZE(mg_regs); i++) { - val = I915_READ(mg_regs[i]); - val |= MG_DP_MODE_CFG_TR2PWR_GATING | - MG_DP_MODE_CFG_TRPWR_GATING | - MG_DP_MODE_CFG_CLNPWR_GATING | - MG_DP_MODE_CFG_DIGPWR_GATING | - MG_DP_MODE_CFG_GAONPWR_GATING; - I915_WRITE(mg_regs[i], val); - } - - val = I915_READ(MG_MISC_SUS0(tc_port)); - val |= MG_MISC_SUS0_SUSCLK_DYNCLKGATE_MODE(3) | - MG_MISC_SUS0_CFG_TR2PWR_GATING | - MG_MISC_SUS0_CFG_CL2PWR_GATING | - MG_MISC_SUS0_CFG_GAONPWR_GATING | - MG_MISC_SUS0_CFG_TRPWR_GATING | - MG_MISC_SUS0_CFG_CL1PWR_GATING | - MG_MISC_SUS0_CFG_DGPWR_GATING; - I915_WRITE(MG_MISC_SUS0(tc_port), val); -} - -void icl_disable_phy_clock_gating(struct intel_digital_port *dig_port) -{ - struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); - enum port port = dig_port->base.port; - enum tc_port tc_port = intel_port_to_tc(dev_priv, port); - i915_reg_t mg_regs[2] = { MG_DP_MODE(port, 0), MG_DP_MODE(port, 1) }; - u32 val; - int i; - - if (tc_port == PORT_TC_NONE) - return; - - for (i = 0; i < ARRAY_SIZE(mg_regs); i++) { - val = I915_READ(mg_regs[i]); - val &= ~(MG_DP_MODE_CFG_TR2PWR_GATING | - MG_DP_MODE_CFG_TRPWR_GATING | - MG_DP_MODE_CFG_CLNPWR_GATING | - MG_DP_MODE_CFG_DIGPWR_GATING | - MG_DP_MODE_CFG_GAONPWR_GATING); - I915_WRITE(mg_regs[i], val); - } - - val = I915_READ(MG_MISC_SUS0(tc_port)); - val &= ~(MG_MISC_SUS0_SUSCLK_DYNCLKGATE_MODE_MASK | - MG_MISC_SUS0_CFG_TR2PWR_GATING | - MG_MISC_SUS0_CFG_CL2PWR_GATING | - MG_MISC_SUS0_CFG_GAONPWR_GATING | - MG_MISC_SUS0_CFG_TRPWR_GATING | - MG_MISC_SUS0_CFG_CL1PWR_GATING | - MG_MISC_SUS0_CFG_DGPWR_GATING); - I915_WRITE(MG_MISC_SUS0(tc_port), val); -} - int intel_dp_max_data_rate(int max_link_clock, int max_lanes) { @@ -415,9 +303,11 @@ static int cnl_max_source_rate(struct intel_dp *intel_dp) static int icl_max_source_rate(struct intel_dp *intel_dp) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); enum port port = dig_port->base.port; - if (port == PORT_B) + if (intel_port_is_combophy(dev_priv, port) && + !intel_dp_is_edp(intel_dp)) return 540000; return 810000; @@ -455,7 +345,7 @@ intel_dp_set_source_rates(struct intel_dp *intel_dp) if (INTEL_GEN(dev_priv) >= 10) { source_rates = cnl_rates; size = ARRAY_SIZE(cnl_rates); - if (INTEL_GEN(dev_priv) == 10) + if (IS_GEN(dev_priv, 10)) max_rate = cnl_max_source_rate(intel_dp); else max_rate = icl_max_source_rate(intel_dp); @@ -539,7 +429,7 @@ static void intel_dp_set_common_rates(struct intel_dp *intel_dp) } static bool intel_dp_link_params_valid(struct intel_dp *intel_dp, int link_rate, - uint8_t lane_count) + u8 lane_count) { /* * FIXME: we need to synchronize the current link parameters with @@ -559,7 +449,7 @@ static bool intel_dp_link_params_valid(struct intel_dp *intel_dp, int link_rate, static bool intel_dp_can_link_train_fallback_for_edp(struct intel_dp *intel_dp, int link_rate, - uint8_t lane_count) + u8 lane_count) { const struct drm_display_mode *fixed_mode = intel_dp->attached_connector->panel.fixed_mode; @@ -574,7 +464,7 @@ static bool intel_dp_can_link_train_fallback_for_edp(struct intel_dp *intel_dp, } int intel_dp_get_link_train_fallback_values(struct intel_dp *intel_dp, - int link_rate, uint8_t lane_count) + int link_rate, u8 lane_count) { int index; @@ -616,9 +506,12 @@ intel_dp_mode_valid(struct drm_connector *connector, struct intel_dp *intel_dp = intel_attached_dp(connector); struct intel_connector *intel_connector = to_intel_connector(connector); struct drm_display_mode *fixed_mode = intel_connector->panel.fixed_mode; + struct drm_i915_private *dev_priv = to_i915(connector->dev); int target_clock = mode->clock; int max_rate, mode_rate, max_lanes, max_link_clock; int max_dotclk; + u16 dsc_max_output_bpp = 0; + u8 dsc_slice_count = 0; if (mode->flags & DRM_MODE_FLAG_DBLSCAN) return MODE_NO_DBLESCAN; @@ -641,7 +534,33 @@ intel_dp_mode_valid(struct drm_connector *connector, max_rate = intel_dp_max_data_rate(max_link_clock, max_lanes); mode_rate = intel_dp_link_required(target_clock, 18); - if (mode_rate > max_rate || target_clock > max_dotclk) + /* + * Output bpp is stored in 6.4 format so right shift by 4 to get the + * integer value since we support only integer values of bpp. + */ + if ((INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) && + drm_dp_sink_supports_dsc(intel_dp->dsc_dpcd)) { + if (intel_dp_is_edp(intel_dp)) { + dsc_max_output_bpp = + drm_edp_dsc_sink_output_bpp(intel_dp->dsc_dpcd) >> 4; + dsc_slice_count = + drm_dp_dsc_sink_max_slice_count(intel_dp->dsc_dpcd, + true); + } else if (drm_dp_sink_supports_fec(intel_dp->fec_capable)) { + dsc_max_output_bpp = + intel_dp_dsc_get_output_bpp(max_link_clock, + max_lanes, + target_clock, + mode->hdisplay) >> 4; + dsc_slice_count = + intel_dp_dsc_get_slice_count(intel_dp, + target_clock, + mode->hdisplay); + } + } + + if ((mode_rate > max_rate && !(dsc_max_output_bpp && dsc_slice_count)) || + target_clock > max_dotclk) return MODE_CLOCK_HIGH; if (mode->clock < 10000) @@ -653,19 +572,19 @@ intel_dp_mode_valid(struct drm_connector *connector, return MODE_OK; } -uint32_t intel_dp_pack_aux(const uint8_t *src, int src_bytes) +u32 intel_dp_pack_aux(const u8 *src, int src_bytes) { - int i; - uint32_t v = 0; + int i; + u32 v = 0; if (src_bytes > 4) src_bytes = 4; for (i = 0; i < src_bytes; i++) - v |= ((uint32_t) src[i]) << ((3-i) * 8); + v |= ((u32)src[i]) << ((3 - i) * 8); return v; } -static void intel_dp_unpack_aux(uint32_t src, uint8_t *dst, int dst_bytes) +static void intel_dp_unpack_aux(u32 src, u8 *dst, int dst_bytes) { int i; if (dst_bytes > 4) @@ -682,28 +601,39 @@ intel_dp_init_panel_power_sequencer_registers(struct intel_dp *intel_dp, static void intel_dp_pps_init(struct intel_dp *intel_dp); -static void pps_lock(struct intel_dp *intel_dp) +static intel_wakeref_t +pps_lock(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + intel_wakeref_t wakeref; /* * See intel_power_sequencer_reset() why we need * a power domain reference here. */ - intel_display_power_get(dev_priv, intel_dp->aux_power_domain); + wakeref = intel_display_power_get(dev_priv, + intel_aux_power_domain(dp_to_dig_port(intel_dp))); mutex_lock(&dev_priv->pps_mutex); + + return wakeref; } -static void pps_unlock(struct intel_dp *intel_dp) +static intel_wakeref_t +pps_unlock(struct intel_dp *intel_dp, intel_wakeref_t wakeref) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); mutex_unlock(&dev_priv->pps_mutex); - - intel_display_power_put(dev_priv, intel_dp->aux_power_domain); + intel_display_power_put(dev_priv, + intel_aux_power_domain(dp_to_dig_port(intel_dp)), + wakeref); + return 0; } +#define with_pps_lock(dp, wf) \ + for ((wf) = pps_lock(dp); (wf); (wf) = pps_unlock((dp), (wf))) + static void vlv_power_sequencer_kick(struct intel_dp *intel_dp) { @@ -713,7 +643,7 @@ vlv_power_sequencer_kick(struct intel_dp *intel_dp) bool pll_enabled, release_cl_override = false; enum dpio_phy phy = DPIO_PHY(pipe); enum dpio_channel ch = vlv_pipe_to_channel(pipe); - uint32_t DP; + u32 DP; if (WARN(I915_READ(intel_dp->output_reg) & DP_PORT_EN, "skipping pipe %c power sequencer kick due to port %c being active\n", @@ -1052,30 +982,29 @@ static int edp_notify_handler(struct notifier_block *this, unsigned long code, struct intel_dp *intel_dp = container_of(this, typeof(* intel_dp), edp_notifier); struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + intel_wakeref_t wakeref; if (!intel_dp_is_edp(intel_dp) || code != SYS_RESTART) return 0; - pps_lock(intel_dp); - - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { - enum pipe pipe = vlv_power_sequencer_pipe(intel_dp); - i915_reg_t pp_ctrl_reg, pp_div_reg; - u32 pp_div; - - pp_ctrl_reg = PP_CONTROL(pipe); - pp_div_reg = PP_DIVISOR(pipe); - pp_div = I915_READ(pp_div_reg); - pp_div &= PP_REFERENCE_DIVIDER_MASK; - - /* 0x1F write to PP_DIV_REG sets max cycle delay */ - I915_WRITE(pp_div_reg, pp_div | 0x1F); - I915_WRITE(pp_ctrl_reg, PANEL_UNLOCK_REGS | PANEL_POWER_OFF); - msleep(intel_dp->panel_power_cycle_delay); + with_pps_lock(intel_dp, wakeref) { + if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { + enum pipe pipe = vlv_power_sequencer_pipe(intel_dp); + i915_reg_t pp_ctrl_reg, pp_div_reg; + u32 pp_div; + + pp_ctrl_reg = PP_CONTROL(pipe); + pp_div_reg = PP_DIVISOR(pipe); + pp_div = I915_READ(pp_div_reg); + pp_div &= PP_REFERENCE_DIVIDER_MASK; + + /* 0x1F write to PP_DIV_REG sets max cycle delay */ + I915_WRITE(pp_div_reg, pp_div | 0x1F); + I915_WRITE(pp_ctrl_reg, PANEL_UNLOCK_REGS); + msleep(intel_dp->panel_power_cycle_delay); + } } - pps_unlock(intel_dp); - return 0; } @@ -1121,17 +1050,21 @@ intel_dp_check_edp(struct intel_dp *intel_dp) } } -static uint32_t +static u32 intel_dp_aux_wait_done(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); i915_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg(intel_dp); - uint32_t status; + u32 status; bool done; #define C (((status = I915_READ_NOTRACE(ch_ctl)) & DP_AUX_CH_CTL_SEND_BUSY) == 0) done = wait_event_timeout(dev_priv->gmbus_wait_queue, C, msecs_to_jiffies_timeout(10)); + + /* just trace the final value */ + trace_i915_reg_rw(false, ch_ctl, status, sizeof(status), true); + if (!done) DRM_ERROR("dp aux hw did not signal timeout!\n"); #undef C @@ -1139,7 +1072,7 @@ intel_dp_aux_wait_done(struct intel_dp *intel_dp) return status; } -static uint32_t g4x_get_aux_clock_divider(struct intel_dp *intel_dp, int index) +static u32 g4x_get_aux_clock_divider(struct intel_dp *intel_dp, int index) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); @@ -1153,9 +1086,10 @@ static uint32_t g4x_get_aux_clock_divider(struct intel_dp *intel_dp, int index) return DIV_ROUND_CLOSEST(dev_priv->rawclk_freq, 2000); } -static uint32_t ilk_get_aux_clock_divider(struct intel_dp *intel_dp, int index) +static u32 ilk_get_aux_clock_divider(struct intel_dp *intel_dp, int index) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); if (index) return 0; @@ -1165,17 +1099,18 @@ static uint32_t ilk_get_aux_clock_divider(struct intel_dp *intel_dp, int index) * like to run at 2MHz. So, take the cdclk or PCH rawclk value and * divide by 2000 and use that */ - if (intel_dp->aux_ch == AUX_CH_A) + if (dig_port->aux_ch == AUX_CH_A) return DIV_ROUND_CLOSEST(dev_priv->cdclk.hw.cdclk, 2000); else return DIV_ROUND_CLOSEST(dev_priv->rawclk_freq, 2000); } -static uint32_t hsw_get_aux_clock_divider(struct intel_dp *intel_dp, int index) +static u32 hsw_get_aux_clock_divider(struct intel_dp *intel_dp, int index) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - if (intel_dp->aux_ch != AUX_CH_A && HAS_PCH_LPT_H(dev_priv)) { + if (dig_port->aux_ch != AUX_CH_A && HAS_PCH_LPT_H(dev_priv)) { /* Workaround for non-ULT HSW */ switch (index) { case 0: return 63; @@ -1187,7 +1122,7 @@ static uint32_t hsw_get_aux_clock_divider(struct intel_dp *intel_dp, int index) return ilk_get_aux_clock_divider(intel_dp, index); } -static uint32_t skl_get_aux_clock_divider(struct intel_dp *intel_dp, int index) +static u32 skl_get_aux_clock_divider(struct intel_dp *intel_dp, int index) { /* * SKL doesn't need us to program the AUX clock divider (Hardware will @@ -1197,16 +1132,16 @@ static uint32_t skl_get_aux_clock_divider(struct intel_dp *intel_dp, int index) return index ? 0 : 1; } -static uint32_t g4x_get_aux_send_ctl(struct intel_dp *intel_dp, - int send_bytes, - uint32_t aux_clock_divider) +static u32 g4x_get_aux_send_ctl(struct intel_dp *intel_dp, + int send_bytes, + u32 aux_clock_divider) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev); - uint32_t precharge, timeout; + u32 precharge, timeout; - if (IS_GEN6(dev_priv)) + if (IS_GEN(dev_priv, 6)) precharge = 3; else precharge = 5; @@ -1227,12 +1162,12 @@ static uint32_t g4x_get_aux_send_ctl(struct intel_dp *intel_dp, (aux_clock_divider << DP_AUX_CH_CTL_BIT_CLOCK_2X_SHIFT); } -static uint32_t skl_get_aux_send_ctl(struct intel_dp *intel_dp, - int send_bytes, - uint32_t unused) +static u32 skl_get_aux_send_ctl(struct intel_dp *intel_dp, + int send_bytes, + u32 unused) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - uint32_t ret; + u32 ret; ret = DP_AUX_CH_CTL_SEND_BUSY | DP_AUX_CH_CTL_DONE | @@ -1252,25 +1187,26 @@ static uint32_t skl_get_aux_send_ctl(struct intel_dp *intel_dp, static int intel_dp_aux_xfer(struct intel_dp *intel_dp, - const uint8_t *send, int send_bytes, - uint8_t *recv, int recv_size, + const u8 *send, int send_bytes, + u8 *recv, int recv_size, u32 aux_send_ctl_flags) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev); i915_reg_t ch_ctl, ch_data[5]; - uint32_t aux_clock_divider; + u32 aux_clock_divider; + intel_wakeref_t wakeref; int i, ret, recv_bytes; - uint32_t status; int try, clock = 0; + u32 status; bool vdd; ch_ctl = intel_dp->aux_ch_ctl_reg(intel_dp); for (i = 0; i < ARRAY_SIZE(ch_data); i++) ch_data[i] = intel_dp->aux_ch_data_reg(intel_dp, i); - pps_lock(intel_dp); + wakeref = pps_lock(intel_dp); /* * We will be called with VDD already enabled for dpcd/edid/oui reads. @@ -1295,6 +1231,8 @@ intel_dp_aux_xfer(struct intel_dp *intel_dp, break; msleep(1); } + /* just trace the final value */ + trace_i915_reg_rw(false, ch_ctl, status, sizeof(status), true); if (try == 3) { static u32 last_status = -1; @@ -1414,7 +1352,7 @@ out: if (vdd) edp_panel_vdd_off(intel_dp, false); - pps_unlock(intel_dp); + pps_unlock(intel_dp, wakeref); return ret; } @@ -1436,7 +1374,7 @@ static ssize_t intel_dp_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg) { struct intel_dp *intel_dp = container_of(aux, struct intel_dp, aux); - uint8_t txbuf[20], rxbuf[20]; + u8 txbuf[20], rxbuf[20]; size_t txsize, rxsize; int ret; @@ -1503,80 +1441,12 @@ intel_dp_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg) return ret; } -static enum aux_ch intel_aux_ch(struct intel_dp *intel_dp) -{ - struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - enum port port = encoder->port; - const struct ddi_vbt_port_info *info = - &dev_priv->vbt.ddi_port_info[port]; - enum aux_ch aux_ch; - - if (!info->alternate_aux_channel) { - aux_ch = (enum aux_ch) port; - - DRM_DEBUG_KMS("using AUX %c for port %c (platform default)\n", - aux_ch_name(aux_ch), port_name(port)); - return aux_ch; - } - - switch (info->alternate_aux_channel) { - case DP_AUX_A: - aux_ch = AUX_CH_A; - break; - case DP_AUX_B: - aux_ch = AUX_CH_B; - break; - case DP_AUX_C: - aux_ch = AUX_CH_C; - break; - case DP_AUX_D: - aux_ch = AUX_CH_D; - break; - case DP_AUX_E: - aux_ch = AUX_CH_E; - break; - case DP_AUX_F: - aux_ch = AUX_CH_F; - break; - default: - MISSING_CASE(info->alternate_aux_channel); - aux_ch = AUX_CH_A; - break; - } - - DRM_DEBUG_KMS("using AUX %c for port %c (VBT)\n", - aux_ch_name(aux_ch), port_name(port)); - - return aux_ch; -} - -static enum intel_display_power_domain -intel_aux_power_domain(struct intel_dp *intel_dp) -{ - switch (intel_dp->aux_ch) { - case AUX_CH_A: - return POWER_DOMAIN_AUX_A; - case AUX_CH_B: - return POWER_DOMAIN_AUX_B; - case AUX_CH_C: - return POWER_DOMAIN_AUX_C; - case AUX_CH_D: - return POWER_DOMAIN_AUX_D; - case AUX_CH_E: - return POWER_DOMAIN_AUX_E; - case AUX_CH_F: - return POWER_DOMAIN_AUX_F; - default: - MISSING_CASE(intel_dp->aux_ch); - return POWER_DOMAIN_AUX_A; - } -} static i915_reg_t g4x_aux_ctl_reg(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - enum aux_ch aux_ch = intel_dp->aux_ch; + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + enum aux_ch aux_ch = dig_port->aux_ch; switch (aux_ch) { case AUX_CH_B: @@ -1592,7 +1462,8 @@ static i915_reg_t g4x_aux_ctl_reg(struct intel_dp *intel_dp) static i915_reg_t g4x_aux_data_reg(struct intel_dp *intel_dp, int index) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - enum aux_ch aux_ch = intel_dp->aux_ch; + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + enum aux_ch aux_ch = dig_port->aux_ch; switch (aux_ch) { case AUX_CH_B: @@ -1608,7 +1479,8 @@ static i915_reg_t g4x_aux_data_reg(struct intel_dp *intel_dp, int index) static i915_reg_t ilk_aux_ctl_reg(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - enum aux_ch aux_ch = intel_dp->aux_ch; + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + enum aux_ch aux_ch = dig_port->aux_ch; switch (aux_ch) { case AUX_CH_A: @@ -1626,7 +1498,8 @@ static i915_reg_t ilk_aux_ctl_reg(struct intel_dp *intel_dp) static i915_reg_t ilk_aux_data_reg(struct intel_dp *intel_dp, int index) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - enum aux_ch aux_ch = intel_dp->aux_ch; + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + enum aux_ch aux_ch = dig_port->aux_ch; switch (aux_ch) { case AUX_CH_A: @@ -1644,7 +1517,8 @@ static i915_reg_t ilk_aux_data_reg(struct intel_dp *intel_dp, int index) static i915_reg_t skl_aux_ctl_reg(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - enum aux_ch aux_ch = intel_dp->aux_ch; + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + enum aux_ch aux_ch = dig_port->aux_ch; switch (aux_ch) { case AUX_CH_A: @@ -1663,7 +1537,8 @@ static i915_reg_t skl_aux_ctl_reg(struct intel_dp *intel_dp) static i915_reg_t skl_aux_data_reg(struct intel_dp *intel_dp, int index) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - enum aux_ch aux_ch = intel_dp->aux_ch; + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + enum aux_ch aux_ch = dig_port->aux_ch; switch (aux_ch) { case AUX_CH_A: @@ -1689,10 +1564,8 @@ static void intel_dp_aux_init(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; - - intel_dp->aux_ch = intel_aux_ch(intel_dp); - intel_dp->aux_power_domain = intel_aux_power_domain(intel_dp); + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + struct intel_encoder *encoder = &dig_port->base; if (INTEL_GEN(dev_priv) >= 9) { intel_dp->aux_ch_ctl_reg = skl_aux_ctl_reg; @@ -1834,7 +1707,7 @@ int intel_dp_rate_select(struct intel_dp *intel_dp, int rate) } void intel_dp_compute_rate(struct intel_dp *intel_dp, int port_clock, - uint8_t *link_bw, uint8_t *rate_select) + u8 *link_bw, u8 *rate_select) { /* eDP 1.4 rate select method. */ if (intel_dp->use_rate_select) { @@ -1853,6 +1726,41 @@ struct link_config_limits { int min_bpp, max_bpp; }; +static bool intel_dp_source_supports_fec(struct intel_dp *intel_dp, + const struct intel_crtc_state *pipe_config) +{ + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + + return INTEL_GEN(dev_priv) >= 11 && + pipe_config->cpu_transcoder != TRANSCODER_A; +} + +static bool intel_dp_supports_fec(struct intel_dp *intel_dp, + const struct intel_crtc_state *pipe_config) +{ + return intel_dp_source_supports_fec(intel_dp, pipe_config) && + drm_dp_sink_supports_fec(intel_dp->fec_capable); +} + +static bool intel_dp_source_supports_dsc(struct intel_dp *intel_dp, + const struct intel_crtc_state *pipe_config) +{ + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + + return INTEL_GEN(dev_priv) >= 10 && + pipe_config->cpu_transcoder != TRANSCODER_A; +} + +static bool intel_dp_supports_dsc(struct intel_dp *intel_dp, + const struct intel_crtc_state *pipe_config) +{ + if (!intel_dp_is_edp(intel_dp) && !pipe_config->fec_enable) + return false; + + return intel_dp_source_supports_dsc(intel_dp, pipe_config) && + drm_dp_sink_supports_dsc(intel_dp->dsc_dpcd); +} + static int intel_dp_compute_bpp(struct intel_dp *intel_dp, struct intel_crtc_state *pipe_config) { @@ -1916,7 +1824,7 @@ intel_dp_adjust_compliance_config(struct intel_dp *intel_dp, } /* Optimize link config in order: max bpp, min clock, min lanes */ -static bool +static int intel_dp_compute_link_config_wide(struct intel_dp *intel_dp, struct intel_crtc_state *pipe_config, const struct link_config_limits *limits) @@ -1942,23 +1850,138 @@ intel_dp_compute_link_config_wide(struct intel_dp *intel_dp, pipe_config->pipe_bpp = bpp; pipe_config->port_clock = link_clock; - return true; + return 0; } } } } - return false; + return -EINVAL; } -static bool +static int intel_dp_dsc_compute_bpp(struct intel_dp *intel_dp, u8 dsc_max_bpc) +{ + int i, num_bpc; + u8 dsc_bpc[3] = {0}; + + num_bpc = drm_dp_dsc_sink_supported_input_bpcs(intel_dp->dsc_dpcd, + dsc_bpc); + for (i = 0; i < num_bpc; i++) { + if (dsc_max_bpc >= dsc_bpc[i]) + return dsc_bpc[i] * 3; + } + + return 0; +} + +static int intel_dp_dsc_compute_config(struct intel_dp *intel_dp, + struct intel_crtc_state *pipe_config, + struct drm_connector_state *conn_state, + struct link_config_limits *limits) +{ + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); + struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; + u8 dsc_max_bpc; + int pipe_bpp; + int ret; + + pipe_config->fec_enable = !intel_dp_is_edp(intel_dp) && + intel_dp_supports_fec(intel_dp, pipe_config); + + if (!intel_dp_supports_dsc(intel_dp, pipe_config)) + return -EINVAL; + + dsc_max_bpc = min_t(u8, DP_DSC_MAX_SUPPORTED_BPC, + conn_state->max_requested_bpc); + + pipe_bpp = intel_dp_dsc_compute_bpp(intel_dp, dsc_max_bpc); + if (pipe_bpp < DP_DSC_MIN_SUPPORTED_BPC * 3) { + DRM_DEBUG_KMS("No DSC support for less than 8bpc\n"); + return -EINVAL; + } + + /* + * For now enable DSC for max bpp, max link rate, max lane count. + * Optimize this later for the minimum possible link rate/lane count + * with DSC enabled for the requested mode. + */ + pipe_config->pipe_bpp = pipe_bpp; + pipe_config->port_clock = intel_dp->common_rates[limits->max_clock]; + pipe_config->lane_count = limits->max_lane_count; + + if (intel_dp_is_edp(intel_dp)) { + pipe_config->dsc_params.compressed_bpp = + min_t(u16, drm_edp_dsc_sink_output_bpp(intel_dp->dsc_dpcd) >> 4, + pipe_config->pipe_bpp); + pipe_config->dsc_params.slice_count = + drm_dp_dsc_sink_max_slice_count(intel_dp->dsc_dpcd, + true); + } else { + u16 dsc_max_output_bpp; + u8 dsc_dp_slice_count; + + dsc_max_output_bpp = + intel_dp_dsc_get_output_bpp(pipe_config->port_clock, + pipe_config->lane_count, + adjusted_mode->crtc_clock, + adjusted_mode->crtc_hdisplay); + dsc_dp_slice_count = + intel_dp_dsc_get_slice_count(intel_dp, + adjusted_mode->crtc_clock, + adjusted_mode->crtc_hdisplay); + if (!dsc_max_output_bpp || !dsc_dp_slice_count) { + DRM_DEBUG_KMS("Compressed BPP/Slice Count not supported\n"); + return -EINVAL; + } + pipe_config->dsc_params.compressed_bpp = min_t(u16, + dsc_max_output_bpp >> 4, + pipe_config->pipe_bpp); + pipe_config->dsc_params.slice_count = dsc_dp_slice_count; + } + /* + * VDSC engine operates at 1 Pixel per clock, so if peak pixel rate + * is greater than the maximum Cdclock and if slice count is even + * then we need to use 2 VDSC instances. + */ + if (adjusted_mode->crtc_clock > dev_priv->max_cdclk_freq) { + if (pipe_config->dsc_params.slice_count > 1) { + pipe_config->dsc_params.dsc_split = true; + } else { + DRM_DEBUG_KMS("Cannot split stream to use 2 VDSC instances\n"); + return -EINVAL; + } + } + + ret = intel_dp_compute_dsc_params(intel_dp, pipe_config); + if (ret < 0) { + DRM_DEBUG_KMS("Cannot compute valid DSC parameters for Input Bpp = %d " + "Compressed BPP = %d\n", + pipe_config->pipe_bpp, + pipe_config->dsc_params.compressed_bpp); + return ret; + } + + pipe_config->dsc_params.compression_enable = true; + DRM_DEBUG_KMS("DP DSC computed with Input Bpp = %d " + "Compressed Bpp = %d Slice Count = %d\n", + pipe_config->pipe_bpp, + pipe_config->dsc_params.compressed_bpp, + pipe_config->dsc_params.slice_count); + + return 0; +} + +static int intel_dp_compute_link_config(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config) + struct intel_crtc_state *pipe_config, + struct drm_connector_state *conn_state) { struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); struct link_config_limits limits; int common_len; + int ret; common_len = intel_dp_common_len_rate_limit(intel_dp, intel_dp->max_link_rate); @@ -1999,23 +2022,43 @@ intel_dp_compute_link_config(struct intel_encoder *encoder, * Optimize for slow and wide. This is the place to add alternative * optimization policy. */ - if (!intel_dp_compute_link_config_wide(intel_dp, pipe_config, &limits)) - return false; + ret = intel_dp_compute_link_config_wide(intel_dp, pipe_config, &limits); + + /* enable compression if the mode doesn't fit available BW */ + DRM_DEBUG_KMS("Force DSC en = %d\n", intel_dp->force_dsc_en); + if (ret || intel_dp->force_dsc_en) { + ret = intel_dp_dsc_compute_config(intel_dp, pipe_config, + conn_state, &limits); + if (ret < 0) + return ret; + } - DRM_DEBUG_KMS("DP lane count %d clock %d bpp %d\n", - pipe_config->lane_count, pipe_config->port_clock, - pipe_config->pipe_bpp); + if (pipe_config->dsc_params.compression_enable) { + DRM_DEBUG_KMS("DP lane count %d clock %d Input bpp %d Compressed bpp %d\n", + pipe_config->lane_count, pipe_config->port_clock, + pipe_config->pipe_bpp, + pipe_config->dsc_params.compressed_bpp); - DRM_DEBUG_KMS("DP link rate required %i available %i\n", - intel_dp_link_required(adjusted_mode->crtc_clock, - pipe_config->pipe_bpp), - intel_dp_max_data_rate(pipe_config->port_clock, - pipe_config->lane_count)); + DRM_DEBUG_KMS("DP link rate required %i available %i\n", + intel_dp_link_required(adjusted_mode->crtc_clock, + pipe_config->dsc_params.compressed_bpp), + intel_dp_max_data_rate(pipe_config->port_clock, + pipe_config->lane_count)); + } else { + DRM_DEBUG_KMS("DP lane count %d clock %d bpp %d\n", + pipe_config->lane_count, pipe_config->port_clock, + pipe_config->pipe_bpp); - return true; + DRM_DEBUG_KMS("DP link rate required %i available %i\n", + intel_dp_link_required(adjusted_mode->crtc_clock, + pipe_config->pipe_bpp), + intel_dp_max_data_rate(pipe_config->port_clock, + pipe_config->lane_count)); + } + return 0; } -bool +int intel_dp_compute_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config, struct drm_connector_state *conn_state) @@ -2023,6 +2066,7 @@ intel_dp_compute_config(struct intel_encoder *encoder, struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); + struct intel_lspcon *lspcon = enc_to_intel_lspcon(&encoder->base); enum port port = encoder->port; struct intel_crtc *intel_crtc = to_intel_crtc(pipe_config->base.crtc); struct intel_connector *intel_connector = intel_dp->attached_connector; @@ -2030,10 +2074,15 @@ intel_dp_compute_config(struct intel_encoder *encoder, to_intel_digital_connector_state(conn_state); bool constant_n = drm_dp_has_quirk(&intel_dp->desc, DP_DPCD_QUIRK_CONSTANT_N); + int ret; if (HAS_PCH_SPLIT(dev_priv) && !HAS_DDI(dev_priv) && port != PORT_A) pipe_config->has_pch_encoder = true; + pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; + if (lspcon->active) + lspcon_ycbcr420_config(&intel_connector->base, pipe_config); + pipe_config->has_drrs = false; if (IS_G4X(dev_priv) || port == PORT_A) pipe_config->has_audio = false; @@ -2047,14 +2096,12 @@ intel_dp_compute_config(struct intel_encoder *encoder, adjusted_mode); if (INTEL_GEN(dev_priv) >= 9) { - int ret; - ret = skl_update_scaler_crtc(pipe_config); if (ret) return ret; } - if (HAS_GMCH_DISPLAY(dev_priv)) + if (HAS_GMCH(dev_priv)) intel_gmch_panel_fitting(intel_crtc, pipe_config, conn_state->scaling_mode); else @@ -2063,17 +2110,18 @@ intel_dp_compute_config(struct intel_encoder *encoder, } if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) - return false; + return -EINVAL; - if (HAS_GMCH_DISPLAY(dev_priv) && + if (HAS_GMCH(dev_priv) && adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) - return false; + return -EINVAL; if (adjusted_mode->flags & DRM_MODE_FLAG_DBLCLK) - return false; + return -EINVAL; - if (!intel_dp_compute_link_config(encoder, pipe_config)) - return false; + ret = intel_dp_compute_link_config(encoder, pipe_config, conn_state); + if (ret < 0) + return ret; if (intel_conn_state->broadcast_rgb == INTEL_BROADCAST_RGB_AUTO) { /* @@ -2090,11 +2138,20 @@ intel_dp_compute_config(struct intel_encoder *encoder, intel_conn_state->broadcast_rgb == INTEL_BROADCAST_RGB_LIMITED; } - intel_link_compute_m_n(pipe_config->pipe_bpp, pipe_config->lane_count, - adjusted_mode->crtc_clock, - pipe_config->port_clock, - &pipe_config->dp_m_n, - constant_n); + if (!pipe_config->dsc_params.compression_enable) + intel_link_compute_m_n(pipe_config->pipe_bpp, + pipe_config->lane_count, + adjusted_mode->crtc_clock, + pipe_config->port_clock, + &pipe_config->dp_m_n, + constant_n); + else + intel_link_compute_m_n(pipe_config->dsc_params.compressed_bpp, + pipe_config->lane_count, + adjusted_mode->crtc_clock, + pipe_config->port_clock, + &pipe_config->dp_m_n, + constant_n); if (intel_connector->panel.downclock_mode != NULL && dev_priv->drrs.type == SEAMLESS_DRRS_SUPPORT) { @@ -2112,11 +2169,11 @@ intel_dp_compute_config(struct intel_encoder *encoder, intel_psr_compute_config(intel_dp, pipe_config); - return true; + return 0; } void intel_dp_set_link_params(struct intel_dp *intel_dp, - int link_rate, uint8_t lane_count, + int link_rate, u8 lane_count, bool link_mst) { intel_dp->link_trained = false; @@ -2338,7 +2395,8 @@ static bool edp_panel_vdd_on(struct intel_dp *intel_dp) if (edp_have_panel_vdd(intel_dp)) return need_to_disable; - intel_display_power_get(dev_priv, intel_dp->aux_power_domain); + intel_display_power_get(dev_priv, + intel_aux_power_domain(intel_dig_port)); DRM_DEBUG_KMS("Turning eDP port %c VDD on\n", port_name(intel_dig_port->base.port)); @@ -2377,15 +2435,15 @@ static bool edp_panel_vdd_on(struct intel_dp *intel_dp) */ void intel_edp_panel_vdd_on(struct intel_dp *intel_dp) { + intel_wakeref_t wakeref; bool vdd; if (!intel_dp_is_edp(intel_dp)) return; - pps_lock(intel_dp); - vdd = edp_panel_vdd_on(intel_dp); - pps_unlock(intel_dp); - + vdd = false; + with_pps_lock(intel_dp, wakeref) + vdd = edp_panel_vdd_on(intel_dp); I915_STATE_WARN(!vdd, "eDP port %c VDD already requested on\n", port_name(dp_to_dig_port(intel_dp)->base.port)); } @@ -2424,18 +2482,21 @@ static void edp_panel_vdd_off_sync(struct intel_dp *intel_dp) if ((pp & PANEL_POWER_ON) == 0) intel_dp->panel_power_off_time = ktime_get_boottime(); - intel_display_power_put(dev_priv, intel_dp->aux_power_domain); + intel_display_power_put_unchecked(dev_priv, + intel_aux_power_domain(intel_dig_port)); } static void edp_panel_vdd_work(struct work_struct *__work) { - struct intel_dp *intel_dp = container_of(to_delayed_work(__work), - struct intel_dp, panel_vdd_work); + struct intel_dp *intel_dp = + container_of(to_delayed_work(__work), + struct intel_dp, panel_vdd_work); + intel_wakeref_t wakeref; - pps_lock(intel_dp); - if (!intel_dp->want_panel_vdd) - edp_panel_vdd_off_sync(intel_dp); - pps_unlock(intel_dp); + with_pps_lock(intel_dp, wakeref) { + if (!intel_dp->want_panel_vdd) + edp_panel_vdd_off_sync(intel_dp); + } } static void edp_panel_vdd_schedule_off(struct intel_dp *intel_dp) @@ -2499,7 +2560,7 @@ static void edp_panel_on(struct intel_dp *intel_dp) pp_ctrl_reg = _pp_ctrl_reg(intel_dp); pp = ironlake_get_pp_control(intel_dp); - if (IS_GEN5(dev_priv)) { + if (IS_GEN(dev_priv, 5)) { /* ILK workaround: disable reset around power sequence */ pp &= ~PANEL_POWER_RESET; I915_WRITE(pp_ctrl_reg, pp); @@ -2507,7 +2568,7 @@ static void edp_panel_on(struct intel_dp *intel_dp) } pp |= PANEL_POWER_ON; - if (!IS_GEN5(dev_priv)) + if (!IS_GEN(dev_priv, 5)) pp |= PANEL_POWER_RESET; I915_WRITE(pp_ctrl_reg, pp); @@ -2516,7 +2577,7 @@ static void edp_panel_on(struct intel_dp *intel_dp) wait_panel_on(intel_dp); intel_dp->last_power_on = jiffies; - if (IS_GEN5(dev_priv)) { + if (IS_GEN(dev_priv, 5)) { pp |= PANEL_POWER_RESET; /* restore panel reset bit */ I915_WRITE(pp_ctrl_reg, pp); POSTING_READ(pp_ctrl_reg); @@ -2525,18 +2586,20 @@ static void edp_panel_on(struct intel_dp *intel_dp) void intel_edp_panel_on(struct intel_dp *intel_dp) { + intel_wakeref_t wakeref; + if (!intel_dp_is_edp(intel_dp)) return; - pps_lock(intel_dp); - edp_panel_on(intel_dp); - pps_unlock(intel_dp); + with_pps_lock(intel_dp, wakeref) + edp_panel_on(intel_dp); } static void edp_panel_off(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); u32 pp; i915_reg_t pp_ctrl_reg; @@ -2546,10 +2609,10 @@ static void edp_panel_off(struct intel_dp *intel_dp) return; DRM_DEBUG_KMS("Turn eDP port %c panel power off\n", - port_name(dp_to_dig_port(intel_dp)->base.port)); + port_name(dig_port->base.port)); WARN(!intel_dp->want_panel_vdd, "Need eDP port %c VDD to turn off panel\n", - port_name(dp_to_dig_port(intel_dp)->base.port)); + port_name(dig_port->base.port)); pp = ironlake_get_pp_control(intel_dp); /* We need to switch off panel power _and_ force vdd, for otherwise some @@ -2568,25 +2631,25 @@ static void edp_panel_off(struct intel_dp *intel_dp) intel_dp->panel_power_off_time = ktime_get_boottime(); /* We got a reference when we enabled the VDD. */ - intel_display_power_put(dev_priv, intel_dp->aux_power_domain); + intel_display_power_put_unchecked(dev_priv, intel_aux_power_domain(dig_port)); } void intel_edp_panel_off(struct intel_dp *intel_dp) { + intel_wakeref_t wakeref; + if (!intel_dp_is_edp(intel_dp)) return; - pps_lock(intel_dp); - edp_panel_off(intel_dp); - pps_unlock(intel_dp); + with_pps_lock(intel_dp, wakeref) + edp_panel_off(intel_dp); } /* Enable backlight in the panel power control. */ static void _intel_edp_backlight_on(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - u32 pp; - i915_reg_t pp_ctrl_reg; + intel_wakeref_t wakeref; /* * If we enable the backlight right away following a panel power @@ -2596,17 +2659,16 @@ static void _intel_edp_backlight_on(struct intel_dp *intel_dp) */ wait_backlight_on(intel_dp); - pps_lock(intel_dp); - - pp = ironlake_get_pp_control(intel_dp); - pp |= EDP_BLC_ENABLE; - - pp_ctrl_reg = _pp_ctrl_reg(intel_dp); + with_pps_lock(intel_dp, wakeref) { + i915_reg_t pp_ctrl_reg = _pp_ctrl_reg(intel_dp); + u32 pp; - I915_WRITE(pp_ctrl_reg, pp); - POSTING_READ(pp_ctrl_reg); + pp = ironlake_get_pp_control(intel_dp); + pp |= EDP_BLC_ENABLE; - pps_unlock(intel_dp); + I915_WRITE(pp_ctrl_reg, pp); + POSTING_READ(pp_ctrl_reg); + } } /* Enable backlight PWM and backlight PP control. */ @@ -2628,23 +2690,21 @@ void intel_edp_backlight_on(const struct intel_crtc_state *crtc_state, static void _intel_edp_backlight_off(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - u32 pp; - i915_reg_t pp_ctrl_reg; + intel_wakeref_t wakeref; if (!intel_dp_is_edp(intel_dp)) return; - pps_lock(intel_dp); - - pp = ironlake_get_pp_control(intel_dp); - pp &= ~EDP_BLC_ENABLE; - - pp_ctrl_reg = _pp_ctrl_reg(intel_dp); + with_pps_lock(intel_dp, wakeref) { + i915_reg_t pp_ctrl_reg = _pp_ctrl_reg(intel_dp); + u32 pp; - I915_WRITE(pp_ctrl_reg, pp); - POSTING_READ(pp_ctrl_reg); + pp = ironlake_get_pp_control(intel_dp); + pp &= ~EDP_BLC_ENABLE; - pps_unlock(intel_dp); + I915_WRITE(pp_ctrl_reg, pp); + POSTING_READ(pp_ctrl_reg); + } intel_dp->last_backlight_off = jiffies; edp_wait_backlight_off(intel_dp); @@ -2672,12 +2732,12 @@ static void intel_edp_backlight_power(struct intel_connector *connector, bool enable) { struct intel_dp *intel_dp = intel_attached_dp(&connector->base); + intel_wakeref_t wakeref; bool is_enabled; - pps_lock(intel_dp); - is_enabled = ironlake_get_pp_control(intel_dp) & EDP_BLC_ENABLE; - pps_unlock(intel_dp); - + is_enabled = false; + with_pps_lock(intel_dp, wakeref) + is_enabled = ironlake_get_pp_control(intel_dp) & EDP_BLC_ENABLE; if (is_enabled == enable) return; @@ -2744,7 +2804,7 @@ static void ironlake_edp_pll_on(struct intel_dp *intel_dp, * 1. Wait for the start of vertical blank on the enabled pipe going to FDI * 2. Program DP PLL enable */ - if (IS_GEN5(dev_priv)) + if (IS_GEN(dev_priv, 5)) intel_wait_for_vblank_if_active(dev_priv, !crtc->pipe); intel_dp->DP |= DP_PLL_ENABLE; @@ -2788,6 +2848,22 @@ static bool downstream_hpd_needs_d0(struct intel_dp *intel_dp) intel_dp->downstream_ports[0] & DP_DS_PORT_HPD; } +void intel_dp_sink_set_decompression_state(struct intel_dp *intel_dp, + const struct intel_crtc_state *crtc_state, + bool enable) +{ + int ret; + + if (!crtc_state->dsc_params.compression_enable) + return; + + ret = drm_dp_dpcd_writeb(&intel_dp->aux, DP_DSC_ENABLE, + enable ? DP_DECOMPRESSION_EN : 0); + if (ret < 0) + DRM_DEBUG_KMS("Failed to %s sink decompression state\n", + enable ? "enable" : "disable"); +} + /* If the sink supports it, try to set the power state appropriately */ void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode) { @@ -2878,16 +2954,18 @@ static bool intel_dp_get_hw_state(struct intel_encoder *encoder, { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); + intel_wakeref_t wakeref; bool ret; - if (!intel_display_power_get_if_enabled(dev_priv, - encoder->power_domain)) + wakeref = intel_display_power_get_if_enabled(dev_priv, + encoder->power_domain); + if (!wakeref) return false; ret = intel_dp_port_enabled(dev_priv, intel_dp->output_reg, encoder->port, pipe); - intel_display_power_put(dev_priv, encoder->power_domain); + intel_display_power_put(dev_priv, encoder->power_domain, wakeref); return ret; } @@ -3055,20 +3133,20 @@ static void chv_post_disable_dp(struct intel_encoder *encoder, static void _intel_dp_set_link_train(struct intel_dp *intel_dp, - uint32_t *DP, - uint8_t dp_train_pat) + u32 *DP, + u8 dp_train_pat) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); enum port port = intel_dig_port->base.port; - uint8_t train_pat_mask = drm_dp_training_pattern_mask(intel_dp->dpcd); + u8 train_pat_mask = drm_dp_training_pattern_mask(intel_dp->dpcd); if (dp_train_pat & train_pat_mask) DRM_DEBUG_KMS("Using DP training pattern TPS%d\n", dp_train_pat & train_pat_mask); if (HAS_DDI(dev_priv)) { - uint32_t temp = I915_READ(DP_TP_CTL(port)); + u32 temp = I915_READ(DP_TP_CTL(port)); if (dp_train_pat & DP_LINK_SCRAMBLING_DISABLE) temp |= DP_TP_CTL_SCRAMBLE_DISABLE; @@ -3167,24 +3245,23 @@ static void intel_enable_dp(struct intel_encoder *encoder, struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc); - uint32_t dp_reg = I915_READ(intel_dp->output_reg); + u32 dp_reg = I915_READ(intel_dp->output_reg); enum pipe pipe = crtc->pipe; + intel_wakeref_t wakeref; if (WARN_ON(dp_reg & DP_PORT_EN)) return; - pps_lock(intel_dp); - - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) - vlv_init_panel_power_sequencer(encoder, pipe_config); - - intel_dp_enable_port(intel_dp, pipe_config); + with_pps_lock(intel_dp, wakeref) { + if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) + vlv_init_panel_power_sequencer(encoder, pipe_config); - edp_panel_vdd_on(intel_dp); - edp_panel_on(intel_dp); - edp_panel_vdd_off(intel_dp, true); + intel_dp_enable_port(intel_dp, pipe_config); - pps_unlock(intel_dp); + edp_panel_vdd_on(intel_dp); + edp_panel_on(intel_dp); + edp_panel_vdd_off(intel_dp, true); + } if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { unsigned int lane_mask = 0x0; @@ -3387,14 +3464,14 @@ static void chv_dp_post_pll_disable(struct intel_encoder *encoder, * link status information */ bool -intel_dp_get_link_status(struct intel_dp *intel_dp, uint8_t link_status[DP_LINK_STATUS_SIZE]) +intel_dp_get_link_status(struct intel_dp *intel_dp, u8 link_status[DP_LINK_STATUS_SIZE]) { return drm_dp_dpcd_read(&intel_dp->aux, DP_LANE0_1_STATUS, link_status, DP_LINK_STATUS_SIZE) == DP_LINK_STATUS_SIZE; } /* These are source-specific values. */ -uint8_t +u8 intel_dp_voltage_max(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); @@ -3413,8 +3490,8 @@ intel_dp_voltage_max(struct intel_dp *intel_dp) return DP_TRAIN_VOLTAGE_SWING_LEVEL_2; } -uint8_t -intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing) +u8 +intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, u8 voltage_swing) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; @@ -3459,12 +3536,12 @@ intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing) } } -static uint32_t vlv_signal_levels(struct intel_dp *intel_dp) +static u32 vlv_signal_levels(struct intel_dp *intel_dp) { struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; unsigned long demph_reg_value, preemph_reg_value, uniqtranscale_reg_value; - uint8_t train_set = intel_dp->train_set[0]; + u8 train_set = intel_dp->train_set[0]; switch (train_set & DP_TRAIN_PRE_EMPHASIS_MASK) { case DP_TRAIN_PRE_EMPH_LEVEL_0: @@ -3545,12 +3622,12 @@ static uint32_t vlv_signal_levels(struct intel_dp *intel_dp) return 0; } -static uint32_t chv_signal_levels(struct intel_dp *intel_dp) +static u32 chv_signal_levels(struct intel_dp *intel_dp) { struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; u32 deemph_reg_value, margin_reg_value; bool uniq_trans_scale = false; - uint8_t train_set = intel_dp->train_set[0]; + u8 train_set = intel_dp->train_set[0]; switch (train_set & DP_TRAIN_PRE_EMPHASIS_MASK) { case DP_TRAIN_PRE_EMPH_LEVEL_0: @@ -3628,10 +3705,10 @@ static uint32_t chv_signal_levels(struct intel_dp *intel_dp) return 0; } -static uint32_t -g4x_signal_levels(uint8_t train_set) +static u32 +g4x_signal_levels(u8 train_set) { - uint32_t signal_levels = 0; + u32 signal_levels = 0; switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) { case DP_TRAIN_VOLTAGE_SWING_LEVEL_0: @@ -3667,8 +3744,8 @@ g4x_signal_levels(uint8_t train_set) } /* SNB CPU eDP voltage swing and pre-emphasis control */ -static uint32_t -snb_cpu_edp_signal_levels(uint8_t train_set) +static u32 +snb_cpu_edp_signal_levels(u8 train_set) { int signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK | DP_TRAIN_PRE_EMPHASIS_MASK); @@ -3695,8 +3772,8 @@ snb_cpu_edp_signal_levels(uint8_t train_set) } /* IVB CPU eDP voltage swing and pre-emphasis control */ -static uint32_t -ivb_cpu_edp_signal_levels(uint8_t train_set) +static u32 +ivb_cpu_edp_signal_levels(u8 train_set) { int signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK | DP_TRAIN_PRE_EMPHASIS_MASK); @@ -3731,8 +3808,8 @@ intel_dp_set_signal_levels(struct intel_dp *intel_dp) struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); enum port port = intel_dig_port->base.port; - uint32_t signal_levels, mask = 0; - uint8_t train_set = intel_dp->train_set[0]; + u32 signal_levels, mask = 0; + u8 train_set = intel_dp->train_set[0]; if (IS_GEN9_LP(dev_priv) || INTEL_GEN(dev_priv) >= 10) { signal_levels = bxt_signal_levels(intel_dp); @@ -3746,7 +3823,7 @@ intel_dp_set_signal_levels(struct intel_dp *intel_dp) } else if (IS_IVYBRIDGE(dev_priv) && port == PORT_A) { signal_levels = ivb_cpu_edp_signal_levels(train_set); mask = EDP_LINK_TRAIN_VOL_EMP_MASK_IVB; - } else if (IS_GEN6(dev_priv) && port == PORT_A) { + } else if (IS_GEN(dev_priv, 6) && port == PORT_A) { signal_levels = snb_cpu_edp_signal_levels(train_set); mask = EDP_LINK_TRAIN_VOL_EMP_MASK_SNB; } else { @@ -3771,7 +3848,7 @@ intel_dp_set_signal_levels(struct intel_dp *intel_dp) void intel_dp_program_link_training_pattern(struct intel_dp *intel_dp, - uint8_t dp_train_pat) + u8 dp_train_pat) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_i915_private *dev_priv = @@ -3788,7 +3865,7 @@ void intel_dp_set_idle_link_train(struct intel_dp *intel_dp) struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); enum port port = intel_dig_port->base.port; - uint32_t val; + u32 val; if (!HAS_DDI(dev_priv)) return; @@ -3823,7 +3900,7 @@ intel_dp_link_down(struct intel_encoder *encoder, struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc); enum port port = encoder->port; - uint32_t DP = intel_dp->DP; + u32 DP = intel_dp->DP; if (WARN_ON(HAS_DDI(dev_priv))) return; @@ -3882,12 +3959,49 @@ intel_dp_link_down(struct intel_encoder *encoder, intel_dp->DP = DP; if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { - pps_lock(intel_dp); - intel_dp->active_pipe = INVALID_PIPE; - pps_unlock(intel_dp); + intel_wakeref_t wakeref; + + with_pps_lock(intel_dp, wakeref) + intel_dp->active_pipe = INVALID_PIPE; } } +static void +intel_dp_extended_receiver_capabilities(struct intel_dp *intel_dp) +{ + u8 dpcd_ext[6]; + + /* + * Prior to DP1.3 the bit represented by + * DP_EXTENDED_RECEIVER_CAP_FIELD_PRESENT was reserved. + * if it is set DP_DPCD_REV at 0000h could be at a value less than + * the true capability of the panel. The only way to check is to + * then compare 0000h and 2200h. + */ + if (!(intel_dp->dpcd[DP_TRAINING_AUX_RD_INTERVAL] & + DP_EXTENDED_RECEIVER_CAP_FIELD_PRESENT)) + return; + + if (drm_dp_dpcd_read(&intel_dp->aux, DP_DP13_DPCD_REV, + &dpcd_ext, sizeof(dpcd_ext)) != sizeof(dpcd_ext)) { + DRM_ERROR("DPCD failed read at extended capabilities\n"); + return; + } + + if (intel_dp->dpcd[DP_DPCD_REV] > dpcd_ext[DP_DPCD_REV]) { + DRM_DEBUG_KMS("DPCD extended DPCD rev less than base DPCD rev\n"); + return; + } + + if (!memcmp(intel_dp->dpcd, dpcd_ext, sizeof(dpcd_ext))) + return; + + DRM_DEBUG_KMS("Base DPCD: %*ph\n", + (int)sizeof(intel_dp->dpcd), intel_dp->dpcd); + + memcpy(intel_dp->dpcd, dpcd_ext, sizeof(dpcd_ext)); +} + bool intel_dp_read_dpcd(struct intel_dp *intel_dp) { @@ -3895,11 +4009,47 @@ intel_dp_read_dpcd(struct intel_dp *intel_dp) sizeof(intel_dp->dpcd)) < 0) return false; /* aux transfer failed */ + intel_dp_extended_receiver_capabilities(intel_dp); + DRM_DEBUG_KMS("DPCD: %*ph\n", (int) sizeof(intel_dp->dpcd), intel_dp->dpcd); return intel_dp->dpcd[DP_DPCD_REV] != 0; } +static void intel_dp_get_dsc_sink_cap(struct intel_dp *intel_dp) +{ + /* + * Clear the cached register set to avoid using stale values + * for the sinks that do not support DSC. + */ + memset(intel_dp->dsc_dpcd, 0, sizeof(intel_dp->dsc_dpcd)); + + /* Clear fec_capable to avoid using stale values */ + intel_dp->fec_capable = 0; + + /* Cache the DSC DPCD if eDP or DP rev >= 1.4 */ + if (intel_dp->dpcd[DP_DPCD_REV] >= 0x14 || + intel_dp->edp_dpcd[0] >= DP_EDP_14) { + if (drm_dp_dpcd_read(&intel_dp->aux, DP_DSC_SUPPORT, + intel_dp->dsc_dpcd, + sizeof(intel_dp->dsc_dpcd)) < 0) + DRM_ERROR("Failed to read DPCD register 0x%x\n", + DP_DSC_SUPPORT); + + DRM_DEBUG_KMS("DSC DPCD: %*ph\n", + (int)sizeof(intel_dp->dsc_dpcd), + intel_dp->dsc_dpcd); + + /* FEC is supported only on DP 1.4 */ + if (!intel_dp_is_edp(intel_dp) && + drm_dp_dpcd_readb(&intel_dp->aux, DP_FEC_CAPABILITY, + &intel_dp->fec_capable) < 0) + DRM_ERROR("Failed to read FEC DPCD register\n"); + + DRM_DEBUG_KMS("FEC CAPABILITY: %x\n", intel_dp->fec_capable); + } +} + static bool intel_edp_init_dpcd(struct intel_dp *intel_dp) { @@ -3976,6 +4126,10 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp) intel_dp_set_common_rates(intel_dp); + /* Read the eDP DSC DPCD registers */ + if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) + intel_dp_get_dsc_sink_cap(intel_dp); + return true; } @@ -3983,8 +4137,6 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp) static bool intel_dp_get_dpcd(struct intel_dp *intel_dp) { - u8 sink_count; - if (!intel_dp_read_dpcd(intel_dp)) return false; @@ -3994,25 +4146,35 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp) intel_dp_set_common_rates(intel_dp); } - if (drm_dp_dpcd_readb(&intel_dp->aux, DP_SINK_COUNT, &sink_count) <= 0) - return false; - /* - * Sink count can change between short pulse hpd hence - * a member variable in intel_dp will track any changes - * between short pulse interrupts. + * Some eDP panels do not set a valid value for sink count, that is why + * it don't care about read it here and in intel_edp_init_dpcd(). */ - intel_dp->sink_count = DP_GET_SINK_COUNT(sink_count); + if (!intel_dp_is_edp(intel_dp)) { + u8 count; + ssize_t r; - /* - * SINK_COUNT == 0 and DOWNSTREAM_PORT_PRESENT == 1 implies that - * a dongle is present but no display. Unless we require to know - * if a dongle is present or not, we don't need to update - * downstream port information. So, an early return here saves - * time from performing other operations which are not required. - */ - if (!intel_dp_is_edp(intel_dp) && !intel_dp->sink_count) - return false; + r = drm_dp_dpcd_readb(&intel_dp->aux, DP_SINK_COUNT, &count); + if (r < 1) + return false; + + /* + * Sink count can change between short pulse hpd hence + * a member variable in intel_dp will track any changes + * between short pulse interrupts. + */ + intel_dp->sink_count = DP_GET_SINK_COUNT(count); + + /* + * SINK_COUNT == 0 and DOWNSTREAM_PORT_PRESENT == 1 implies that + * a dongle is present but no display. Unless we require to know + * if a dongle is present or not, we don't need to update + * downstream port information. So, an early return here saves + * time from performing other operations which are not required. + */ + if (!intel_dp->sink_count) + return false; + } if (!drm_dp_is_branch(intel_dp->dpcd)) return true; /* native DP sink */ @@ -4029,16 +4191,10 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp) } static bool -intel_dp_can_mst(struct intel_dp *intel_dp) +intel_dp_sink_can_mst(struct intel_dp *intel_dp) { u8 mstm_cap; - if (!i915_modparams.enable_dp_mst) - return false; - - if (!intel_dp->can_mst) - return false; - if (intel_dp->dpcd[DP_DPCD_REV] < 0x12) return false; @@ -4048,34 +4204,36 @@ intel_dp_can_mst(struct intel_dp *intel_dp) return mstm_cap & DP_MST_CAP; } +static bool +intel_dp_can_mst(struct intel_dp *intel_dp) +{ + return i915_modparams.enable_dp_mst && + intel_dp->can_mst && + intel_dp_sink_can_mst(intel_dp); +} + static void intel_dp_configure_mst(struct intel_dp *intel_dp) { - if (!i915_modparams.enable_dp_mst) - return; + struct intel_encoder *encoder = + &dp_to_dig_port(intel_dp)->base; + bool sink_can_mst = intel_dp_sink_can_mst(intel_dp); + + DRM_DEBUG_KMS("MST support? port %c: %s, sink: %s, modparam: %s\n", + port_name(encoder->port), yesno(intel_dp->can_mst), + yesno(sink_can_mst), yesno(i915_modparams.enable_dp_mst)); if (!intel_dp->can_mst) return; - intel_dp->is_mst = intel_dp_can_mst(intel_dp); - - if (intel_dp->is_mst) - DRM_DEBUG_KMS("Sink is MST capable\n"); - else - DRM_DEBUG_KMS("Sink is not MST capable\n"); + intel_dp->is_mst = sink_can_mst && + i915_modparams.enable_dp_mst; drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr, intel_dp->is_mst); } static bool -intel_dp_get_sink_irq(struct intel_dp *intel_dp, u8 *sink_irq_vector) -{ - return drm_dp_dpcd_readb(&intel_dp->aux, DP_DEVICE_SERVICE_IRQ_VECTOR, - sink_irq_vector) == 1; -} - -static bool intel_dp_get_sink_irq_esi(struct intel_dp *intel_dp, u8 *sink_irq_vector) { return drm_dp_dpcd_read(&intel_dp->aux, DP_SINK_COUNT_ESI, @@ -4083,11 +4241,96 @@ intel_dp_get_sink_irq_esi(struct intel_dp *intel_dp, u8 *sink_irq_vector) DP_DPRX_ESI_LEN; } -static uint8_t intel_dp_autotest_link_training(struct intel_dp *intel_dp) +u16 intel_dp_dsc_get_output_bpp(int link_clock, u8 lane_count, + int mode_clock, int mode_hdisplay) +{ + u16 bits_per_pixel, max_bpp_small_joiner_ram; + int i; + + /* + * Available Link Bandwidth(Kbits/sec) = (NumberOfLanes)* + * (LinkSymbolClock)* 8 * ((100-FECOverhead)/100)*(TimeSlotsPerMTP) + * FECOverhead = 2.4%, for SST -> TimeSlotsPerMTP is 1, + * for MST -> TimeSlotsPerMTP has to be calculated + */ + bits_per_pixel = (link_clock * lane_count * 8 * + DP_DSC_FEC_OVERHEAD_FACTOR) / + mode_clock; + + /* Small Joiner Check: output bpp <= joiner RAM (bits) / Horiz. width */ + max_bpp_small_joiner_ram = DP_DSC_MAX_SMALL_JOINER_RAM_BUFFER / + mode_hdisplay; + + /* + * Greatest allowed DSC BPP = MIN (output BPP from avaialble Link BW + * check, output bpp from small joiner RAM check) + */ + bits_per_pixel = min(bits_per_pixel, max_bpp_small_joiner_ram); + + /* Error out if the max bpp is less than smallest allowed valid bpp */ + if (bits_per_pixel < valid_dsc_bpp[0]) { + DRM_DEBUG_KMS("Unsupported BPP %d\n", bits_per_pixel); + return 0; + } + + /* Find the nearest match in the array of known BPPs from VESA */ + for (i = 0; i < ARRAY_SIZE(valid_dsc_bpp) - 1; i++) { + if (bits_per_pixel < valid_dsc_bpp[i + 1]) + break; + } + bits_per_pixel = valid_dsc_bpp[i]; + + /* + * Compressed BPP in U6.4 format so multiply by 16, for Gen 11, + * fractional part is 0 + */ + return bits_per_pixel << 4; +} + +u8 intel_dp_dsc_get_slice_count(struct intel_dp *intel_dp, + int mode_clock, + int mode_hdisplay) +{ + u8 min_slice_count, i; + int max_slice_width; + + if (mode_clock <= DP_DSC_PEAK_PIXEL_RATE) + min_slice_count = DIV_ROUND_UP(mode_clock, + DP_DSC_MAX_ENC_THROUGHPUT_0); + else + min_slice_count = DIV_ROUND_UP(mode_clock, + DP_DSC_MAX_ENC_THROUGHPUT_1); + + max_slice_width = drm_dp_dsc_sink_max_slice_width(intel_dp->dsc_dpcd); + if (max_slice_width < DP_DSC_MIN_SLICE_WIDTH_VALUE) { + DRM_DEBUG_KMS("Unsupported slice width %d by DP DSC Sink device\n", + max_slice_width); + return 0; + } + /* Also take into account max slice width */ + min_slice_count = min_t(u8, min_slice_count, + DIV_ROUND_UP(mode_hdisplay, + max_slice_width)); + + /* Find the closest match to the valid slice count values */ + for (i = 0; i < ARRAY_SIZE(valid_dsc_slicecount); i++) { + if (valid_dsc_slicecount[i] > + drm_dp_dsc_sink_max_slice_count(intel_dp->dsc_dpcd, + false)) + break; + if (min_slice_count <= valid_dsc_slicecount[i]) + return valid_dsc_slicecount[i]; + } + + DRM_DEBUG_KMS("Unsupported Slice Count %d\n", min_slice_count); + return 0; +} + +static u8 intel_dp_autotest_link_training(struct intel_dp *intel_dp) { int status = 0; int test_link_rate; - uint8_t test_lane_count, test_link_bw; + u8 test_lane_count, test_link_bw; /* (DP CTS 1.2) * 4.3.1.11 */ @@ -4120,10 +4363,10 @@ static uint8_t intel_dp_autotest_link_training(struct intel_dp *intel_dp) return DP_TEST_ACK; } -static uint8_t intel_dp_autotest_video_pattern(struct intel_dp *intel_dp) +static u8 intel_dp_autotest_video_pattern(struct intel_dp *intel_dp) { - uint8_t test_pattern; - uint8_t test_misc; + u8 test_pattern; + u8 test_misc; __be16 h_width, v_height; int status = 0; @@ -4181,9 +4424,9 @@ static uint8_t intel_dp_autotest_video_pattern(struct intel_dp *intel_dp) return DP_TEST_ACK; } -static uint8_t intel_dp_autotest_edid(struct intel_dp *intel_dp) +static u8 intel_dp_autotest_edid(struct intel_dp *intel_dp) { - uint8_t test_result = DP_TEST_ACK; + u8 test_result = DP_TEST_ACK; struct intel_connector *intel_connector = intel_dp->attached_connector; struct drm_connector *connector = &intel_connector->base; @@ -4225,16 +4468,16 @@ static uint8_t intel_dp_autotest_edid(struct intel_dp *intel_dp) return test_result; } -static uint8_t intel_dp_autotest_phy_pattern(struct intel_dp *intel_dp) +static u8 intel_dp_autotest_phy_pattern(struct intel_dp *intel_dp) { - uint8_t test_result = DP_TEST_NAK; + u8 test_result = DP_TEST_NAK; return test_result; } static void intel_dp_handle_test_request(struct intel_dp *intel_dp) { - uint8_t response = DP_TEST_NAK; - uint8_t request = 0; + u8 response = DP_TEST_NAK; + u8 request = 0; int status; status = drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_REQUEST, &request); @@ -4322,12 +4565,10 @@ go_again: return ret; } else { - struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); DRM_DEBUG_KMS("failed to get ESI - device may have failed\n"); intel_dp->is_mst = false; - drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr, intel_dp->is_mst); - /* send a hotplug event */ - drm_kms_helper_hotplug_event(intel_dig_port->base.base.dev); + drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr, + intel_dp->is_mst); } } return -EINVAL; @@ -4341,6 +4582,17 @@ intel_dp_needs_link_retrain(struct intel_dp *intel_dp) if (!intel_dp->link_trained) return false; + /* + * While PSR source HW is enabled, it will control main-link sending + * frames, enabling and disabling it so trying to do a retrain will fail + * as the link would or not be on or it could mix training patterns + * and frame data at the same time causing retrain to fail. + * Also when exiting PSR, HW will retrain the link anyways fixing + * any link status error. + */ + if (intel_psr_enabled(intel_dp)) + return false; + if (!intel_dp_get_link_status(intel_dp, link_status)) return false; @@ -4403,7 +4655,7 @@ int intel_dp_retrain_link(struct intel_encoder *encoder, /* Suppress underruns caused by re-training */ intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false); - if (crtc->config->has_pch_encoder) + if (crtc_state->has_pch_encoder) intel_set_pch_fifo_underrun_reporting(dev_priv, intel_crtc_pch_transcoder(crtc), false); @@ -4414,7 +4666,7 @@ int intel_dp_retrain_link(struct intel_encoder *encoder, intel_wait_for_vblank(dev_priv, crtc->pipe); intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true); - if (crtc->config->has_pch_encoder) + if (crtc_state->has_pch_encoder) intel_set_pch_fifo_underrun_reporting(dev_priv, intel_crtc_pch_transcoder(crtc), true); @@ -4462,6 +4714,29 @@ static bool intel_dp_hotplug(struct intel_encoder *encoder, return changed; } +static void intel_dp_check_service_irq(struct intel_dp *intel_dp) +{ + u8 val; + + if (intel_dp->dpcd[DP_DPCD_REV] < 0x11) + return; + + if (drm_dp_dpcd_readb(&intel_dp->aux, + DP_DEVICE_SERVICE_IRQ_VECTOR, &val) != 1 || !val) + return; + + drm_dp_dpcd_writeb(&intel_dp->aux, DP_DEVICE_SERVICE_IRQ_VECTOR, val); + + if (val & DP_AUTOMATED_TEST_REQUEST) + intel_dp_handle_test_request(intel_dp); + + if (val & DP_CP_IRQ) + intel_hdcp_check_link(intel_dp->attached_connector); + + if (val & DP_SINK_SPECIFIC_IRQ) + DRM_DEBUG_DRIVER("Sink specific irq unhandled\n"); +} + /* * According to DP spec * 5.1.2: @@ -4479,7 +4754,6 @@ static bool intel_dp_short_pulse(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - u8 sink_irq_vector = 0; u8 old_sink_count = intel_dp->sink_count; bool ret; @@ -4502,20 +4776,7 @@ intel_dp_short_pulse(struct intel_dp *intel_dp) return false; } - /* Try to read the source of the interrupt */ - if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11 && - intel_dp_get_sink_irq(intel_dp, &sink_irq_vector) && - sink_irq_vector != 0) { - /* Clear interrupt source */ - drm_dp_dpcd_writeb(&intel_dp->aux, - DP_DEVICE_SERVICE_IRQ_VECTOR, - sink_irq_vector); - - if (sink_irq_vector & DP_AUTOMATED_TEST_REQUEST) - intel_dp_handle_test_request(intel_dp); - if (sink_irq_vector & (DP_CP_IRQ | DP_SINK_SPECIFIC_IRQ)) - DRM_DEBUG_DRIVER("CP or sink specific irq unhandled\n"); - } + intel_dp_check_service_irq(intel_dp); /* Handle CEC interrupts, if any */ drm_dp_cec_irq(&intel_dp->aux); @@ -4540,8 +4801,8 @@ static enum drm_connector_status intel_dp_detect_dpcd(struct intel_dp *intel_dp) { struct intel_lspcon *lspcon = dp_to_lspcon(intel_dp); - uint8_t *dpcd = intel_dp->dpcd; - uint8_t type; + u8 *dpcd = intel_dp->dpcd; + u8 type; if (lspcon->active) lspcon_resume(lspcon); @@ -4778,28 +5039,38 @@ static bool icl_combo_port_connected(struct drm_i915_private *dev_priv, return I915_READ(SDEISR) & SDE_DDI_HOTPLUG_ICP(port); } +static const char *tc_type_name(enum tc_port_type type) +{ + static const char * const names[] = { + [TC_PORT_UNKNOWN] = "unknown", + [TC_PORT_LEGACY] = "legacy", + [TC_PORT_TYPEC] = "typec", + [TC_PORT_TBT] = "tbt", + }; + + if (WARN_ON(type >= ARRAY_SIZE(names))) + type = TC_PORT_UNKNOWN; + + return names[type]; +} + static void icl_update_tc_port_type(struct drm_i915_private *dev_priv, struct intel_digital_port *intel_dig_port, bool is_legacy, bool is_typec, bool is_tbt) { enum port port = intel_dig_port->base.port; enum tc_port_type old_type = intel_dig_port->tc_type; - const char *type_str; WARN_ON(is_legacy + is_typec + is_tbt != 1); - if (is_legacy) { + if (is_legacy) intel_dig_port->tc_type = TC_PORT_LEGACY; - type_str = "legacy"; - } else if (is_typec) { + else if (is_typec) intel_dig_port->tc_type = TC_PORT_TYPEC; - type_str = "typec"; - } else if (is_tbt) { + else if (is_tbt) intel_dig_port->tc_type = TC_PORT_TBT; - type_str = "tbt"; - } else { + else return; - } /* Types are not supposed to be changed at runtime. */ WARN_ON(old_type != TC_PORT_UNKNOWN && @@ -4807,7 +5078,7 @@ static void icl_update_tc_port_type(struct drm_i915_private *dev_priv, if (old_type != intel_dig_port->tc_type) DRM_DEBUG_KMS("Port %c has TC type %s\n", port_name(port), - type_str); + tc_type_name(intel_dig_port->tc_type)); } /* @@ -4844,6 +5115,7 @@ static bool icl_tc_phy_connect(struct drm_i915_private *dev_priv, val = I915_READ(PORT_TX_DFLEXDPPMS); if (!(val & DP_PHY_MODE_STATUS_COMPLETED(tc_port))) { DRM_DEBUG_KMS("DP PHY for TC port %d not ready\n", tc_port); + WARN_ON(dig_port->tc_legacy_port); return false; } @@ -4864,9 +5136,7 @@ static bool icl_tc_phy_connect(struct drm_i915_private *dev_priv, if (dig_port->tc_type == TC_PORT_TYPEC && !(I915_READ(PORT_TX_DFLEXDPSP) & TC_LIVE_STATE_TC(tc_port))) { DRM_DEBUG_KMS("TC PHY %d sudden disconnect.\n", tc_port); - val = I915_READ(PORT_TX_DFLEXDPCSSS); - val &= ~DP_PHY_MODE_STATUS_NOT_SAFE(tc_port); - I915_WRITE(PORT_TX_DFLEXDPCSSS, val); + icl_tc_phy_disconnect(dev_priv, dig_port); return false; } @@ -4877,25 +5147,32 @@ static bool icl_tc_phy_connect(struct drm_i915_private *dev_priv, * See the comment at the connect function. This implements the Disconnect * Flow. */ -static void icl_tc_phy_disconnect(struct drm_i915_private *dev_priv, - struct intel_digital_port *dig_port) +void icl_tc_phy_disconnect(struct drm_i915_private *dev_priv, + struct intel_digital_port *dig_port) { enum tc_port tc_port = intel_port_to_tc(dev_priv, dig_port->base.port); - u32 val; - if (dig_port->tc_type != TC_PORT_LEGACY && - dig_port->tc_type != TC_PORT_TYPEC) + if (dig_port->tc_type == TC_PORT_UNKNOWN) return; /* - * This function may be called many times in a row without an HPD event - * in between, so try to avoid the write when we can. + * TBT disconnection flow is read the live status, what was done in + * caller. */ - val = I915_READ(PORT_TX_DFLEXDPCSSS); - if (val & DP_PHY_MODE_STATUS_NOT_SAFE(tc_port)) { + if (dig_port->tc_type == TC_PORT_TYPEC || + dig_port->tc_type == TC_PORT_LEGACY) { + u32 val; + + val = I915_READ(PORT_TX_DFLEXDPCSSS); val &= ~DP_PHY_MODE_STATUS_NOT_SAFE(tc_port); I915_WRITE(PORT_TX_DFLEXDPCSSS, val); } + + DRM_DEBUG_KMS("Port %c TC type %s disconnected\n", + port_name(dig_port->base.port), + tc_type_name(dig_port->tc_type)); + + dig_port->tc_type = TC_PORT_UNKNOWN; } /* @@ -4916,7 +5193,14 @@ static bool icl_tc_port_connected(struct drm_i915_private *dev_priv, bool is_legacy, is_typec, is_tbt; u32 dpsp; - is_legacy = I915_READ(SDEISR) & SDE_TC_HOTPLUG_ICP(tc_port); + /* + * WARN if we got a legacy port HPD, but VBT didn't mark the port as + * legacy. Treat the port as legacy from now on. + */ + if (WARN_ON(!intel_dig_port->tc_legacy_port && + I915_READ(SDEISR) & SDE_TC_HOTPLUG_ICP(tc_port))) + intel_dig_port->tc_legacy_port = true; + is_legacy = intel_dig_port->tc_legacy_port; /* * The spec says we shouldn't be using the ISR bits for detecting @@ -4928,6 +5212,7 @@ static bool icl_tc_port_connected(struct drm_i915_private *dev_priv, if (!is_legacy && !is_typec && !is_tbt) { icl_tc_phy_disconnect(dev_priv, intel_dig_port); + return false; } @@ -4945,19 +5230,14 @@ static bool icl_digital_port_connected(struct intel_encoder *encoder) struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base); - switch (encoder->hpd_pin) { - case HPD_PORT_A: - case HPD_PORT_B: + if (intel_port_is_combophy(dev_priv, encoder->port)) return icl_combo_port_connected(dev_priv, dig_port); - case HPD_PORT_C: - case HPD_PORT_D: - case HPD_PORT_E: - case HPD_PORT_F: + else if (intel_port_is_tc(dev_priv, encoder->port)) return icl_tc_port_connected(dev_priv, dig_port); - default: + else MISSING_CASE(encoder->hpd_pin); - return false; - } + + return false; } /* @@ -4975,27 +5255,30 @@ bool intel_digital_port_connected(struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - if (HAS_GMCH_DISPLAY(dev_priv)) { + if (HAS_GMCH(dev_priv)) { if (IS_GM45(dev_priv)) return gm45_digital_port_connected(encoder); else return g4x_digital_port_connected(encoder); } - if (IS_GEN5(dev_priv)) - return ilk_digital_port_connected(encoder); - else if (IS_GEN6(dev_priv)) - return snb_digital_port_connected(encoder); - else if (IS_GEN7(dev_priv)) - return ivb_digital_port_connected(encoder); - else if (IS_GEN8(dev_priv)) - return bdw_digital_port_connected(encoder); + if (INTEL_GEN(dev_priv) >= 11) + return icl_digital_port_connected(encoder); + else if (IS_GEN(dev_priv, 10) || IS_GEN9_BC(dev_priv)) + return spt_digital_port_connected(encoder); else if (IS_GEN9_LP(dev_priv)) return bxt_digital_port_connected(encoder); - else if (IS_GEN9_BC(dev_priv) || IS_GEN10(dev_priv)) - return spt_digital_port_connected(encoder); - else - return icl_digital_port_connected(encoder); + else if (IS_GEN(dev_priv, 8)) + return bdw_digital_port_connected(encoder); + else if (IS_GEN(dev_priv, 7)) + return ivb_digital_port_connected(encoder); + else if (IS_GEN(dev_priv, 6)) + return snb_digital_port_connected(encoder); + else if (IS_GEN(dev_priv, 5)) + return ilk_digital_port_connected(encoder); + + MISSING_CASE(INTEL_GEN(dev_priv)); + return false; } static struct edid * @@ -5042,28 +5325,36 @@ intel_dp_unset_edid(struct intel_dp *intel_dp) } static int -intel_dp_long_pulse(struct intel_connector *connector, - struct drm_modeset_acquire_ctx *ctx) +intel_dp_detect(struct drm_connector *connector, + struct drm_modeset_acquire_ctx *ctx, + bool force) { - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_dp *intel_dp = intel_attached_dp(&connector->base); + struct drm_i915_private *dev_priv = to_i915(connector->dev); + struct intel_dp *intel_dp = intel_attached_dp(connector); + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + struct intel_encoder *encoder = &dig_port->base; enum drm_connector_status status; - u8 sink_irq_vector = 0; + enum intel_display_power_domain aux_domain = + intel_aux_power_domain(dig_port); + intel_wakeref_t wakeref; + DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", + connector->base.id, connector->name); WARN_ON(!drm_modeset_is_locked(&dev_priv->drm.mode_config.connection_mutex)); - intel_display_power_get(dev_priv, intel_dp->aux_power_domain); + wakeref = intel_display_power_get(dev_priv, aux_domain); /* Can't disconnect eDP */ if (intel_dp_is_edp(intel_dp)) status = edp_detect(intel_dp); - else if (intel_digital_port_connected(&dp_to_dig_port(intel_dp)->base)) + else if (intel_digital_port_connected(encoder)) status = intel_dp_detect_dpcd(intel_dp); else status = connector_status_disconnected; if (status == connector_status_disconnected) { memset(&intel_dp->compliance, 0, sizeof(intel_dp->compliance)); + memset(intel_dp->dsc_dpcd, 0, sizeof(intel_dp->dsc_dpcd)); if (intel_dp->is_mst) { DRM_DEBUG_KMS("MST device may have disappeared %d vs %d\n", @@ -5089,6 +5380,10 @@ intel_dp_long_pulse(struct intel_connector *connector, intel_dp_print_rates(intel_dp); + /* Read DP Sink DSC Cap DPCD regs for DP v1.4 */ + if (INTEL_GEN(dev_priv) >= 11) + intel_dp_get_dsc_sink_cap(intel_dp); + drm_dp_read_desc(&intel_dp->aux, &intel_dp->desc, drm_dp_is_branch(intel_dp->dpcd)); @@ -5109,9 +5404,13 @@ intel_dp_long_pulse(struct intel_connector *connector, * with an IRQ_HPD, so force a link status check. */ if (!intel_dp_is_edp(intel_dp)) { - struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; + int ret; - intel_dp_retrain_link(encoder, ctx); + ret = intel_dp_retrain_link(encoder, ctx); + if (ret) { + intel_display_power_put(dev_priv, aux_domain, wakeref); + return ret; + } } /* @@ -5123,61 +5422,17 @@ intel_dp_long_pulse(struct intel_connector *connector, intel_dp->aux.i2c_defer_count = 0; intel_dp_set_edid(intel_dp); - if (intel_dp_is_edp(intel_dp) || connector->detect_edid) + if (intel_dp_is_edp(intel_dp) || + to_intel_connector(connector)->detect_edid) status = connector_status_connected; - intel_dp->detect_done = true; - - /* Try to read the source of the interrupt */ - if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11 && - intel_dp_get_sink_irq(intel_dp, &sink_irq_vector) && - sink_irq_vector != 0) { - /* Clear interrupt source */ - drm_dp_dpcd_writeb(&intel_dp->aux, - DP_DEVICE_SERVICE_IRQ_VECTOR, - sink_irq_vector); - if (sink_irq_vector & DP_AUTOMATED_TEST_REQUEST) - intel_dp_handle_test_request(intel_dp); - if (sink_irq_vector & (DP_CP_IRQ | DP_SINK_SPECIFIC_IRQ)) - DRM_DEBUG_DRIVER("CP or sink specific irq unhandled\n"); - } + intel_dp_check_service_irq(intel_dp); out: if (status != connector_status_connected && !intel_dp->is_mst) intel_dp_unset_edid(intel_dp); - intel_display_power_put(dev_priv, intel_dp->aux_power_domain); - return status; -} - -static int -intel_dp_detect(struct drm_connector *connector, - struct drm_modeset_acquire_ctx *ctx, - bool force) -{ - struct intel_dp *intel_dp = intel_attached_dp(connector); - int status = connector->status; - - DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", - connector->base.id, connector->name); - - /* If full detect is not performed yet, do a full detect */ - if (!intel_dp->detect_done) { - struct drm_crtc *crtc; - int ret; - - crtc = connector->state->crtc; - if (crtc) { - ret = drm_modeset_lock(&crtc->mutex, ctx); - if (ret) - return ret; - } - - status = intel_dp_long_pulse(intel_dp->attached_connector, ctx); - } - - intel_dp->detect_done = false; - + intel_display_power_put(dev_priv, aux_domain, wakeref); return status; } @@ -5185,8 +5440,12 @@ static void intel_dp_force(struct drm_connector *connector) { struct intel_dp *intel_dp = intel_attached_dp(connector); - struct intel_encoder *intel_encoder = &dp_to_dig_port(intel_dp)->base; + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + struct intel_encoder *intel_encoder = &dig_port->base; struct drm_i915_private *dev_priv = to_i915(intel_encoder->base.dev); + enum intel_display_power_domain aux_domain = + intel_aux_power_domain(dig_port); + intel_wakeref_t wakeref; DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", connector->base.id, connector->name); @@ -5195,11 +5454,11 @@ intel_dp_force(struct drm_connector *connector) if (connector->status != connector_status_connected) return; - intel_display_power_get(dev_priv, intel_dp->aux_power_domain); + wakeref = intel_display_power_get(dev_priv, aux_domain); intel_dp_set_edid(intel_dp); - intel_display_power_put(dev_priv, intel_dp->aux_power_domain); + intel_display_power_put(dev_priv, aux_domain, wakeref); } static int intel_dp_get_modes(struct drm_connector *connector) @@ -5264,42 +5523,22 @@ intel_dp_connector_unregister(struct drm_connector *connector) intel_connector_unregister(connector); } -static void -intel_dp_connector_destroy(struct drm_connector *connector) -{ - struct intel_connector *intel_connector = to_intel_connector(connector); - - kfree(intel_connector->detect_edid); - - if (!IS_ERR_OR_NULL(intel_connector->edid)) - kfree(intel_connector->edid); - - /* - * Can't call intel_dp_is_edp() since the encoder may have been - * destroyed already. - */ - if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) - intel_panel_fini(&intel_connector->panel); - - drm_connector_cleanup(connector); - kfree(connector); -} - -void intel_dp_encoder_destroy(struct drm_encoder *encoder) +void intel_dp_encoder_flush_work(struct drm_encoder *encoder) { struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); struct intel_dp *intel_dp = &intel_dig_port->dp; intel_dp_mst_encoder_cleanup(intel_dig_port); if (intel_dp_is_edp(intel_dp)) { + intel_wakeref_t wakeref; + cancel_delayed_work_sync(&intel_dp->panel_vdd_work); /* * vdd might still be enabled do to the delayed vdd off. * Make sure vdd is actually turned off here. */ - pps_lock(intel_dp); - edp_panel_vdd_off_sync(intel_dp); - pps_unlock(intel_dp); + with_pps_lock(intel_dp, wakeref) + edp_panel_vdd_off_sync(intel_dp); if (intel_dp->edp_notifier.notifier_call) { unregister_reboot_notifier(&intel_dp->edp_notifier); @@ -5308,14 +5547,20 @@ void intel_dp_encoder_destroy(struct drm_encoder *encoder) } intel_dp_aux_fini(intel_dp); +} + +static void intel_dp_encoder_destroy(struct drm_encoder *encoder) +{ + intel_dp_encoder_flush_work(encoder); drm_encoder_cleanup(encoder); - kfree(intel_dig_port); + kfree(enc_to_dig_port(encoder)); } void intel_dp_encoder_suspend(struct intel_encoder *intel_encoder) { struct intel_dp *intel_dp = enc_to_intel_dp(&intel_encoder->base); + intel_wakeref_t wakeref; if (!intel_dp_is_edp(intel_dp)) return; @@ -5325,9 +5570,8 @@ void intel_dp_encoder_suspend(struct intel_encoder *intel_encoder) * Make sure vdd is actually turned off here. */ cancel_delayed_work_sync(&intel_dp->panel_vdd_work); - pps_lock(intel_dp); - edp_panel_vdd_off_sync(intel_dp); - pps_unlock(intel_dp); + with_pps_lock(intel_dp, wakeref) + edp_panel_vdd_off_sync(intel_dp); } static @@ -5340,7 +5584,7 @@ int intel_dp_hdcp_write_an_aksv(struct intel_digital_port *intel_dig_port, .address = DP_AUX_HDCP_AKSV, .size = DRM_HDCP_KSV_LEN, }; - uint8_t txbuf[HEADER_SIZE + DRM_HDCP_KSV_LEN] = {}, rxbuf[2], reply = 0; + u8 txbuf[HEADER_SIZE + DRM_HDCP_KSV_LEN] = {}, rxbuf[2], reply = 0; ssize_t dpcd_ret; int ret; @@ -5348,7 +5592,8 @@ int intel_dp_hdcp_write_an_aksv(struct intel_digital_port *intel_dig_port, dpcd_ret = drm_dp_dpcd_write(&intel_dig_port->dp.aux, DP_AUX_HDCP_AN, an, DRM_HDCP_AN_LEN); if (dpcd_ret != DRM_HDCP_AN_LEN) { - DRM_ERROR("Failed to write An over DP/AUX (%zd)\n", dpcd_ret); + DRM_DEBUG_KMS("Failed to write An over DP/AUX (%zd)\n", + dpcd_ret); return dpcd_ret >= 0 ? -EIO : dpcd_ret; } @@ -5364,15 +5609,20 @@ int intel_dp_hdcp_write_an_aksv(struct intel_digital_port *intel_dig_port, rxbuf, sizeof(rxbuf), DP_AUX_CH_CTL_AUX_AKSV_SELECT); if (ret < 0) { - DRM_ERROR("Write Aksv over DP/AUX failed (%d)\n", ret); + DRM_DEBUG_KMS("Write Aksv over DP/AUX failed (%d)\n", ret); return ret; } else if (ret == 0) { - DRM_ERROR("Aksv write over DP/AUX was empty\n"); + DRM_DEBUG_KMS("Aksv write over DP/AUX was empty\n"); return -EIO; } reply = (rxbuf[0] >> 4) & DP_AUX_NATIVE_REPLY_MASK; - return reply == DP_AUX_NATIVE_REPLY_ACK ? 0 : -EIO; + if (reply != DP_AUX_NATIVE_REPLY_ACK) { + DRM_DEBUG_KMS("Aksv write: no DP_AUX_NATIVE_REPLY_ACK %x\n", + reply); + return -EIO; + } + return 0; } static int intel_dp_hdcp_read_bksv(struct intel_digital_port *intel_dig_port, @@ -5382,7 +5632,7 @@ static int intel_dp_hdcp_read_bksv(struct intel_digital_port *intel_dig_port, ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, DP_AUX_HDCP_BKSV, bksv, DRM_HDCP_KSV_LEN); if (ret != DRM_HDCP_KSV_LEN) { - DRM_ERROR("Read Bksv from DP/AUX failed (%zd)\n", ret); + DRM_DEBUG_KMS("Read Bksv from DP/AUX failed (%zd)\n", ret); return ret >= 0 ? -EIO : ret; } return 0; @@ -5400,7 +5650,7 @@ static int intel_dp_hdcp_read_bstatus(struct intel_digital_port *intel_dig_port, ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, DP_AUX_HDCP_BINFO, bstatus, DRM_HDCP_BSTATUS_LEN); if (ret != DRM_HDCP_BSTATUS_LEN) { - DRM_ERROR("Read bstatus from DP/AUX failed (%zd)\n", ret); + DRM_DEBUG_KMS("Read bstatus from DP/AUX failed (%zd)\n", ret); return ret >= 0 ? -EIO : ret; } return 0; @@ -5415,7 +5665,7 @@ int intel_dp_hdcp_read_bcaps(struct intel_digital_port *intel_dig_port, ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, DP_AUX_HDCP_BCAPS, bcaps, 1); if (ret != 1) { - DRM_ERROR("Read bcaps from DP/AUX failed (%zd)\n", ret); + DRM_DEBUG_KMS("Read bcaps from DP/AUX failed (%zd)\n", ret); return ret >= 0 ? -EIO : ret; } @@ -5445,7 +5695,7 @@ int intel_dp_hdcp_read_ri_prime(struct intel_digital_port *intel_dig_port, ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, DP_AUX_HDCP_RI_PRIME, ri_prime, DRM_HDCP_RI_LEN); if (ret != DRM_HDCP_RI_LEN) { - DRM_ERROR("Read Ri' from DP/AUX failed (%zd)\n", ret); + DRM_DEBUG_KMS("Read Ri' from DP/AUX failed (%zd)\n", ret); return ret >= 0 ? -EIO : ret; } return 0; @@ -5460,7 +5710,7 @@ int intel_dp_hdcp_read_ksv_ready(struct intel_digital_port *intel_dig_port, ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, DP_AUX_HDCP_BSTATUS, &bstatus, 1); if (ret != 1) { - DRM_ERROR("Read bstatus from DP/AUX failed (%zd)\n", ret); + DRM_DEBUG_KMS("Read bstatus from DP/AUX failed (%zd)\n", ret); return ret >= 0 ? -EIO : ret; } *ksv_ready = bstatus & DP_BSTATUS_READY; @@ -5482,8 +5732,8 @@ int intel_dp_hdcp_read_ksv_fifo(struct intel_digital_port *intel_dig_port, ksv_fifo + i * DRM_HDCP_KSV_LEN, len); if (ret != len) { - DRM_ERROR("Read ksv[%d] from DP/AUX failed (%zd)\n", i, - ret); + DRM_DEBUG_KMS("Read ksv[%d] from DP/AUX failed (%zd)\n", + i, ret); return ret >= 0 ? -EIO : ret; } } @@ -5503,7 +5753,7 @@ int intel_dp_hdcp_read_v_prime_part(struct intel_digital_port *intel_dig_port, DP_AUX_HDCP_V_PRIME(i), part, DRM_HDCP_V_PRIME_PART_LEN); if (ret != DRM_HDCP_V_PRIME_PART_LEN) { - DRM_ERROR("Read v'[%d] from DP/AUX failed (%zd)\n", i, ret); + DRM_DEBUG_KMS("Read v'[%d] from DP/AUX failed (%zd)\n", i, ret); return ret >= 0 ? -EIO : ret; } return 0; @@ -5526,7 +5776,7 @@ bool intel_dp_hdcp_check_link(struct intel_digital_port *intel_dig_port) ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, DP_AUX_HDCP_BSTATUS, &bstatus, 1); if (ret != 1) { - DRM_ERROR("Read bstatus from DP/AUX failed (%zd)\n", ret); + DRM_DEBUG_KMS("Read bstatus from DP/AUX failed (%zd)\n", ret); return false; } @@ -5565,6 +5815,7 @@ static const struct intel_hdcp_shim intel_dp_hdcp_shim = { static void intel_edp_panel_vdd_sanitize(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); lockdep_assert_held(&dev_priv->pps_mutex); @@ -5578,7 +5829,7 @@ static void intel_edp_panel_vdd_sanitize(struct intel_dp *intel_dp) * indefinitely. */ DRM_DEBUG_KMS("VDD left on by BIOS, adjusting state tracking\n"); - intel_display_power_get(dev_priv, intel_dp->aux_power_domain); + intel_display_power_get(dev_priv, intel_aux_power_domain(dig_port)); edp_panel_vdd_schedule_off(intel_dp); } @@ -5601,6 +5852,7 @@ void intel_dp_encoder_reset(struct drm_encoder *encoder) struct drm_i915_private *dev_priv = to_i915(encoder->dev); struct intel_dp *intel_dp = enc_to_intel_dp(encoder); struct intel_lspcon *lspcon = dp_to_lspcon(intel_dp); + intel_wakeref_t wakeref; if (!HAS_DDI(dev_priv)) intel_dp->DP = I915_READ(intel_dp->output_reg); @@ -5610,18 +5862,19 @@ void intel_dp_encoder_reset(struct drm_encoder *encoder) intel_dp->reset_link_params = true; - pps_lock(intel_dp); - - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) - intel_dp->active_pipe = vlv_active_pipe(intel_dp); + with_pps_lock(intel_dp, wakeref) { + if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) + intel_dp->active_pipe = vlv_active_pipe(intel_dp); - if (intel_dp_is_edp(intel_dp)) { - /* Reinit the power sequencer, in case BIOS did something with it. */ - intel_dp_pps_init(intel_dp); - intel_edp_panel_vdd_sanitize(intel_dp); + if (intel_dp_is_edp(intel_dp)) { + /* + * Reinit the power sequencer, in case BIOS did + * something nasty with it. + */ + intel_dp_pps_init(intel_dp); + intel_edp_panel_vdd_sanitize(intel_dp); + } } - - pps_unlock(intel_dp); } static const struct drm_connector_funcs intel_dp_connector_funcs = { @@ -5631,7 +5884,7 @@ static const struct drm_connector_funcs intel_dp_connector_funcs = { .atomic_set_property = intel_digital_connector_atomic_set_property, .late_register = intel_dp_connector_register, .early_unregister = intel_dp_connector_unregister, - .destroy = intel_dp_connector_destroy, + .destroy = intel_connector_destroy, .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, .atomic_duplicate_state = intel_digital_connector_duplicate_state, }; @@ -5654,6 +5907,7 @@ intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd) struct intel_dp *intel_dp = &intel_dig_port->dp; struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); enum irqreturn ret = IRQ_NONE; + intel_wakeref_t wakeref; if (long_hpd && intel_dig_port->base.type == INTEL_OUTPUT_EDP) { /* @@ -5673,11 +5927,11 @@ intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd) if (long_hpd) { intel_dp->reset_link_params = true; - intel_dp->detect_done = false; return IRQ_NONE; } - intel_display_power_get(dev_priv, intel_dp->aux_power_domain); + wakeref = intel_display_power_get(dev_priv, + intel_aux_power_domain(intel_dig_port)); if (intel_dp->is_mst) { if (intel_dp_check_mst_status(intel_dp) == -EINVAL) { @@ -5690,7 +5944,6 @@ intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd) intel_dp->is_mst = false; drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr, intel_dp->is_mst); - intel_dp->detect_done = false; goto put_power; } } @@ -5700,19 +5953,16 @@ intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd) handled = intel_dp_short_pulse(intel_dp); - /* Short pulse can signify loss of hdcp authentication */ - intel_hdcp_check_link(intel_dp->attached_connector); - - if (!handled) { - intel_dp->detect_done = false; + if (!handled) goto put_power; - } } ret = IRQ_HANDLED; put_power: - intel_display_power_put(dev_priv, intel_dp->aux_power_domain); + intel_display_power_put(dev_priv, + intel_aux_power_domain(intel_dig_port), + wakeref); return ret; } @@ -5743,12 +5993,16 @@ intel_dp_add_properties(struct intel_dp *intel_dp, struct drm_connector *connect intel_attach_force_audio_property(connector); intel_attach_broadcast_rgb_property(connector); + if (HAS_GMCH(dev_priv)) + drm_connector_attach_max_bpc_property(connector, 6, 10); + else if (INTEL_GEN(dev_priv) >= 5) + drm_connector_attach_max_bpc_property(connector, 6, 12); if (intel_dp_is_edp(intel_dp)) { u32 allowed_scalers; allowed_scalers = BIT(DRM_MODE_SCALE_ASPECT) | BIT(DRM_MODE_SCALE_FULLSCREEN); - if (!HAS_GMCH_DISPLAY(dev_priv)) + if (!HAS_GMCH(dev_priv)) allowed_scalers |= BIT(DRM_MODE_SCALE_CENTER); drm_connector_attach_scaling_mode_property(connector, allowed_scalers); @@ -6099,10 +6353,10 @@ static void intel_dp_set_drrs_state(struct drm_i915_private *dev_priv, if (INTEL_GEN(dev_priv) >= 8 && !IS_CHERRYVIEW(dev_priv)) { switch (index) { case DRRS_HIGH_RR: - intel_dp_set_m_n(intel_crtc, M1_N1); + intel_dp_set_m_n(crtc_state, M1_N1); break; case DRRS_LOW_RR: - intel_dp_set_m_n(intel_crtc, M2_N2); + intel_dp_set_m_n(crtc_state, M2_N2); break; case DRRS_MAX_RR: default: @@ -6155,8 +6409,8 @@ void intel_edp_drrs_enable(struct intel_dp *intel_dp, } mutex_lock(&dev_priv->drrs.mutex); - if (WARN_ON(dev_priv->drrs.dp)) { - DRM_ERROR("DRRS already enabled\n"); + if (dev_priv->drrs.dp) { + DRM_DEBUG_KMS("DRRS already enabled\n"); goto unlock; } @@ -6416,12 +6670,15 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp, struct drm_display_mode *downclock_mode = NULL; bool has_dpcd; struct drm_display_mode *scan; - struct edid *edid; enum pipe pipe = INVALID_PIPE; + intel_wakeref_t wakeref; + struct edid *edid; if (!intel_dp_is_edp(intel_dp)) return true; + INIT_DELAYED_WORK(&intel_dp->panel_vdd_work, edp_panel_vdd_work); + /* * On IBX/CPT we may get here with LVDS already registered. Since the * driver uses the only internal power sequencer available for both @@ -6435,13 +6692,11 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp, return false; } - pps_lock(intel_dp); - - intel_dp_init_panel_power_timestamps(intel_dp); - intel_dp_pps_init(intel_dp); - intel_edp_panel_vdd_sanitize(intel_dp); - - pps_unlock(intel_dp); + with_pps_lock(intel_dp, wakeref) { + intel_dp_init_panel_power_timestamps(intel_dp); + intel_dp_pps_init(intel_dp); + intel_edp_panel_vdd_sanitize(intel_dp); + } /* Cache DPCD and EDID for edp. */ has_dpcd = intel_edp_init_dpcd(intel_dp); @@ -6514,6 +6769,10 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp, intel_connector->panel.backlight.power = intel_edp_backlight_power; intel_panel_setup_backlight(connector, pipe); + if (fixed_mode) + drm_connector_init_panel_orientation_property( + connector, fixed_mode->hdisplay, fixed_mode->vdisplay); + return true; out_vdd_off: @@ -6522,9 +6781,8 @@ out_vdd_off: * vdd might still be enabled do to the delayed vdd off. * Make sure vdd is actually turned off here. */ - pps_lock(intel_dp); - edp_panel_vdd_off_sync(intel_dp); - pps_unlock(intel_dp); + with_pps_lock(intel_dp, wakeref) + edp_panel_vdd_off_sync(intel_dp); return false; } @@ -6616,7 +6874,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, drm_connector_init(dev, connector, &intel_dp_connector_funcs, type); drm_connector_helper_add(connector, &intel_dp_connector_helper_funcs); - if (!HAS_GMCH_DISPLAY(dev_priv)) + if (!HAS_GMCH(dev_priv)) connector->interlace_allowed = true; connector->doublescan_allowed = 0; @@ -6624,9 +6882,6 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, intel_dp_aux_init(intel_dp); - INIT_DELAYED_WORK(&intel_dp->panel_vdd_work, - edp_panel_vdd_work); - intel_connector_attach_encoder(intel_connector, intel_encoder); if (HAS_DDI(dev_priv)) @@ -6701,6 +6956,7 @@ bool intel_dp_init(struct drm_i915_private *dev_priv, intel_encoder->compute_config = intel_dp_compute_config; intel_encoder->get_hw_state = intel_dp_get_hw_state; intel_encoder->get_config = intel_dp_get_config; + intel_encoder->update_pipe = intel_panel_update_backlight; intel_encoder->suspend = intel_dp_encoder_suspend; if (IS_CHERRYVIEW(dev_priv)) { intel_encoder->pre_pll_enable = chv_dp_pre_pll_enable; @@ -6743,6 +6999,7 @@ bool intel_dp_init(struct drm_i915_private *dev_priv, if (port != PORT_A) intel_infoframe_init(intel_dig_port); + intel_dig_port->aux_ch = intel_bios_port_aux_ch(dev_priv, port); if (!intel_dp_init_connector(intel_dig_port, intel_connector)) goto err_init_connector; @@ -6794,7 +7051,10 @@ void intel_dp_mst_resume(struct drm_i915_private *dev_priv) continue; ret = drm_dp_mst_topology_mgr_resume(&intel_dp->mst_mgr); - if (ret) - intel_dp_check_mst_status(intel_dp); + if (ret) { + intel_dp->is_mst = false; + drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr, + false); + } } } diff --git a/drivers/gpu/drm/i915/intel_dp_link_training.c b/drivers/gpu/drm/i915/intel_dp_link_training.c index 30be0e39bd5f..b59c87daa4f7 100644 --- a/drivers/gpu/drm/i915/intel_dp_link_training.c +++ b/drivers/gpu/drm/i915/intel_dp_link_training.c @@ -24,7 +24,7 @@ #include "intel_drv.h" static void -intel_dp_dump_link_status(const uint8_t link_status[DP_LINK_STATUS_SIZE]) +intel_dp_dump_link_status(const u8 link_status[DP_LINK_STATUS_SIZE]) { DRM_DEBUG_KMS("ln0_1:0x%x ln2_3:0x%x align:0x%x sink:0x%x adj_req0_1:0x%x adj_req2_3:0x%x", @@ -34,17 +34,17 @@ intel_dp_dump_link_status(const uint8_t link_status[DP_LINK_STATUS_SIZE]) static void intel_get_adjust_train(struct intel_dp *intel_dp, - const uint8_t link_status[DP_LINK_STATUS_SIZE]) + const u8 link_status[DP_LINK_STATUS_SIZE]) { - uint8_t v = 0; - uint8_t p = 0; + u8 v = 0; + u8 p = 0; int lane; - uint8_t voltage_max; - uint8_t preemph_max; + u8 voltage_max; + u8 preemph_max; for (lane = 0; lane < intel_dp->lane_count; lane++) { - uint8_t this_v = drm_dp_get_adjust_request_voltage(link_status, lane); - uint8_t this_p = drm_dp_get_adjust_request_pre_emphasis(link_status, lane); + u8 this_v = drm_dp_get_adjust_request_voltage(link_status, lane); + u8 this_p = drm_dp_get_adjust_request_pre_emphasis(link_status, lane); if (this_v > v) v = this_v; @@ -66,9 +66,9 @@ intel_get_adjust_train(struct intel_dp *intel_dp, static bool intel_dp_set_link_train(struct intel_dp *intel_dp, - uint8_t dp_train_pat) + u8 dp_train_pat) { - uint8_t buf[sizeof(intel_dp->train_set) + 1]; + u8 buf[sizeof(intel_dp->train_set) + 1]; int ret, len; intel_dp_program_link_training_pattern(intel_dp, dp_train_pat); @@ -92,7 +92,7 @@ intel_dp_set_link_train(struct intel_dp *intel_dp, static bool intel_dp_reset_link_train(struct intel_dp *intel_dp, - uint8_t dp_train_pat) + u8 dp_train_pat) { memset(intel_dp->train_set, 0, sizeof(intel_dp->train_set)); intel_dp_set_signal_levels(intel_dp); @@ -128,11 +128,11 @@ static bool intel_dp_link_max_vswing_reached(struct intel_dp *intel_dp) static bool intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp) { - uint8_t voltage; + u8 voltage; int voltage_tries, cr_tries, max_cr_tries; bool max_vswing_reached = false; - uint8_t link_config[2]; - uint8_t link_bw, rate_select; + u8 link_config[2]; + u8 link_bw, rate_select; if (intel_dp->prepare_link_retrain) intel_dp->prepare_link_retrain(intel_dp); @@ -186,7 +186,7 @@ intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp) voltage_tries = 1; for (cr_tries = 0; cr_tries < max_cr_tries; ++cr_tries) { - uint8_t link_status[DP_LINK_STATUS_SIZE]; + u8 link_status[DP_LINK_STATUS_SIZE]; drm_dp_link_train_clock_recovery_delay(intel_dp->dpcd); @@ -282,7 +282,7 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp) { int tries; u32 training_pattern; - uint8_t link_status[DP_LINK_STATUS_SIZE]; + u8 link_status[DP_LINK_STATUS_SIZE]; bool channel_eq = false; training_pattern = intel_dp_training_pattern(intel_dp); diff --git a/drivers/gpu/drm/i915/intel_dp_mst.c b/drivers/gpu/drm/i915/intel_dp_mst.c index a911691dbd0f..fb67cd931117 100644 --- a/drivers/gpu/drm/i915/intel_dp_mst.c +++ b/drivers/gpu/drm/i915/intel_dp_mst.c @@ -23,16 +23,15 @@ * */ -#include <drm/drmP.h> #include "i915_drv.h" #include "intel_drv.h" #include <drm/drm_atomic_helper.h> -#include <drm/drm_crtc_helper.h> #include <drm/drm_edid.h> +#include <drm/drm_probe_helper.h> -static bool intel_dp_mst_compute_config(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config, - struct drm_connector_state *conn_state) +static int intel_dp_mst_compute_config(struct intel_encoder *encoder, + struct intel_crtc_state *pipe_config, + struct drm_connector_state *conn_state) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base); @@ -41,16 +40,21 @@ static bool intel_dp_mst_compute_config(struct intel_encoder *encoder, struct drm_connector *connector = conn_state->connector; void *port = to_intel_connector(connector)->port; struct drm_atomic_state *state = pipe_config->base.state; + struct drm_crtc *crtc = pipe_config->base.crtc; + struct drm_crtc_state *old_crtc_state = + drm_atomic_get_old_crtc_state(state, crtc); int bpp; - int lane_count, slots = 0; + int lane_count, slots = + to_intel_crtc_state(old_crtc_state)->dp_m_n.tu; const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; int mst_pbn; bool constant_n = drm_dp_has_quirk(&intel_dp->desc, DP_DPCD_QUIRK_CONSTANT_N); if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) - return false; + return -EINVAL; + pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; pipe_config->has_pch_encoder = false; bpp = 24; if (intel_dp->compliance.test_data.bpc) { @@ -76,17 +80,12 @@ static bool intel_dp_mst_compute_config(struct intel_encoder *encoder, mst_pbn = drm_dp_calc_pbn_mode(adjusted_mode->crtc_clock, bpp); pipe_config->pbn = mst_pbn; - /* Zombie connectors can't have VCPI slots */ - if (!drm_connector_is_unregistered(connector)) { - slots = drm_dp_atomic_find_vcpi_slots(state, - &intel_dp->mst_mgr, - port, - mst_pbn); - if (slots < 0) { - DRM_DEBUG_KMS("failed finding vcpi slots:%d\n", - slots); - return false; - } + slots = drm_dp_atomic_find_vcpi_slots(state, &intel_dp->mst_mgr, port, + mst_pbn); + if (slots < 0) { + DRM_DEBUG_KMS("failed finding vcpi slots:%d\n", + slots); + return slots; } intel_link_compute_m_n(bpp, lane_count, @@ -103,38 +102,42 @@ static bool intel_dp_mst_compute_config(struct intel_encoder *encoder, intel_ddi_compute_min_voltage_level(dev_priv, pipe_config); - return true; + return 0; } -static int intel_dp_mst_atomic_check(struct drm_connector *connector, - struct drm_connector_state *new_conn_state) +static int +intel_dp_mst_atomic_check(struct drm_connector *connector, + struct drm_connector_state *new_conn_state) { struct drm_atomic_state *state = new_conn_state->state; - struct drm_connector_state *old_conn_state; - struct drm_crtc *old_crtc; + struct drm_connector_state *old_conn_state = + drm_atomic_get_old_connector_state(state, connector); + struct intel_connector *intel_connector = + to_intel_connector(connector); + struct drm_crtc *new_crtc = new_conn_state->crtc; struct drm_crtc_state *crtc_state; - int slots, ret = 0; - - old_conn_state = drm_atomic_get_old_connector_state(state, connector); - old_crtc = old_conn_state->crtc; - if (!old_crtc) - return ret; + struct drm_dp_mst_topology_mgr *mgr; + int ret = 0; - crtc_state = drm_atomic_get_new_crtc_state(state, old_crtc); - slots = to_intel_crtc_state(crtc_state)->dp_m_n.tu; - if (drm_atomic_crtc_needs_modeset(crtc_state) && slots > 0) { - struct drm_dp_mst_topology_mgr *mgr; - struct drm_encoder *old_encoder; + if (!old_conn_state->crtc) + return 0; - old_encoder = old_conn_state->best_encoder; - mgr = &enc_to_mst(old_encoder)->primary->dp.mst_mgr; + /* We only want to free VCPI if this state disables the CRTC on this + * connector + */ + if (new_crtc) { + crtc_state = drm_atomic_get_new_crtc_state(state, new_crtc); - ret = drm_dp_atomic_release_vcpi_slots(state, mgr, slots); - if (ret) - DRM_DEBUG_KMS("failed releasing %d vcpi slots:%d\n", slots, ret); - else - to_intel_crtc_state(crtc_state)->dp_m_n.tu = 0; + if (!crtc_state || + !drm_atomic_crtc_needs_modeset(crtc_state) || + crtc_state->enable) + return 0; } + + mgr = &enc_to_mst(old_conn_state->best_encoder)->primary->dp.mst_mgr; + ret = drm_dp_atomic_release_vcpi_slots(state, mgr, + intel_connector->port); + return ret; } @@ -208,12 +211,25 @@ static void intel_mst_pre_pll_enable_dp(struct intel_encoder *encoder, struct intel_digital_port *intel_dig_port = intel_mst->primary; struct intel_dp *intel_dp = &intel_dig_port->dp; - if (intel_dp->active_mst_links == 0 && - intel_dig_port->base.pre_pll_enable) + if (intel_dp->active_mst_links == 0) intel_dig_port->base.pre_pll_enable(&intel_dig_port->base, pipe_config, NULL); } +static void intel_mst_post_pll_disable_dp(struct intel_encoder *encoder, + const struct intel_crtc_state *old_crtc_state, + const struct drm_connector_state *old_conn_state) +{ + struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base); + struct intel_digital_port *intel_dig_port = intel_mst->primary; + struct intel_dp *intel_dp = &intel_dig_port->dp; + + if (intel_dp->active_mst_links == 0) + intel_dig_port->base.post_pll_disable(&intel_dig_port->base, + old_crtc_state, + old_conn_state); +} + static void intel_mst_pre_enable_dp(struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) @@ -226,7 +242,7 @@ static void intel_mst_pre_enable_dp(struct intel_encoder *encoder, struct intel_connector *connector = to_intel_connector(conn_state->connector); int ret; - uint32_t temp; + u32 temp; /* MST encoders are bound to a crtc, not to a connector, * force the mapping here for get_hw_state. @@ -335,24 +351,12 @@ intel_dp_mst_detect(struct drm_connector *connector, bool force) intel_connector->port); } -static void -intel_dp_mst_connector_destroy(struct drm_connector *connector) -{ - struct intel_connector *intel_connector = to_intel_connector(connector); - - if (!IS_ERR_OR_NULL(intel_connector->edid)) - kfree(intel_connector->edid); - - drm_connector_cleanup(connector); - kfree(connector); -} - static const struct drm_connector_funcs intel_dp_mst_connector_funcs = { .detect = intel_dp_mst_detect, .fill_modes = drm_helper_probe_single_connector_modes, .late_register = intel_connector_register, .early_unregister = intel_connector_unregister, - .destroy = intel_dp_mst_connector_destroy, + .destroy = intel_connector_destroy, .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, }; @@ -455,6 +459,7 @@ static struct drm_connector *intel_dp_add_mst_connector(struct drm_dp_mst_topolo intel_connector->get_hw_state = intel_dp_mst_get_hw_state; intel_connector->mst_port = intel_dp; intel_connector->port = port; + drm_dp_mst_get_port_malloc(port); connector = &intel_connector->base; ret = drm_connector_init(dev, connector, &intel_dp_mst_connector_funcs, @@ -515,20 +520,10 @@ static void intel_dp_destroy_mst_connector(struct drm_dp_mst_topology_mgr *mgr, drm_connector_put(connector); } -static void intel_dp_mst_hotplug(struct drm_dp_mst_topology_mgr *mgr) -{ - struct intel_dp *intel_dp = container_of(mgr, struct intel_dp, mst_mgr); - struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - struct drm_device *dev = intel_dig_port->base.base.dev; - - drm_kms_helper_hotplug_event(dev); -} - static const struct drm_dp_mst_topology_cbs mst_cbs = { .add_connector = intel_dp_add_mst_connector, .register_connector = intel_dp_register_mst_connector, .destroy_connector = intel_dp_destroy_mst_connector, - .hotplug = intel_dp_mst_hotplug, }; static struct intel_dp_mst_encoder * @@ -560,6 +555,7 @@ intel_dp_create_fake_mst_encoder(struct intel_digital_port *intel_dig_port, enum intel_encoder->disable = intel_mst_disable_dp; intel_encoder->post_disable = intel_mst_post_disable_dp; intel_encoder->pre_pll_enable = intel_mst_pre_pll_enable_dp; + intel_encoder->post_pll_disable = intel_mst_post_pll_disable_dp; intel_encoder->pre_enable = intel_mst_pre_enable_dp; intel_encoder->enable = intel_mst_enable_dp; intel_encoder->get_hw_state = intel_dp_mst_enc_get_hw_state; diff --git a/drivers/gpu/drm/i915/intel_dpio_phy.c b/drivers/gpu/drm/i915/intel_dpio_phy.c index 00b3ab656b06..95cb8b154f87 100644 --- a/drivers/gpu/drm/i915/intel_dpio_phy.c +++ b/drivers/gpu/drm/i915/intel_dpio_phy.c @@ -413,7 +413,7 @@ static void _bxt_ddi_phy_init(struct drm_i915_private *dev_priv, } if (phy_info->rcomp_phy != -1) { - uint32_t grc_code; + u32 grc_code; bxt_phy_wait_grc_done(dev_priv, phy_info->rcomp_phy); @@ -445,7 +445,7 @@ static void _bxt_ddi_phy_init(struct drm_i915_private *dev_priv, void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy) { const struct bxt_ddi_phy_info *phy_info; - uint32_t val; + u32 val; phy_info = bxt_get_phy_info(dev_priv, phy); @@ -515,7 +515,7 @@ bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv, enum dpio_phy phy) { const struct bxt_ddi_phy_info *phy_info; - uint32_t mask; + u32 mask; bool ok; phy_info = bxt_get_phy_info(dev_priv, phy); @@ -567,8 +567,8 @@ bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv, #undef _CHK } -uint8_t -bxt_ddi_phy_calc_lane_lat_optim_mask(uint8_t lane_count) +u8 +bxt_ddi_phy_calc_lane_lat_optim_mask(u8 lane_count) { switch (lane_count) { case 1: @@ -585,7 +585,7 @@ bxt_ddi_phy_calc_lane_lat_optim_mask(uint8_t lane_count) } void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder, - uint8_t lane_lat_optim_mask) + u8 lane_lat_optim_mask) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); enum port port = encoder->port; @@ -610,7 +610,7 @@ void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder, } } -uint8_t +u8 bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); @@ -618,7 +618,7 @@ bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder) enum dpio_phy phy; enum dpio_channel ch; int lane; - uint8_t mask; + u8 mask; bxt_port_to_phy_channel(dev_priv, port, &phy, &ch); @@ -739,7 +739,7 @@ void chv_data_lane_soft_reset(struct intel_encoder *encoder, enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base)); struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); enum pipe pipe = crtc->pipe; - uint32_t val; + u32 val; val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch)); if (reset) @@ -748,7 +748,7 @@ void chv_data_lane_soft_reset(struct intel_encoder *encoder, val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET; vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val); - if (crtc->config->lane_count > 2) { + if (crtc_state->lane_count > 2) { val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch)); if (reset) val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); @@ -765,7 +765,7 @@ void chv_data_lane_soft_reset(struct intel_encoder *encoder, val |= DPIO_PCS_CLK_SOFT_RESET; vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW1(ch), val); - if (crtc->config->lane_count > 2) { + if (crtc_state->lane_count > 2) { val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch)); val |= CHV_PCS_REQ_SOFTRESET_EN; if (reset) diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.c b/drivers/gpu/drm/i915/intel_dpll_mgr.c index e6cac9225536..0a42d11c4c33 100644 --- a/drivers/gpu/drm/i915/intel_dpll_mgr.c +++ b/drivers/gpu/drm/i915/intel_dpll_mgr.c @@ -126,16 +126,16 @@ void assert_shared_dpll(struct drm_i915_private *dev_priv, /** * intel_prepare_shared_dpll - call a dpll's prepare hook - * @crtc: CRTC which has a shared dpll + * @crtc_state: CRTC, and its state, which has a shared dpll * * This calls the PLL's prepare hook if it has one and if the PLL is not * already enabled. The prepare hook is platform specific. */ -void intel_prepare_shared_dpll(struct intel_crtc *crtc) +void intel_prepare_shared_dpll(const struct intel_crtc_state *crtc_state) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_shared_dpll *pll = crtc->config->shared_dpll; + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct intel_shared_dpll *pll = crtc_state->shared_dpll; if (WARN_ON(pll == NULL)) return; @@ -154,15 +154,15 @@ void intel_prepare_shared_dpll(struct intel_crtc *crtc) /** * intel_enable_shared_dpll - enable a CRTC's shared DPLL - * @crtc: CRTC which has a shared DPLL + * @crtc_state: CRTC, and its state, which has a shared DPLL * * Enable the shared DPLL used by @crtc. */ -void intel_enable_shared_dpll(struct intel_crtc *crtc) +void intel_enable_shared_dpll(const struct intel_crtc_state *crtc_state) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_shared_dpll *pll = crtc->config->shared_dpll; + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct intel_shared_dpll *pll = crtc_state->shared_dpll; unsigned int crtc_mask = drm_crtc_mask(&crtc->base); unsigned int old_mask; @@ -199,14 +199,15 @@ out: /** * intel_disable_shared_dpll - disable a CRTC's shared DPLL - * @crtc: CRTC which has a shared DPLL + * @crtc_state: CRTC, and its state, which has a shared DPLL * * Disable the shared DPLL used by @crtc. */ -void intel_disable_shared_dpll(struct intel_crtc *crtc) +void intel_disable_shared_dpll(const struct intel_crtc_state *crtc_state) { + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - struct intel_shared_dpll *pll = crtc->config->shared_dpll; + struct intel_shared_dpll *pll = crtc_state->shared_dpll; unsigned int crtc_mask = drm_crtc_mask(&crtc->base); /* PCH only available on ILK+ */ @@ -246,7 +247,7 @@ intel_find_shared_dpll(struct intel_crtc *crtc, enum intel_dpll_id range_max) { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - struct intel_shared_dpll *pll; + struct intel_shared_dpll *pll, *unused_pll = NULL; struct intel_shared_dpll_state *shared_dpll; enum intel_dpll_id i; @@ -256,8 +257,11 @@ intel_find_shared_dpll(struct intel_crtc *crtc, pll = &dev_priv->shared_dplls[i]; /* Only want to check enabled timings first */ - if (shared_dpll[i].crtc_mask == 0) + if (shared_dpll[i].crtc_mask == 0) { + if (!unused_pll) + unused_pll = pll; continue; + } if (memcmp(&crtc_state->dpll_hw_state, &shared_dpll[i].hw_state, @@ -272,14 +276,11 @@ intel_find_shared_dpll(struct intel_crtc *crtc, } /* Ok no matching timings, maybe there's a free one? */ - for (i = range_min; i <= range_max; i++) { - pll = &dev_priv->shared_dplls[i]; - if (shared_dpll[i].crtc_mask == 0) { - DRM_DEBUG_KMS("[CRTC:%d:%s] allocated %s\n", - crtc->base.base.id, crtc->base.name, - pll->info->name); - return pll; - } + if (unused_pll) { + DRM_DEBUG_KMS("[CRTC:%d:%s] allocated %s\n", + crtc->base.base.id, crtc->base.name, + unused_pll->info->name); + return unused_pll; } return NULL; @@ -344,9 +345,12 @@ static bool ibx_pch_dpll_get_hw_state(struct drm_i915_private *dev_priv, struct intel_dpll_hw_state *hw_state) { const enum intel_dpll_id id = pll->info->id; - uint32_t val; + intel_wakeref_t wakeref; + u32 val; - if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_PLLS)) + wakeref = intel_display_power_get_if_enabled(dev_priv, + POWER_DOMAIN_PLLS); + if (!wakeref) return false; val = I915_READ(PCH_DPLL(id)); @@ -354,7 +358,7 @@ static bool ibx_pch_dpll_get_hw_state(struct drm_i915_private *dev_priv, hw_state->fp0 = I915_READ(PCH_FP0(id)); hw_state->fp1 = I915_READ(PCH_FP1(id)); - intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS); + intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS, wakeref); return val & DPLL_VCO_ENABLE; } @@ -409,14 +413,6 @@ static void ibx_pch_dpll_disable(struct drm_i915_private *dev_priv, struct intel_shared_dpll *pll) { const enum intel_dpll_id id = pll->info->id; - struct drm_device *dev = &dev_priv->drm; - struct intel_crtc *crtc; - - /* Make sure no transcoder isn't still depending on us. */ - for_each_intel_crtc(dev, crtc) { - if (crtc->config->shared_dpll == pll) - assert_pch_transcoder_disabled(dev_priv, crtc->pipe); - } I915_WRITE(PCH_DPLL(id), 0); POSTING_READ(PCH_DPLL(id)); @@ -494,7 +490,7 @@ static void hsw_ddi_wrpll_disable(struct drm_i915_private *dev_priv, struct intel_shared_dpll *pll) { const enum intel_dpll_id id = pll->info->id; - uint32_t val; + u32 val; val = I915_READ(WRPLL_CTL(id)); I915_WRITE(WRPLL_CTL(id), val & ~WRPLL_PLL_ENABLE); @@ -504,7 +500,7 @@ static void hsw_ddi_wrpll_disable(struct drm_i915_private *dev_priv, static void hsw_ddi_spll_disable(struct drm_i915_private *dev_priv, struct intel_shared_dpll *pll) { - uint32_t val; + u32 val; val = I915_READ(SPLL_CTL); I915_WRITE(SPLL_CTL, val & ~SPLL_PLL_ENABLE); @@ -516,15 +512,18 @@ static bool hsw_ddi_wrpll_get_hw_state(struct drm_i915_private *dev_priv, struct intel_dpll_hw_state *hw_state) { const enum intel_dpll_id id = pll->info->id; - uint32_t val; + intel_wakeref_t wakeref; + u32 val; - if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_PLLS)) + wakeref = intel_display_power_get_if_enabled(dev_priv, + POWER_DOMAIN_PLLS); + if (!wakeref) return false; val = I915_READ(WRPLL_CTL(id)); hw_state->wrpll = val; - intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS); + intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS, wakeref); return val & WRPLL_PLL_ENABLE; } @@ -533,15 +532,18 @@ static bool hsw_ddi_spll_get_hw_state(struct drm_i915_private *dev_priv, struct intel_shared_dpll *pll, struct intel_dpll_hw_state *hw_state) { - uint32_t val; + intel_wakeref_t wakeref; + u32 val; - if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_PLLS)) + wakeref = intel_display_power_get_if_enabled(dev_priv, + POWER_DOMAIN_PLLS); + if (!wakeref) return false; val = I915_READ(SPLL_CTL); hw_state->spll = val; - intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS); + intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS, wakeref); return val & SPLL_PLL_ENABLE; } @@ -637,11 +639,12 @@ static unsigned hsw_wrpll_get_budget_for_freq(int clock) return budget; } -static void hsw_wrpll_update_rnp(uint64_t freq2k, unsigned budget, - unsigned r2, unsigned n2, unsigned p, +static void hsw_wrpll_update_rnp(u64 freq2k, unsigned int budget, + unsigned int r2, unsigned int n2, + unsigned int p, struct hsw_wrpll_rnp *best) { - uint64_t a, b, c, d, diff, diff_best; + u64 a, b, c, d, diff, diff_best; /* No best (r,n,p) yet */ if (best->p == 0) { @@ -700,7 +703,7 @@ static void hsw_ddi_calculate_wrpll(int clock /* in Hz */, unsigned *r2_out, unsigned *n2_out, unsigned *p_out) { - uint64_t freq2k; + u64 freq2k; unsigned p, n2, r2; struct hsw_wrpll_rnp best = { 0, 0, 0 }; unsigned budget; @@ -766,7 +769,7 @@ static struct intel_shared_dpll *hsw_ddi_hdmi_get_dpll(int clock, struct intel_crtc_state *crtc_state) { struct intel_shared_dpll *pll; - uint32_t val; + u32 val; unsigned int p, n2, r2; hsw_ddi_calculate_wrpll(clock * 1000, &r2, &n2, &p); @@ -928,7 +931,7 @@ static void skl_ddi_pll_write_ctrl1(struct drm_i915_private *dev_priv, struct intel_shared_dpll *pll) { const enum intel_dpll_id id = pll->info->id; - uint32_t val; + u32 val; val = I915_READ(DPLL_CTRL1); @@ -993,12 +996,15 @@ static bool skl_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv, struct intel_shared_dpll *pll, struct intel_dpll_hw_state *hw_state) { - uint32_t val; + u32 val; const struct skl_dpll_regs *regs = skl_dpll_regs; const enum intel_dpll_id id = pll->info->id; + intel_wakeref_t wakeref; bool ret; - if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_PLLS)) + wakeref = intel_display_power_get_if_enabled(dev_priv, + POWER_DOMAIN_PLLS); + if (!wakeref) return false; ret = false; @@ -1018,7 +1024,7 @@ static bool skl_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv, ret = true; out: - intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS); + intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS, wakeref); return ret; } @@ -1027,12 +1033,15 @@ static bool skl_ddi_dpll0_get_hw_state(struct drm_i915_private *dev_priv, struct intel_shared_dpll *pll, struct intel_dpll_hw_state *hw_state) { - uint32_t val; const struct skl_dpll_regs *regs = skl_dpll_regs; const enum intel_dpll_id id = pll->info->id; + intel_wakeref_t wakeref; + u32 val; bool ret; - if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_PLLS)) + wakeref = intel_display_power_get_if_enabled(dev_priv, + POWER_DOMAIN_PLLS); + if (!wakeref) return false; ret = false; @@ -1048,15 +1057,15 @@ static bool skl_ddi_dpll0_get_hw_state(struct drm_i915_private *dev_priv, ret = true; out: - intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS); + intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS, wakeref); return ret; } struct skl_wrpll_context { - uint64_t min_deviation; /* current minimal deviation */ - uint64_t central_freq; /* chosen central freq */ - uint64_t dco_freq; /* chosen dco freq */ + u64 min_deviation; /* current minimal deviation */ + u64 central_freq; /* chosen central freq */ + u64 dco_freq; /* chosen dco freq */ unsigned int p; /* chosen divider */ }; @@ -1072,11 +1081,11 @@ static void skl_wrpll_context_init(struct skl_wrpll_context *ctx) #define SKL_DCO_MAX_NDEVIATION 600 static void skl_wrpll_try_divider(struct skl_wrpll_context *ctx, - uint64_t central_freq, - uint64_t dco_freq, + u64 central_freq, + u64 dco_freq, unsigned int divider) { - uint64_t deviation; + u64 deviation; deviation = div64_u64(10000 * abs_diff(dco_freq, central_freq), central_freq); @@ -1150,21 +1159,21 @@ static void skl_wrpll_get_multipliers(unsigned int p, } struct skl_wrpll_params { - uint32_t dco_fraction; - uint32_t dco_integer; - uint32_t qdiv_ratio; - uint32_t qdiv_mode; - uint32_t kdiv; - uint32_t pdiv; - uint32_t central_freq; + u32 dco_fraction; + u32 dco_integer; + u32 qdiv_ratio; + u32 qdiv_mode; + u32 kdiv; + u32 pdiv; + u32 central_freq; }; static void skl_wrpll_params_populate(struct skl_wrpll_params *params, - uint64_t afe_clock, - uint64_t central_freq, - uint32_t p0, uint32_t p1, uint32_t p2) + u64 afe_clock, + u64 central_freq, + u32 p0, u32 p1, u32 p2) { - uint64_t dco_freq; + u64 dco_freq; switch (central_freq) { case 9600000000ULL: @@ -1230,10 +1239,10 @@ static bool skl_ddi_calculate_wrpll(int clock /* in Hz */, struct skl_wrpll_params *wrpll_params) { - uint64_t afe_clock = clock * 5; /* AFE Clock is 5x Pixel clock */ - uint64_t dco_central_freq[3] = {8400000000ULL, - 9000000000ULL, - 9600000000ULL}; + u64 afe_clock = clock * 5; /* AFE Clock is 5x Pixel clock */ + u64 dco_central_freq[3] = { 8400000000ULL, + 9000000000ULL, + 9600000000ULL }; static const int even_dividers[] = { 4, 6, 8, 10, 12, 14, 16, 18, 20, 24, 28, 30, 32, 36, 40, 42, 44, 48, 52, 54, 56, 60, 64, 66, 68, @@ -1257,7 +1266,7 @@ skl_ddi_calculate_wrpll(int clock /* in Hz */, for (dco = 0; dco < ARRAY_SIZE(dco_central_freq); dco++) { for (i = 0; i < dividers[d].n_dividers; i++) { unsigned int p = dividers[d].list[i]; - uint64_t dco_freq = p * afe_clock; + u64 dco_freq = p * afe_clock; skl_wrpll_try_divider(&ctx, dco_central_freq[dco], @@ -1303,7 +1312,7 @@ static bool skl_ddi_hdmi_pll_dividers(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state, int clock) { - uint32_t ctrl1, cfgcr1, cfgcr2; + u32 ctrl1, cfgcr1, cfgcr2; struct skl_wrpll_params wrpll_params = { 0, }; /* @@ -1340,7 +1349,7 @@ static bool skl_ddi_dp_set_dpll_hw_state(int clock, struct intel_dpll_hw_state *dpll_hw_state) { - uint32_t ctrl1; + u32 ctrl1; /* * See comment in intel_dpll_hw_state to understand why we always use 0 @@ -1442,7 +1451,7 @@ static const struct intel_shared_dpll_funcs skl_ddi_dpll0_funcs = { static void bxt_ddi_pll_enable(struct drm_i915_private *dev_priv, struct intel_shared_dpll *pll) { - uint32_t temp; + u32 temp; enum port port = (enum port)pll->info->id; /* 1:1 port->PLL mapping */ enum dpio_phy phy; enum dpio_channel ch; @@ -1563,7 +1572,7 @@ static void bxt_ddi_pll_disable(struct drm_i915_private *dev_priv, struct intel_shared_dpll *pll) { enum port port = (enum port)pll->info->id; /* 1:1 port->PLL mapping */ - uint32_t temp; + u32 temp; temp = I915_READ(BXT_PORT_PLL_ENABLE(port)); temp &= ~PORT_PLL_ENABLE; @@ -1586,14 +1595,17 @@ static bool bxt_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv, struct intel_dpll_hw_state *hw_state) { enum port port = (enum port)pll->info->id; /* 1:1 port->PLL mapping */ - uint32_t val; - bool ret; + intel_wakeref_t wakeref; enum dpio_phy phy; enum dpio_channel ch; + u32 val; + bool ret; bxt_port_to_phy_channel(dev_priv, port, &phy, &ch); - if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_PLLS)) + wakeref = intel_display_power_get_if_enabled(dev_priv, + POWER_DOMAIN_PLLS); + if (!wakeref) return false; ret = false; @@ -1650,7 +1662,7 @@ static bool bxt_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv, ret = true; out: - intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS); + intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS, wakeref); return ret; } @@ -1658,12 +1670,12 @@ out: /* bxt clock parameters */ struct bxt_clk_div { int clock; - uint32_t p1; - uint32_t p2; - uint32_t m2_int; - uint32_t m2_frac; + u32 p1; + u32 p2; + u32 m2_int; + u32 m2_frac; bool m2_frac_en; - uint32_t n; + u32 n; int vco; }; @@ -1730,8 +1742,8 @@ static bool bxt_ddi_set_dpll_hw_state(int clock, struct intel_dpll_hw_state *dpll_hw_state) { int vco = clk_div->vco; - uint32_t prop_coef, int_coef, gain_ctl, targ_cnt; - uint32_t lanestagger; + u32 prop_coef, int_coef, gain_ctl, targ_cnt; + u32 lanestagger; if (vco >= 6200000 && vco <= 6700000) { prop_coef = 4; @@ -1880,7 +1892,7 @@ static void intel_ddi_pll_init(struct drm_device *dev) struct drm_i915_private *dev_priv = to_i915(dev); if (INTEL_GEN(dev_priv) < 9) { - uint32_t val = I915_READ(LCPLL_CTL); + u32 val = I915_READ(LCPLL_CTL); /* * The LCPLL register should be turned on by the BIOS. For now @@ -1966,7 +1978,7 @@ static void cnl_ddi_pll_enable(struct drm_i915_private *dev_priv, struct intel_shared_dpll *pll) { const enum intel_dpll_id id = pll->info->id; - uint32_t val; + u32 val; /* 1. Enable DPLL power in DPLL_ENABLE. */ val = I915_READ(CNL_DPLL_ENABLE(id)); @@ -2041,7 +2053,7 @@ static void cnl_ddi_pll_disable(struct drm_i915_private *dev_priv, struct intel_shared_dpll *pll) { const enum intel_dpll_id id = pll->info->id; - uint32_t val; + u32 val; /* * 1. Configure DPCLKA_CFGCR0 to turn off the clock for the DDI. @@ -2098,10 +2110,13 @@ static bool cnl_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv, struct intel_dpll_hw_state *hw_state) { const enum intel_dpll_id id = pll->info->id; - uint32_t val; + intel_wakeref_t wakeref; + u32 val; bool ret; - if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_PLLS)) + wakeref = intel_display_power_get_if_enabled(dev_priv, + POWER_DOMAIN_PLLS); + if (!wakeref) return false; ret = false; @@ -2120,7 +2135,7 @@ static bool cnl_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv, ret = true; out: - intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS); + intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS, wakeref); return ret; } @@ -2232,7 +2247,7 @@ cnl_ddi_calculate_wrpll(int clock, struct skl_wrpll_params *wrpll_params) { u32 afe_clock = clock * 5; - uint32_t ref_clock; + u32 ref_clock; u32 dco_min = 7998000; u32 dco_max = 10000000; u32 dco_mid = (dco_min + dco_max) / 2; @@ -2278,7 +2293,7 @@ static bool cnl_ddi_hdmi_pll_dividers(struct intel_crtc *crtc, int clock) { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - uint32_t cfgcr0, cfgcr1; + u32 cfgcr0, cfgcr1; struct skl_wrpll_params wrpll_params = { 0, }; cfgcr0 = DPLL_CFGCR0_HDMI_MODE; @@ -2307,7 +2322,7 @@ static bool cnl_ddi_dp_set_dpll_hw_state(int clock, struct intel_dpll_hw_state *dpll_hw_state) { - uint32_t cfgcr0; + u32 cfgcr0; cfgcr0 = DPLL_CFGCR0_SSC_ENABLE; @@ -2524,13 +2539,14 @@ static bool icl_calc_dpll_state(struct intel_crtc_state *crtc_state, struct intel_dpll_hw_state *pll_state) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - uint32_t cfgcr0, cfgcr1; + u32 cfgcr0, cfgcr1; struct skl_wrpll_params pll_params = { 0 }; bool ret; if (intel_port_is_tc(dev_priv, encoder->port)) ret = icl_calc_tbt_pll(dev_priv, clock, &pll_params); - else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) + else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) || + intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI)) ret = cnl_ddi_calculate_wrpll(clock, dev_priv, &pll_params); else ret = icl_calc_dp_combo_pll(dev_priv, clock, &pll_params); @@ -2553,10 +2569,10 @@ static bool icl_calc_dpll_state(struct intel_crtc_state *crtc_state, } int icl_calc_dp_combo_pll_link(struct drm_i915_private *dev_priv, - uint32_t pll_id) + u32 pll_id) { - uint32_t cfgcr0, cfgcr1; - uint32_t pdiv, kdiv, qdiv_mode, qdiv_ratio, dco_integer, dco_fraction; + u32 cfgcr0, cfgcr1; + u32 pdiv, kdiv, qdiv_mode, qdiv_ratio, dco_integer, dco_fraction; const struct skl_wrpll_params *params; int index, n_entries, link_clock; @@ -2623,21 +2639,26 @@ int icl_calc_dp_combo_pll_link(struct drm_i915_private *dev_priv, return link_clock; } -static enum port icl_mg_pll_id_to_port(enum intel_dpll_id id) +static enum tc_port icl_pll_id_to_tc_port(enum intel_dpll_id id) +{ + return id - DPLL_ID_ICL_MGPLL1; +} + +enum intel_dpll_id icl_tc_port_to_pll_id(enum tc_port tc_port) { - return id - DPLL_ID_ICL_MGPLL1 + PORT_C; + return tc_port + DPLL_ID_ICL_MGPLL1; } -static enum intel_dpll_id icl_port_to_mg_pll_id(enum port port) +bool intel_dpll_is_combophy(enum intel_dpll_id id) { - return port - PORT_C + DPLL_ID_ICL_MGPLL1; + return id == DPLL_ID_ICL_DPLL0 || id == DPLL_ID_ICL_DPLL1; } static bool icl_mg_pll_find_divisors(int clock_khz, bool is_dp, bool use_ssc, - uint32_t *target_dco_khz, + u32 *target_dco_khz, struct intel_dpll_hw_state *state) { - uint32_t dco_min_freq, dco_max_freq; + u32 dco_min_freq, dco_max_freq; int div1_vals[] = {7, 5, 3, 2}; unsigned int i; int div2; @@ -2713,12 +2734,12 @@ static bool icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state, { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); int refclk_khz = dev_priv->cdclk.hw.ref; - uint32_t dco_khz, m1div, m2div_int, m2div_rem, m2div_frac; - uint32_t iref_ndiv, iref_trim, iref_pulse_w; - uint32_t prop_coeff, int_coeff; - uint32_t tdc_targetcnt, feedfwgain; - uint64_t ssc_stepsize, ssc_steplen, ssc_steplog; - uint64_t tmp; + u32 dco_khz, m1div, m2div_int, m2div_rem, m2div_frac; + u32 iref_ndiv, iref_trim, iref_pulse_w; + u32 prop_coeff, int_coeff; + u32 tdc_targetcnt, feedfwgain; + u64 ssc_stepsize, ssc_steplen, ssc_steplog; + u64 tmp; bool use_ssc = false; bool is_dp = !intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI); @@ -2741,7 +2762,7 @@ static bool icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state, } m2div_rem = dco_khz % (refclk_khz * m1div); - tmp = (uint64_t)m2div_rem * (1 << 22); + tmp = (u64)m2div_rem * (1 << 22); do_div(tmp, refclk_khz * m1div); m2div_frac = tmp; @@ -2800,11 +2821,11 @@ static bool icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state, } if (use_ssc) { - tmp = (uint64_t)dco_khz * 47 * 32; + tmp = (u64)dco_khz * 47 * 32; do_div(tmp, refclk_khz * m1div * 10000); ssc_stepsize = tmp; - tmp = (uint64_t)dco_khz * 1000; + tmp = (u64)dco_khz * 1000; ssc_steplen = DIV_ROUND_UP_ULL(tmp, 32 * 2 * 32); } else { ssc_stepsize = 0; @@ -2874,8 +2895,8 @@ static struct intel_shared_dpll * icl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state, struct intel_encoder *encoder) { - struct intel_digital_port *intel_dig_port = - enc_to_dig_port(&encoder->base); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_digital_port *intel_dig_port; struct intel_shared_dpll *pll; struct intel_dpll_hw_state pll_state = {}; enum port port = encoder->port; @@ -2883,31 +2904,36 @@ icl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state, int clock = crtc_state->port_clock; bool ret; - switch (port) { - case PORT_A: - case PORT_B: + if (intel_port_is_combophy(dev_priv, port)) { min = DPLL_ID_ICL_DPLL0; max = DPLL_ID_ICL_DPLL1; ret = icl_calc_dpll_state(crtc_state, encoder, clock, &pll_state); - break; - case PORT_C: - case PORT_D: - case PORT_E: - case PORT_F: + } else if (intel_port_is_tc(dev_priv, port)) { + if (encoder->type == INTEL_OUTPUT_DP_MST) { + struct intel_dp_mst_encoder *mst_encoder; + + mst_encoder = enc_to_mst(&encoder->base); + intel_dig_port = mst_encoder->primary; + } else { + intel_dig_port = enc_to_dig_port(&encoder->base); + } + if (intel_dig_port->tc_type == TC_PORT_TBT) { min = DPLL_ID_ICL_TBTPLL; max = min; ret = icl_calc_dpll_state(crtc_state, encoder, clock, &pll_state); } else { - min = icl_port_to_mg_pll_id(port); + enum tc_port tc_port; + + tc_port = intel_port_to_tc(dev_priv, port); + min = icl_tc_port_to_pll_id(tc_port); max = min; ret = icl_calc_mg_pll_state(crtc_state, encoder, clock, &pll_state); } - break; - default: + } else { MISSING_CASE(port); return NULL; } @@ -2932,21 +2958,12 @@ icl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state, static i915_reg_t icl_pll_id_to_enable_reg(enum intel_dpll_id id) { - switch (id) { - default: - MISSING_CASE(id); - /* fall through */ - case DPLL_ID_ICL_DPLL0: - case DPLL_ID_ICL_DPLL1: + if (intel_dpll_is_combophy(id)) return CNL_DPLL_ENABLE(id); - case DPLL_ID_ICL_TBTPLL: + else if (id == DPLL_ID_ICL_TBTPLL) return TBT_PLL_ENABLE; - case DPLL_ID_ICL_MGPLL1: - case DPLL_ID_ICL_MGPLL2: - case DPLL_ID_ICL_MGPLL3: - case DPLL_ID_ICL_MGPLL4: - return MG_PLL_ENABLE(icl_mg_pll_id_to_port(id)); - } + + return MG_PLL_ENABLE(icl_pll_id_to_tc_port(id)); } static bool icl_pll_get_hw_state(struct drm_i915_private *dev_priv, @@ -2954,54 +2971,51 @@ static bool icl_pll_get_hw_state(struct drm_i915_private *dev_priv, struct intel_dpll_hw_state *hw_state) { const enum intel_dpll_id id = pll->info->id; - uint32_t val; - enum port port; + intel_wakeref_t wakeref; bool ret = false; + u32 val; - if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_PLLS)) + wakeref = intel_display_power_get_if_enabled(dev_priv, + POWER_DOMAIN_PLLS); + if (!wakeref) return false; val = I915_READ(icl_pll_id_to_enable_reg(id)); if (!(val & PLL_ENABLE)) goto out; - switch (id) { - case DPLL_ID_ICL_DPLL0: - case DPLL_ID_ICL_DPLL1: - case DPLL_ID_ICL_TBTPLL: + if (intel_dpll_is_combophy(id) || + id == DPLL_ID_ICL_TBTPLL) { hw_state->cfgcr0 = I915_READ(ICL_DPLL_CFGCR0(id)); hw_state->cfgcr1 = I915_READ(ICL_DPLL_CFGCR1(id)); - break; - case DPLL_ID_ICL_MGPLL1: - case DPLL_ID_ICL_MGPLL2: - case DPLL_ID_ICL_MGPLL3: - case DPLL_ID_ICL_MGPLL4: - port = icl_mg_pll_id_to_port(id); - hw_state->mg_refclkin_ctl = I915_READ(MG_REFCLKIN_CTL(port)); + } else { + enum tc_port tc_port = icl_pll_id_to_tc_port(id); + + hw_state->mg_refclkin_ctl = I915_READ(MG_REFCLKIN_CTL(tc_port)); hw_state->mg_refclkin_ctl &= MG_REFCLKIN_CTL_OD_2_MUX_MASK; hw_state->mg_clktop2_coreclkctl1 = - I915_READ(MG_CLKTOP2_CORECLKCTL1(port)); + I915_READ(MG_CLKTOP2_CORECLKCTL1(tc_port)); hw_state->mg_clktop2_coreclkctl1 &= MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO_MASK; hw_state->mg_clktop2_hsclkctl = - I915_READ(MG_CLKTOP2_HSCLKCTL(port)); + I915_READ(MG_CLKTOP2_HSCLKCTL(tc_port)); hw_state->mg_clktop2_hsclkctl &= MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL_MASK | MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL_MASK | MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_MASK | MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK; - hw_state->mg_pll_div0 = I915_READ(MG_PLL_DIV0(port)); - hw_state->mg_pll_div1 = I915_READ(MG_PLL_DIV1(port)); - hw_state->mg_pll_lf = I915_READ(MG_PLL_LF(port)); - hw_state->mg_pll_frac_lock = I915_READ(MG_PLL_FRAC_LOCK(port)); - hw_state->mg_pll_ssc = I915_READ(MG_PLL_SSC(port)); + hw_state->mg_pll_div0 = I915_READ(MG_PLL_DIV0(tc_port)); + hw_state->mg_pll_div1 = I915_READ(MG_PLL_DIV1(tc_port)); + hw_state->mg_pll_lf = I915_READ(MG_PLL_LF(tc_port)); + hw_state->mg_pll_frac_lock = I915_READ(MG_PLL_FRAC_LOCK(tc_port)); + hw_state->mg_pll_ssc = I915_READ(MG_PLL_SSC(tc_port)); - hw_state->mg_pll_bias = I915_READ(MG_PLL_BIAS(port)); + hw_state->mg_pll_bias = I915_READ(MG_PLL_BIAS(tc_port)); hw_state->mg_pll_tdc_coldst_bias = - I915_READ(MG_PLL_TDC_COLDST_BIAS(port)); + I915_READ(MG_PLL_TDC_COLDST_BIAS(tc_port)); if (dev_priv->cdclk.hw.ref == 38400) { hw_state->mg_pll_tdc_coldst_bias_mask = MG_PLL_TDC_COLDST_COLDSTART; @@ -3013,14 +3027,11 @@ static bool icl_pll_get_hw_state(struct drm_i915_private *dev_priv, hw_state->mg_pll_tdc_coldst_bias &= hw_state->mg_pll_tdc_coldst_bias_mask; hw_state->mg_pll_bias &= hw_state->mg_pll_bias_mask; - break; - default: - MISSING_CASE(id); } ret = true; out: - intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS); + intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS, wakeref); return ret; } @@ -3039,7 +3050,7 @@ static void icl_mg_pll_write(struct drm_i915_private *dev_priv, struct intel_shared_dpll *pll) { struct intel_dpll_hw_state *hw_state = &pll->state.hw_state; - enum port port = icl_mg_pll_id_to_port(pll->info->id); + enum tc_port tc_port = icl_pll_id_to_tc_port(pll->info->id); u32 val; /* @@ -3048,41 +3059,41 @@ static void icl_mg_pll_write(struct drm_i915_private *dev_priv, * during the calc/readout phase if the mask depends on some other HW * state like refclk, see icl_calc_mg_pll_state(). */ - val = I915_READ(MG_REFCLKIN_CTL(port)); + val = I915_READ(MG_REFCLKIN_CTL(tc_port)); val &= ~MG_REFCLKIN_CTL_OD_2_MUX_MASK; val |= hw_state->mg_refclkin_ctl; - I915_WRITE(MG_REFCLKIN_CTL(port), val); + I915_WRITE(MG_REFCLKIN_CTL(tc_port), val); - val = I915_READ(MG_CLKTOP2_CORECLKCTL1(port)); + val = I915_READ(MG_CLKTOP2_CORECLKCTL1(tc_port)); val &= ~MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO_MASK; val |= hw_state->mg_clktop2_coreclkctl1; - I915_WRITE(MG_CLKTOP2_CORECLKCTL1(port), val); + I915_WRITE(MG_CLKTOP2_CORECLKCTL1(tc_port), val); - val = I915_READ(MG_CLKTOP2_HSCLKCTL(port)); + val = I915_READ(MG_CLKTOP2_HSCLKCTL(tc_port)); val &= ~(MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL_MASK | MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL_MASK | MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_MASK | MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK); val |= hw_state->mg_clktop2_hsclkctl; - I915_WRITE(MG_CLKTOP2_HSCLKCTL(port), val); + I915_WRITE(MG_CLKTOP2_HSCLKCTL(tc_port), val); - I915_WRITE(MG_PLL_DIV0(port), hw_state->mg_pll_div0); - I915_WRITE(MG_PLL_DIV1(port), hw_state->mg_pll_div1); - I915_WRITE(MG_PLL_LF(port), hw_state->mg_pll_lf); - I915_WRITE(MG_PLL_FRAC_LOCK(port), hw_state->mg_pll_frac_lock); - I915_WRITE(MG_PLL_SSC(port), hw_state->mg_pll_ssc); + I915_WRITE(MG_PLL_DIV0(tc_port), hw_state->mg_pll_div0); + I915_WRITE(MG_PLL_DIV1(tc_port), hw_state->mg_pll_div1); + I915_WRITE(MG_PLL_LF(tc_port), hw_state->mg_pll_lf); + I915_WRITE(MG_PLL_FRAC_LOCK(tc_port), hw_state->mg_pll_frac_lock); + I915_WRITE(MG_PLL_SSC(tc_port), hw_state->mg_pll_ssc); - val = I915_READ(MG_PLL_BIAS(port)); + val = I915_READ(MG_PLL_BIAS(tc_port)); val &= ~hw_state->mg_pll_bias_mask; val |= hw_state->mg_pll_bias; - I915_WRITE(MG_PLL_BIAS(port), val); + I915_WRITE(MG_PLL_BIAS(tc_port), val); - val = I915_READ(MG_PLL_TDC_COLDST_BIAS(port)); + val = I915_READ(MG_PLL_TDC_COLDST_BIAS(tc_port)); val &= ~hw_state->mg_pll_tdc_coldst_bias_mask; val |= hw_state->mg_pll_tdc_coldst_bias; - I915_WRITE(MG_PLL_TDC_COLDST_BIAS(port), val); + I915_WRITE(MG_PLL_TDC_COLDST_BIAS(tc_port), val); - POSTING_READ(MG_PLL_TDC_COLDST_BIAS(port)); + POSTING_READ(MG_PLL_TDC_COLDST_BIAS(tc_port)); } static void icl_pll_enable(struct drm_i915_private *dev_priv, @@ -3090,7 +3101,7 @@ static void icl_pll_enable(struct drm_i915_private *dev_priv, { const enum intel_dpll_id id = pll->info->id; i915_reg_t enable_reg = icl_pll_id_to_enable_reg(id); - uint32_t val; + u32 val; val = I915_READ(enable_reg); val |= PLL_POWER_ENABLE; @@ -3104,21 +3115,10 @@ static void icl_pll_enable(struct drm_i915_private *dev_priv, PLL_POWER_STATE, 1)) DRM_ERROR("PLL %d Power not enabled\n", id); - switch (id) { - case DPLL_ID_ICL_DPLL0: - case DPLL_ID_ICL_DPLL1: - case DPLL_ID_ICL_TBTPLL: + if (intel_dpll_is_combophy(id) || id == DPLL_ID_ICL_TBTPLL) icl_dpll_write(dev_priv, pll); - break; - case DPLL_ID_ICL_MGPLL1: - case DPLL_ID_ICL_MGPLL2: - case DPLL_ID_ICL_MGPLL3: - case DPLL_ID_ICL_MGPLL4: + else icl_mg_pll_write(dev_priv, pll); - break; - default: - MISSING_CASE(id); - } /* * DVFS pre sequence would be here, but in our driver the cdclk code @@ -3142,7 +3142,7 @@ static void icl_pll_disable(struct drm_i915_private *dev_priv, { const enum intel_dpll_id id = pll->info->id; i915_reg_t enable_reg = icl_pll_id_to_enable_reg(id); - uint32_t val; + u32 val; /* The first steps are done by intel_ddi_post_disable(). */ diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.h b/drivers/gpu/drm/i915/intel_dpll_mgr.h index bf0de8a4dc63..40e8391a92f2 100644 --- a/drivers/gpu/drm/i915/intel_dpll_mgr.h +++ b/drivers/gpu/drm/i915/intel_dpll_mgr.h @@ -138,14 +138,14 @@ enum intel_dpll_id { struct intel_dpll_hw_state { /* i9xx, pch plls */ - uint32_t dpll; - uint32_t dpll_md; - uint32_t fp0; - uint32_t fp1; + u32 dpll; + u32 dpll_md; + u32 fp0; + u32 fp1; /* hsw, bdw */ - uint32_t wrpll; - uint32_t spll; + u32 wrpll; + u32 spll; /* skl */ /* @@ -154,34 +154,33 @@ struct intel_dpll_hw_state { * the register. This allows us to easily compare the state to share * the DPLL. */ - uint32_t ctrl1; + u32 ctrl1; /* HDMI only, 0 when used for DP */ - uint32_t cfgcr1, cfgcr2; + u32 cfgcr1, cfgcr2; /* cnl */ - uint32_t cfgcr0; + u32 cfgcr0; /* CNL also uses cfgcr1 */ /* bxt */ - uint32_t ebb0, ebb4, pll0, pll1, pll2, pll3, pll6, pll8, pll9, pll10, - pcsdw12; + u32 ebb0, ebb4, pll0, pll1, pll2, pll3, pll6, pll8, pll9, pll10, pcsdw12; /* * ICL uses the following, already defined: - * uint32_t cfgcr0, cfgcr1; - */ - uint32_t mg_refclkin_ctl; - uint32_t mg_clktop2_coreclkctl1; - uint32_t mg_clktop2_hsclkctl; - uint32_t mg_pll_div0; - uint32_t mg_pll_div1; - uint32_t mg_pll_lf; - uint32_t mg_pll_frac_lock; - uint32_t mg_pll_ssc; - uint32_t mg_pll_bias; - uint32_t mg_pll_tdc_coldst_bias; - uint32_t mg_pll_bias_mask; - uint32_t mg_pll_tdc_coldst_bias_mask; + * u32 cfgcr0, cfgcr1; + */ + u32 mg_refclkin_ctl; + u32 mg_clktop2_coreclkctl1; + u32 mg_clktop2_hsclkctl; + u32 mg_pll_div0; + u32 mg_pll_div1; + u32 mg_pll_lf; + u32 mg_pll_frac_lock; + u32 mg_pll_ssc; + u32 mg_pll_bias; + u32 mg_pll_tdc_coldst_bias; + u32 mg_pll_bias_mask; + u32 mg_pll_tdc_coldst_bias_mask; }; /** @@ -280,7 +279,7 @@ struct dpll_info { * Inform the state checker that the DPLL is kept enabled even if * not in use by any CRTC. */ - uint32_t flags; + u32 flags; }; /** @@ -334,16 +333,18 @@ struct intel_shared_dpll *intel_get_shared_dpll(struct intel_crtc *crtc, void intel_release_shared_dpll(struct intel_shared_dpll *dpll, struct intel_crtc *crtc, struct drm_atomic_state *state); -void intel_prepare_shared_dpll(struct intel_crtc *crtc); -void intel_enable_shared_dpll(struct intel_crtc *crtc); -void intel_disable_shared_dpll(struct intel_crtc *crtc); +void intel_prepare_shared_dpll(const struct intel_crtc_state *crtc_state); +void intel_enable_shared_dpll(const struct intel_crtc_state *crtc_state); +void intel_disable_shared_dpll(const struct intel_crtc_state *crtc_state); void intel_shared_dpll_swap_state(struct drm_atomic_state *state); void intel_shared_dpll_init(struct drm_device *dev); void intel_dpll_dump_hw_state(struct drm_i915_private *dev_priv, struct intel_dpll_hw_state *hw_state); int icl_calc_dp_combo_pll_link(struct drm_i915_private *dev_priv, - uint32_t pll_id); + u32 pll_id); int cnl_hdmi_pll_ref_clock(struct drm_i915_private *dev_priv); +enum intel_dpll_id icl_tc_port_to_pll_id(enum tc_port tc_port); +bool intel_dpll_is_combophy(enum intel_dpll_id id); #endif /* _INTEL_DPLL_MGR_H_ */ diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h index db6fa1d0cbda..d5660ac1b0d6 100644 --- a/drivers/gpu/drm/i915/intel_drv.h +++ b/drivers/gpu/drm/i915/intel_drv.h @@ -29,18 +29,22 @@ #include <linux/i2c.h> #include <linux/hdmi.h> #include <linux/sched/clock.h> +#include <linux/stackdepot.h> #include <drm/i915_drm.h> #include "i915_drv.h" #include <drm/drm_crtc.h> -#include <drm/drm_crtc_helper.h> #include <drm/drm_encoder.h> #include <drm/drm_fb_helper.h> #include <drm/drm_dp_dual_mode_helper.h> #include <drm/drm_dp_mst_helper.h> +#include <drm/drm_probe_helper.h> #include <drm/drm_rect.h> +#include <drm/drm_vblank.h> #include <drm/drm_atomic.h> #include <media/cec-notifier.h> +struct drm_printer; + /** * __wait_for - magic wait macro * @@ -209,6 +213,16 @@ struct intel_fbdev { unsigned long vma_flags; async_cookie_t cookie; int preferred_bpp; + + /* Whether or not fbdev hpd processing is temporarily suspended */ + bool hpd_suspended : 1; + /* Set when a hotplug was received while HPD processing was + * suspended + */ + bool hpd_waiting : 1; + + /* Protects hpd_suspended */ + struct mutex hpd_lock; }; struct intel_encoder { @@ -222,9 +236,9 @@ struct intel_encoder { enum intel_output_type (*compute_output_type)(struct intel_encoder *, struct intel_crtc_state *, struct drm_connector_state *); - bool (*compute_config)(struct intel_encoder *, - struct intel_crtc_state *, - struct drm_connector_state *); + int (*compute_config)(struct intel_encoder *, + struct intel_crtc_state *, + struct drm_connector_state *); void (*pre_pll_enable)(struct intel_encoder *, const struct intel_crtc_state *, const struct drm_connector_state *); @@ -243,6 +257,9 @@ struct intel_encoder { void (*post_pll_disable)(struct intel_encoder *, const struct intel_crtc_state *, const struct drm_connector_state *); + void (*update_pipe)(struct intel_encoder *, + const struct intel_crtc_state *, + const struct drm_connector_state *); /* Read out the current hw state of this connector, returning true if * the encoder is active. If the encoder is enabled it also set the pipe * it is connected to in the pipe parameter. */ @@ -253,10 +270,12 @@ struct intel_encoder { * be set correctly before calling this function. */ void (*get_config)(struct intel_encoder *, struct intel_crtc_state *pipe_config); - /* Returns a mask of power domains that need to be referenced as part - * of the hardware state readout code. */ - u64 (*get_power_domains)(struct intel_encoder *encoder, - struct intel_crtc_state *crtc_state); + /* + * Acquires the power domains needed for an active encoder during + * hardware state readout. + */ + void (*get_power_domains)(struct intel_encoder *encoder, + struct intel_crtc_state *crtc_state); /* * Called during system suspend after all pending requests for the * encoder are flushed (for example for DP AUX transactions) and @@ -294,13 +313,12 @@ struct intel_panel { /* Connector and platform specific backlight functions */ int (*setup)(struct intel_connector *connector, enum pipe pipe); - uint32_t (*get)(struct intel_connector *connector); - void (*set)(const struct drm_connector_state *conn_state, uint32_t level); + u32 (*get)(struct intel_connector *connector); + void (*set)(const struct drm_connector_state *conn_state, u32 level); void (*disable)(const struct drm_connector_state *conn_state); void (*enable)(const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state); - uint32_t (*hz_to_pwm)(struct intel_connector *connector, - uint32_t hz); + u32 (*hz_to_pwm)(struct intel_connector *connector, u32 hz); void (*power)(struct intel_connector *, bool enable); } backlight; }; @@ -381,6 +399,15 @@ struct intel_hdcp_shim { bool *hdcp_capable); }; +struct intel_hdcp { + const struct intel_hdcp_shim *shim; + /* Mutex for hdcp state of the connector */ + struct mutex mutex; + u64 value; + struct delayed_work check_work; + struct work_struct prop_work; +}; + struct intel_connector { struct drm_connector base; /* @@ -413,11 +440,7 @@ struct intel_connector { /* Work struct to schedule a uevent on link train failure */ struct work_struct modeset_retry_work; - const struct intel_hdcp_shim *hdcp_shim; - struct mutex hdcp_mutex; - uint64_t hdcp_value; /* protected by hdcp_mutex */ - struct delayed_work hdcp_check_work; - struct work_struct hdcp_prop_work; + struct intel_hdcp hdcp; }; struct intel_digital_connector_state { @@ -539,6 +562,26 @@ struct intel_plane_state { */ int scaler_id; + /* + * linked_plane: + * + * ICL planar formats require 2 planes that are updated as pairs. + * This member is used to make sure the other plane is also updated + * when required, and for update_slave() to find the correct + * plane_state to pass as argument. + */ + struct intel_plane *linked_plane; + + /* + * slave: + * If set don't update use the linked plane's state for updating + * this plane during atomic commit with the update_slave() callback. + * + * It's also used by the watermark code to ignore wm calculations on + * this plane. They're calculated by the linked plane's wm code. + */ + u32 slave; + struct drm_intel_sprite_colorkey ckey; }; @@ -567,7 +610,7 @@ struct intel_initial_plane_config { struct intel_scaler { int in_use; - uint32_t mode; + u32 mode; }; struct intel_crtc_scaler_state { @@ -599,13 +642,15 @@ struct intel_crtc_scaler_state { }; /* drm_mode->private_flags */ -#define I915_MODE_FLAG_INHERITED 1 +#define I915_MODE_FLAG_INHERITED (1<<0) /* Flag to get scanline using frame time stamps */ #define I915_MODE_FLAG_GET_SCANLINE_FROM_TIMESTAMP (1<<1) +/* Flag to use the scanline counter instead of the pixel counter */ +#define I915_MODE_FLAG_USE_SCANLINE_COUNTER (1<<2) struct intel_pipe_wm { struct intel_wm_level wm[5]; - uint32_t linetime; + u32 linetime; bool fbc_wm_enabled; bool pipe_enabled; bool sprites_enabled; @@ -621,7 +666,7 @@ struct skl_plane_wm { struct skl_pipe_wm { struct skl_plane_wm planes[I915_MAX_PLANES]; - uint32_t linetime; + u32 linetime; }; enum vlv_wm_level { @@ -634,7 +679,7 @@ enum vlv_wm_level { struct vlv_wm_state { struct g4x_pipe_wm wm[NUM_VLV_WM_LEVELS]; struct g4x_sr_wm sr[NUM_VLV_WM_LEVELS]; - uint8_t num_levels; + u8 num_levels; bool cxsr; }; @@ -681,6 +726,8 @@ struct intel_crtc_wm_state { /* gen9+ only needs 1-step wm programming */ struct skl_pipe_wm optimal; struct skl_ddb_entry ddb; + struct skl_ddb_entry plane_ddb_y[I915_MAX_PLANES]; + struct skl_ddb_entry plane_ddb_uv[I915_MAX_PLANES]; } skl; struct { @@ -713,6 +760,13 @@ struct intel_crtc_wm_state { bool need_postvbl_update; }; +enum intel_output_format { + INTEL_OUTPUT_FORMAT_INVALID, + INTEL_OUTPUT_FORMAT_RGB, + INTEL_OUTPUT_FORMAT_YCBCR420, + INTEL_OUTPUT_FORMAT_YCBCR444, +}; + struct intel_crtc_state { struct drm_crtc_state base; @@ -838,13 +892,13 @@ struct intel_crtc_state { /* Used by SDVO (and if we ever fix it, HDMI). */ unsigned pixel_multiplier; - uint8_t lane_count; + u8 lane_count; /* * Used by platforms having DP/HDMI PHY with programmable lane * latency optimization. */ - uint8_t lane_lat_optim_mask; + u8 lane_lat_optim_mask; /* minimum acceptable voltage level */ u8 min_voltage_level; @@ -888,20 +942,38 @@ struct intel_crtc_state { struct intel_crtc_wm_state wm; /* Gamma mode programmed on the pipe */ - uint32_t gamma_mode; + u32 gamma_mode; /* bitmask of visible planes (enum plane_id) */ u8 active_planes; u8 nv12_planes; + /* bitmask of planes that will be updated during the commit */ + u8 update_planes; + /* HDMI scrambling status */ bool hdmi_scrambling; /* HDMI High TMDS char rate ratio */ bool hdmi_high_tmds_clock_ratio; - /* output format is YCBCR 4:2:0 */ - bool ycbcr420; + /* Output format RGB/YCBCR etc */ + enum intel_output_format output_format; + + /* Output down scaling is done in LSPCON device */ + bool lspcon_downsampling; + + /* Display Stream compression state */ + struct { + bool compression_enable; + bool dsc_split; + u16 compressed_bpp; + u8 slice_count; + } dsc_params; + struct drm_dsc_config dp_dsc_cfg; + + /* Forward Error correction State */ + bool fec_enable; }; struct intel_crtc { @@ -956,7 +1028,7 @@ struct intel_plane { enum pipe pipe; bool has_fbc; bool has_ccs; - uint32_t frontbuffer_bit; + u32 frontbuffer_bit; struct { u32 base, cntl, size; @@ -974,8 +1046,11 @@ struct intel_plane { void (*update_plane)(struct intel_plane *plane, const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state); + void (*update_slave)(struct intel_plane *plane, + const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state); void (*disable_plane)(struct intel_plane *plane, - struct intel_crtc *crtc); + const struct intel_crtc_state *crtc_state); bool (*get_hw_state)(struct intel_plane *plane, enum pipe *pipe); int (*check_plane)(struct intel_crtc_state *crtc_state, struct intel_plane_state *plane_state); @@ -1019,7 +1094,6 @@ struct intel_hdmi { } dp_dual_mode; bool has_hdmi_sink; bool has_audio; - bool rgb_quant_range_selectable; struct intel_connector *attached_connector; struct cec_notifier *cec_notifier; }; @@ -1049,9 +1123,9 @@ enum link_m_n_set { struct intel_dp_compliance_data { unsigned long edid; - uint8_t video_pattern; - uint16_t hdisplay, vdisplay; - uint8_t bpc; + u8 video_pattern; + u16 hdisplay, vdisplay; + u8 bpc; }; struct intel_dp_compliance { @@ -1064,20 +1138,20 @@ struct intel_dp_compliance { struct intel_dp { i915_reg_t output_reg; - uint32_t DP; + u32 DP; int link_rate; - uint8_t lane_count; - uint8_t sink_count; + u8 lane_count; + u8 sink_count; bool link_mst; bool link_trained; bool has_audio; - bool detect_done; bool reset_link_params; - enum aux_ch aux_ch; - uint8_t dpcd[DP_RECEIVER_CAP_SIZE]; - uint8_t psr_dpcd[EDP_PSR_RECEIVER_CAP_SIZE]; - uint8_t downstream_ports[DP_MAX_DOWNSTREAM_PORTS]; - uint8_t edp_dpcd[EDP_DISPLAY_CTL_CAP_SIZE]; + u8 dpcd[DP_RECEIVER_CAP_SIZE]; + u8 psr_dpcd[EDP_PSR_RECEIVER_CAP_SIZE]; + u8 downstream_ports[DP_MAX_DOWNSTREAM_PORTS]; + u8 edp_dpcd[EDP_DISPLAY_CTL_CAP_SIZE]; + u8 dsc_dpcd[DP_DSC_RECEIVER_CAP_SIZE]; + u8 fec_capable; /* source rates */ int num_source_rates; const int *source_rates; @@ -1095,8 +1169,7 @@ struct intel_dp { /* sink or branch descriptor */ struct drm_dp_desc desc; struct drm_dp_aux aux; - enum intel_display_power_domain aux_power_domain; - uint8_t train_set[4]; + u8 train_set[4]; int panel_power_up_delay; int panel_power_down_delay; int panel_power_cycle_delay; @@ -1138,14 +1211,13 @@ struct intel_dp { struct intel_dp_mst_encoder *mst_encoders[I915_MAX_PIPES]; struct drm_dp_mst_topology_mgr mst_mgr; - uint32_t (*get_aux_clock_divider)(struct intel_dp *dp, int index); + u32 (*get_aux_clock_divider)(struct intel_dp *dp, int index); /* * This function returns the value we have to program the AUX_CTL * register with to kick off an AUX transaction. */ - uint32_t (*get_aux_send_ctl)(struct intel_dp *dp, - int send_bytes, - uint32_t aux_clock_divider); + u32 (*get_aux_send_ctl)(struct intel_dp *dp, int send_bytes, + u32 aux_clock_divider); i915_reg_t (*aux_ch_ctl_reg)(struct intel_dp *dp); i915_reg_t (*aux_ch_data_reg)(struct intel_dp *dp, int index); @@ -1155,11 +1227,20 @@ struct intel_dp { /* Displayport compliance testing */ struct intel_dp_compliance compliance; + + /* Display stream compression testing */ + bool force_dsc_en; +}; + +enum lspcon_vendor { + LSPCON_VENDOR_MCA, + LSPCON_VENDOR_PARADE }; struct intel_lspcon { bool active; enum drm_lspcon_mode mode; + enum lspcon_vendor vendor; }; struct intel_digital_port { @@ -1170,19 +1251,22 @@ struct intel_digital_port { struct intel_lspcon lspcon; enum irqreturn (*hpd_pulse)(struct intel_digital_port *, bool); bool release_cl2_override; - uint8_t max_lanes; + u8 max_lanes; + /* Used for DP and ICL+ TypeC/DP and TypeC/HDMI ports. */ + enum aux_ch aux_ch; enum intel_display_power_domain ddi_io_power_domain; + bool tc_legacy_port:1; enum tc_port_type tc_type; - void (*write_infoframe)(struct drm_encoder *encoder, + void (*write_infoframe)(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, unsigned int type, const void *frame, ssize_t len); - void (*set_infoframes)(struct drm_encoder *encoder, + void (*set_infoframes)(struct intel_encoder *encoder, bool enable, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state); - bool (*infoframe_enabled)(struct drm_encoder *encoder, + bool (*infoframe_enabled)(struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config); }; @@ -1282,6 +1366,12 @@ enc_to_dig_port(struct drm_encoder *encoder) return NULL; } +static inline struct intel_digital_port * +conn_to_dig_port(struct intel_connector *connector) +{ + return enc_to_dig_port(&intel_attached_encoder(&connector->base)->base); +} + static inline struct intel_dp_mst_encoder * enc_to_mst(struct drm_encoder *encoder) { @@ -1307,6 +1397,12 @@ static inline bool intel_encoder_is_dp(struct intel_encoder *encoder) } } +static inline struct intel_lspcon * +enc_to_intel_lspcon(struct drm_encoder *encoder) +{ + return &enc_to_dig_port(encoder)->lspcon; +} + static inline struct intel_digital_port * dp_to_dig_port(struct intel_dp *intel_dp) { @@ -1332,6 +1428,27 @@ hdmi_to_dig_port(struct intel_hdmi *intel_hdmi) } static inline struct intel_plane_state * +intel_atomic_get_plane_state(struct intel_atomic_state *state, + struct intel_plane *plane) +{ + struct drm_plane_state *ret = + drm_atomic_get_plane_state(&state->base, &plane->base); + + if (IS_ERR(ret)) + return ERR_CAST(ret); + + return to_intel_plane_state(ret); +} + +static inline struct intel_plane_state * +intel_atomic_get_old_plane_state(struct intel_atomic_state *state, + struct intel_plane *plane) +{ + return to_intel_plane_state(drm_atomic_get_old_plane_state(&state->base, + &plane->base)); +} + +static inline struct intel_plane_state * intel_atomic_get_new_plane_state(struct intel_atomic_state *state, struct intel_plane *plane) { @@ -1369,8 +1486,8 @@ void intel_check_cpu_fifo_underruns(struct drm_i915_private *dev_priv); void intel_check_pch_fifo_underruns(struct drm_i915_private *dev_priv); /* i915_irq.c */ -void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask); -void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask); +void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, u32 mask); +void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, u32 mask); void gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask); void gen6_unmask_pm_irq(struct drm_i915_private *dev_priv, u32 mask); void gen11_reset_rps_interrupts(struct drm_i915_private *dev_priv); @@ -1433,18 +1550,15 @@ void intel_ddi_set_vc_payload_alloc(const struct intel_crtc_state *crtc_state, void intel_ddi_compute_min_voltage_level(struct drm_i915_private *dev_priv, struct intel_crtc_state *crtc_state); u32 bxt_signal_levels(struct intel_dp *intel_dp); -uint32_t ddi_signal_levels(struct intel_dp *intel_dp); +u32 ddi_signal_levels(struct intel_dp *intel_dp); u8 intel_ddi_dp_voltage_max(struct intel_encoder *encoder); u8 intel_ddi_dp_pre_emphasis_max(struct intel_encoder *encoder, u8 voltage_swing); int intel_ddi_toggle_hdcp_signalling(struct intel_encoder *intel_encoder, bool enable); -void icl_map_plls_to_ports(struct drm_crtc *crtc, - struct intel_crtc_state *crtc_state, - struct drm_atomic_state *old_state); -void icl_unmap_plls_to_ports(struct drm_crtc *crtc, - struct intel_crtc_state *crtc_state, - struct drm_atomic_state *old_state); +void icl_sanitize_encoder_pll_mapping(struct intel_encoder *encoder); +int cnl_calc_wrpll_link(struct drm_i915_private *dev_priv, + enum intel_dpll_id pll_id); unsigned int intel_fb_align_height(const struct drm_framebuffer *fb, int color_plane, unsigned int height); @@ -1489,7 +1603,6 @@ void intel_dump_cdclk_state(const struct intel_cdclk_state *cdclk_state, void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe); void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe); enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc); -void intel_update_rawclk(struct drm_i915_private *dev_priv); int vlv_get_hpll_vco(struct drm_i915_private *dev_priv); int vlv_get_cck_clock(struct drm_i915_private *dev_priv, const char *name, u32 reg, int ref_freq); @@ -1510,20 +1623,12 @@ void intel_mark_idle(struct drm_i915_private *dev_priv); int intel_display_suspend(struct drm_device *dev); void intel_pps_unlock_regs_wa(struct drm_i915_private *dev_priv); void intel_encoder_destroy(struct drm_encoder *encoder); -int intel_connector_init(struct intel_connector *); -struct intel_connector *intel_connector_alloc(void); -void intel_connector_free(struct intel_connector *connector); -bool intel_connector_get_hw_state(struct intel_connector *connector); -void intel_connector_attach_encoder(struct intel_connector *connector, - struct intel_encoder *encoder); struct drm_display_mode * intel_encoder_current_mode(struct intel_encoder *encoder); bool intel_port_is_combophy(struct drm_i915_private *dev_priv, enum port port); bool intel_port_is_tc(struct drm_i915_private *dev_priv, enum port port); enum tc_port intel_port_to_tc(struct drm_i915_private *dev_priv, enum port port); - -enum pipe intel_get_pipe_from_connector(struct intel_connector *connector); int intel_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv, @@ -1585,11 +1690,11 @@ void intel_cleanup_plane_fb(struct drm_plane *plane, int intel_plane_atomic_get_property(struct drm_plane *plane, const struct drm_plane_state *state, struct drm_property *property, - uint64_t *val); + u64 *val); int intel_plane_atomic_set_property(struct drm_plane *plane, struct drm_plane_state *state, struct drm_property *property, - uint64_t val); + u64 val); int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state, struct drm_crtc_state *crtc_state, const struct intel_plane_state *old_plane_state, @@ -1629,9 +1734,11 @@ void bxt_enable_dc9(struct drm_i915_private *dev_priv); void bxt_disable_dc9(struct drm_i915_private *dev_priv); void gen9_enable_dc5(struct drm_i915_private *dev_priv); unsigned int skl_cdclk_get_vco(unsigned int freq); +void skl_enable_dc6(struct drm_i915_private *dev_priv); void intel_dp_get_m_n(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config); -void intel_dp_set_m_n(struct intel_crtc *crtc, enum link_m_n_set m_n); +void intel_dp_set_m_n(const struct intel_crtc_state *crtc_state, + enum link_m_n_set m_n); int intel_dotclock_calculate(int link_freq, const struct intel_link_m_n *m_n); bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, int target_clock, struct dpll *best_clock); @@ -1642,6 +1749,8 @@ bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state); void hsw_enable_ips(const struct intel_crtc_state *crtc_state); void hsw_disable_ips(const struct intel_crtc_state *crtc_state); enum intel_display_power_domain intel_port_to_power_domain(enum port port); +enum intel_display_power_domain +intel_aux_power_domain(struct intel_digital_port *dig_port); void intel_mode_from_pipe_config(struct drm_display_mode *mode, struct intel_crtc_state *pipe_config); void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc, @@ -1659,9 +1768,10 @@ static inline u32 intel_plane_ggtt_offset(const struct intel_plane_state *state) u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state); +u32 glk_plane_color_ctl_crtc(const struct intel_crtc_state *crtc_state); u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state); -u32 glk_color_ctl(const struct intel_plane_state *plane_state); +u32 skl_plane_ctl_crtc(const struct intel_crtc_state *crtc_state); u32 skl_plane_stride(const struct intel_plane_state *plane_state, int plane); int skl_check_plane_surface(struct intel_plane_state *plane_state); @@ -1671,6 +1781,24 @@ unsigned int i9xx_plane_max_stride(struct intel_plane *plane, u32 pixel_format, u64 modifier, unsigned int rotation); +/* intel_connector.c */ +int intel_connector_init(struct intel_connector *connector); +struct intel_connector *intel_connector_alloc(void); +void intel_connector_free(struct intel_connector *connector); +void intel_connector_destroy(struct drm_connector *connector); +int intel_connector_register(struct drm_connector *connector); +void intel_connector_unregister(struct drm_connector *connector); +void intel_connector_attach_encoder(struct intel_connector *connector, + struct intel_encoder *encoder); +bool intel_connector_get_hw_state(struct intel_connector *connector); +enum pipe intel_connector_get_pipe(struct intel_connector *connector); +int intel_connector_update_modes(struct drm_connector *connector, + struct edid *edid); +int intel_ddc_get_modes(struct drm_connector *c, struct i2c_adapter *adapter); +void intel_attach_force_audio_property(struct drm_connector *connector); +void intel_attach_broadcast_rgb_property(struct drm_connector *connector); +void intel_attach_aspect_ratio_property(struct drm_connector *connector); + /* intel_csr.c */ void intel_csr_ucode_init(struct drm_i915_private *); void intel_csr_load_program(struct drm_i915_private *); @@ -1687,21 +1815,24 @@ bool intel_dp_init(struct drm_i915_private *dev_priv, i915_reg_t output_reg, bool intel_dp_init_connector(struct intel_digital_port *intel_dig_port, struct intel_connector *intel_connector); void intel_dp_set_link_params(struct intel_dp *intel_dp, - int link_rate, uint8_t lane_count, + int link_rate, u8 lane_count, bool link_mst); int intel_dp_get_link_train_fallback_values(struct intel_dp *intel_dp, - int link_rate, uint8_t lane_count); + int link_rate, u8 lane_count); void intel_dp_start_link_train(struct intel_dp *intel_dp); void intel_dp_stop_link_train(struct intel_dp *intel_dp); int intel_dp_retrain_link(struct intel_encoder *encoder, struct drm_modeset_acquire_ctx *ctx); void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode); +void intel_dp_sink_set_decompression_state(struct intel_dp *intel_dp, + const struct intel_crtc_state *crtc_state, + bool enable); void intel_dp_encoder_reset(struct drm_encoder *encoder); void intel_dp_encoder_suspend(struct intel_encoder *intel_encoder); -void intel_dp_encoder_destroy(struct drm_encoder *encoder); -bool intel_dp_compute_config(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config, - struct drm_connector_state *conn_state); +void intel_dp_encoder_flush_work(struct drm_encoder *encoder); +int intel_dp_compute_config(struct intel_encoder *encoder, + struct intel_crtc_state *pipe_config, + struct drm_connector_state *conn_state); bool intel_dp_is_edp(struct intel_dp *intel_dp); bool intel_dp_is_port_edp(struct drm_i915_private *dev_priv, enum port port); enum irqreturn intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, @@ -1719,7 +1850,7 @@ int intel_dp_max_lane_count(struct intel_dp *intel_dp); int intel_dp_rate_select(struct intel_dp *intel_dp, int rate); void intel_dp_hot_plug(struct intel_encoder *intel_encoder); void intel_power_sequencer_reset(struct drm_i915_private *dev_priv); -uint32_t intel_dp_pack_aux(const uint8_t *src, int src_bytes); +u32 intel_dp_pack_aux(const u8 *src, int src_bytes); void intel_plane_destroy(struct drm_plane *plane); void intel_edp_drrs_enable(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state); @@ -1729,26 +1860,33 @@ void intel_edp_drrs_invalidate(struct drm_i915_private *dev_priv, unsigned int frontbuffer_bits); void intel_edp_drrs_flush(struct drm_i915_private *dev_priv, unsigned int frontbuffer_bits); -void icl_program_mg_dp_mode(struct intel_dp *intel_dp); -void icl_enable_phy_clock_gating(struct intel_digital_port *dig_port); -void icl_disable_phy_clock_gating(struct intel_digital_port *dig_port); void intel_dp_program_link_training_pattern(struct intel_dp *intel_dp, - uint8_t dp_train_pat); + u8 dp_train_pat); void intel_dp_set_signal_levels(struct intel_dp *intel_dp); void intel_dp_set_idle_link_train(struct intel_dp *intel_dp); -uint8_t +u8 intel_dp_voltage_max(struct intel_dp *intel_dp); -uint8_t -intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing); +u8 +intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, u8 voltage_swing); void intel_dp_compute_rate(struct intel_dp *intel_dp, int port_clock, - uint8_t *link_bw, uint8_t *rate_select); + u8 *link_bw, u8 *rate_select); bool intel_dp_source_supports_hbr2(struct intel_dp *intel_dp); bool intel_dp_source_supports_hbr3(struct intel_dp *intel_dp); bool -intel_dp_get_link_status(struct intel_dp *intel_dp, uint8_t link_status[DP_LINK_STATUS_SIZE]); +intel_dp_get_link_status(struct intel_dp *intel_dp, u8 link_status[DP_LINK_STATUS_SIZE]); +u16 intel_dp_dsc_get_output_bpp(int link_clock, u8 lane_count, + int mode_clock, int mode_hdisplay); +u8 intel_dp_dsc_get_slice_count(struct intel_dp *intel_dp, int mode_clock, + int mode_hdisplay); + +/* intel_vdsc.c */ +int intel_dp_compute_dsc_params(struct intel_dp *intel_dp, + struct intel_crtc_state *pipe_config); +enum intel_display_power_domain +intel_dsc_power_domain(const struct intel_crtc_state *crtc_state); static inline unsigned int intel_dp_unused_lane_mask(int lane_count) { @@ -1759,6 +1897,8 @@ bool intel_dp_read_dpcd(struct intel_dp *intel_dp); int intel_dp_link_required(int pixel_clock, int bpp); int intel_dp_max_data_rate(int max_link_clock, int max_lanes); bool intel_digital_port_connected(struct intel_encoder *encoder); +void icl_tc_phy_disconnect(struct drm_i915_private *dev_priv, + struct intel_digital_port *dig_port); /* intel_dp_aux_backlight.c */ int intel_dp_aux_init_backlight_funcs(struct intel_connector *intel_connector); @@ -1769,6 +1909,9 @@ void intel_dp_mst_encoder_cleanup(struct intel_digital_port *intel_dig_port); /* vlv_dsi.c */ void vlv_dsi_init(struct drm_i915_private *dev_priv); +/* icl_dsi.c */ +void icl_dsi_init(struct drm_i915_private *dev_priv); + /* intel_dsi_dcs_backlight.c */ int intel_dsi_dcs_init_backlight_funcs(struct intel_connector *intel_connector); @@ -1849,9 +1992,9 @@ void intel_hdmi_init(struct drm_i915_private *dev_priv, i915_reg_t hdmi_reg, void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port, struct intel_connector *intel_connector); struct intel_hdmi *enc_to_intel_hdmi(struct drm_encoder *encoder); -bool intel_hdmi_compute_config(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config, - struct drm_connector_state *conn_state); +int intel_hdmi_compute_config(struct intel_encoder *encoder, + struct intel_crtc_state *pipe_config, + struct drm_connector_state *conn_state); bool intel_hdmi_handle_sink_scrambling(struct intel_encoder *encoder, struct drm_connector *connector, bool high_tmds_clock_ratio, @@ -1859,7 +2002,6 @@ bool intel_hdmi_handle_sink_scrambling(struct intel_encoder *encoder, void intel_dp_dual_mode_set_tmds_output(struct intel_hdmi *hdmi, bool enable); void intel_infoframe_init(struct intel_digital_port *intel_dig_port); - /* intel_lvds.c */ bool intel_lvds_port_enabled(struct drm_i915_private *dev_priv, i915_reg_t lvds_reg, enum pipe *pipe); @@ -1867,19 +2009,9 @@ void intel_lvds_init(struct drm_i915_private *dev_priv); struct intel_encoder *intel_get_lvds_encoder(struct drm_device *dev); bool intel_is_dual_link_lvds(struct drm_device *dev); - -/* intel_modes.c */ -int intel_connector_update_modes(struct drm_connector *connector, - struct edid *edid); -int intel_ddc_get_modes(struct drm_connector *c, struct i2c_adapter *adapter); -void intel_attach_force_audio_property(struct drm_connector *connector); -void intel_attach_broadcast_rgb_property(struct drm_connector *connector); -void intel_attach_aspect_ratio_property(struct drm_connector *connector); - - /* intel_overlay.c */ -void intel_setup_overlay(struct drm_i915_private *dev_priv); -void intel_cleanup_overlay(struct drm_i915_private *dev_priv); +void intel_overlay_setup(struct drm_i915_private *dev_priv); +void intel_overlay_cleanup(struct drm_i915_private *dev_priv); int intel_overlay_switch_off(struct intel_overlay *overlay); int intel_overlay_put_image_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); @@ -1907,8 +2039,10 @@ int intel_panel_setup_backlight(struct drm_connector *connector, enum pipe pipe); void intel_panel_enable_backlight(const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state); +void intel_panel_update_backlight(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state); void intel_panel_disable_backlight(const struct drm_connector_state *old_conn_state); -void intel_panel_destroy_backlight(struct drm_connector *connector); extern struct drm_display_mode *intel_find_panel_downclock( struct drm_i915_private *dev_priv, struct drm_display_mode *fixed_mode, @@ -1937,6 +2071,7 @@ int intel_hdcp_enable(struct intel_connector *connector); int intel_hdcp_disable(struct intel_connector *connector); int intel_hdcp_check_link(struct intel_connector *connector); bool is_hdcp_supported(struct drm_i915_private *dev_priv, enum port port); +bool intel_hdcp_capable(struct intel_connector *connector); /* intel_psr.c */ #define CAN_PSR(dev_priv) (HAS_PSR(dev_priv) && dev_priv->psr.sink_support) @@ -1962,12 +2097,19 @@ void intel_psr_irq_handler(struct drm_i915_private *dev_priv, u32 psr_iir); void intel_psr_short_pulse(struct intel_dp *intel_dp); int intel_psr_wait_for_idle(const struct intel_crtc_state *new_crtc_state, u32 *out_value); +bool intel_psr_enabled(struct intel_dp *intel_dp); + +/* intel_quirks.c */ +void intel_init_quirks(struct drm_i915_private *dev_priv); /* intel_runtime_pm.c */ +void intel_runtime_pm_init_early(struct drm_i915_private *dev_priv); int intel_power_domains_init(struct drm_i915_private *); void intel_power_domains_cleanup(struct drm_i915_private *dev_priv); void intel_power_domains_init_hw(struct drm_i915_private *dev_priv, bool resume); void intel_power_domains_fini_hw(struct drm_i915_private *dev_priv); +void icl_display_core_init(struct drm_i915_private *dev_priv, bool resume); +void icl_display_core_uninit(struct drm_i915_private *dev_priv); void intel_power_domains_enable(struct drm_i915_private *dev_priv); void intel_power_domains_disable(struct drm_i915_private *dev_priv); @@ -1984,6 +2126,7 @@ void bxt_display_core_init(struct drm_i915_private *dev_priv, bool resume); void bxt_display_core_uninit(struct drm_i915_private *dev_priv); void intel_runtime_pm_enable(struct drm_i915_private *dev_priv); void intel_runtime_pm_disable(struct drm_i915_private *dev_priv); +void intel_runtime_pm_cleanup(struct drm_i915_private *dev_priv); const char * intel_display_power_domain_str(enum intel_display_power_domain domain); @@ -1991,33 +2134,42 @@ bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain); bool __intel_display_power_is_enabled(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain); -void intel_display_power_get(struct drm_i915_private *dev_priv, - enum intel_display_power_domain domain); -bool intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv, +intel_wakeref_t intel_display_power_get(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain); +intel_wakeref_t +intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv, + enum intel_display_power_domain domain); +void intel_display_power_put_unchecked(struct drm_i915_private *dev_priv, + enum intel_display_power_domain domain); +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM) void intel_display_power_put(struct drm_i915_private *dev_priv, - enum intel_display_power_domain domain); + enum intel_display_power_domain domain, + intel_wakeref_t wakeref); +#else +#define intel_display_power_put(i915, domain, wakeref) \ + intel_display_power_put_unchecked(i915, domain) +#endif void icl_dbuf_slices_update(struct drm_i915_private *dev_priv, u8 req_slices); static inline void -assert_rpm_device_not_suspended(struct drm_i915_private *dev_priv) +assert_rpm_device_not_suspended(struct drm_i915_private *i915) { - WARN_ONCE(dev_priv->runtime_pm.suspended, + WARN_ONCE(i915->runtime_pm.suspended, "Device suspended during HW access\n"); } static inline void -assert_rpm_wakelock_held(struct drm_i915_private *dev_priv) +assert_rpm_wakelock_held(struct drm_i915_private *i915) { - assert_rpm_device_not_suspended(dev_priv); - WARN_ONCE(!atomic_read(&dev_priv->runtime_pm.wakeref_count), + assert_rpm_device_not_suspended(i915); + WARN_ONCE(!atomic_read(&i915->runtime_pm.wakeref_count), "RPM wakelock ref not held during HW access"); } /** * disable_rpm_wakeref_asserts - disable the RPM assert checks - * @dev_priv: i915 device instance + * @i915: i915 device instance * * This function disable asserts that check if we hold an RPM wakelock * reference, while keeping the device-not-suspended checks still enabled. @@ -2034,14 +2186,14 @@ assert_rpm_wakelock_held(struct drm_i915_private *dev_priv) * enable_rpm_wakeref_asserts(). */ static inline void -disable_rpm_wakeref_asserts(struct drm_i915_private *dev_priv) +disable_rpm_wakeref_asserts(struct drm_i915_private *i915) { - atomic_inc(&dev_priv->runtime_pm.wakeref_count); + atomic_inc(&i915->runtime_pm.wakeref_count); } /** * enable_rpm_wakeref_asserts - re-enable the RPM assert checks - * @dev_priv: i915 device instance + * @i915: i915 device instance * * This function re-enables the RPM assert checks after disabling them with * disable_rpm_wakeref_asserts. It's meant to be used only in special @@ -2051,15 +2203,39 @@ disable_rpm_wakeref_asserts(struct drm_i915_private *dev_priv) * disable_rpm_wakeref_asserts(). */ static inline void -enable_rpm_wakeref_asserts(struct drm_i915_private *dev_priv) +enable_rpm_wakeref_asserts(struct drm_i915_private *i915) { - atomic_dec(&dev_priv->runtime_pm.wakeref_count); + atomic_dec(&i915->runtime_pm.wakeref_count); } -void intel_runtime_pm_get(struct drm_i915_private *dev_priv); -bool intel_runtime_pm_get_if_in_use(struct drm_i915_private *dev_priv); -void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv); -void intel_runtime_pm_put(struct drm_i915_private *dev_priv); +intel_wakeref_t intel_runtime_pm_get(struct drm_i915_private *i915); +intel_wakeref_t intel_runtime_pm_get_if_in_use(struct drm_i915_private *i915); +intel_wakeref_t intel_runtime_pm_get_noresume(struct drm_i915_private *i915); + +#define with_intel_runtime_pm(i915, wf) \ + for ((wf) = intel_runtime_pm_get(i915); (wf); \ + intel_runtime_pm_put((i915), (wf)), (wf) = 0) + +#define with_intel_runtime_pm_if_in_use(i915, wf) \ + for ((wf) = intel_runtime_pm_get_if_in_use(i915); (wf); \ + intel_runtime_pm_put((i915), (wf)), (wf) = 0) + +void intel_runtime_pm_put_unchecked(struct drm_i915_private *i915); +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM) +void intel_runtime_pm_put(struct drm_i915_private *i915, intel_wakeref_t wref); +#else +#define intel_runtime_pm_put(i915, wref) intel_runtime_pm_put_unchecked(i915) +#endif + +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM) +void print_intel_runtime_pm_wakeref(struct drm_i915_private *i915, + struct drm_printer *p); +#else +static inline void print_intel_runtime_pm_wakeref(struct drm_i915_private *i915, + struct drm_printer *p) +{ +} +#endif void chv_phy_powergate_lanes(struct intel_encoder *encoder, bool override, unsigned int mask); @@ -2087,13 +2263,16 @@ void gen6_rps_busy(struct drm_i915_private *dev_priv); void gen6_rps_reset_ei(struct drm_i915_private *dev_priv); void gen6_rps_idle(struct drm_i915_private *dev_priv); void gen6_rps_boost(struct i915_request *rq, struct intel_rps_client *rps); -void g4x_wm_get_hw_state(struct drm_device *dev); -void vlv_wm_get_hw_state(struct drm_device *dev); -void ilk_wm_get_hw_state(struct drm_device *dev); -void skl_wm_get_hw_state(struct drm_device *dev); +void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv); +void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv); +void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv); +void skl_wm_get_hw_state(struct drm_i915_private *dev_priv); +void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc, + struct skl_ddb_entry *ddb_y, + struct skl_ddb_entry *ddb_uv); void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv, struct skl_ddb_allocation *ddb /* out */); -void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc, +void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc, struct skl_pipe_wm *out); void g4x_wm_sanitize(struct drm_i915_private *dev_priv); void vlv_wm_sanitize(struct drm_i915_private *dev_priv); @@ -2102,10 +2281,13 @@ int intel_enable_sagv(struct drm_i915_private *dev_priv); int intel_disable_sagv(struct drm_i915_private *dev_priv); bool skl_wm_level_equals(const struct skl_wm_level *l1, const struct skl_wm_level *l2); -bool skl_ddb_allocation_overlaps(struct drm_i915_private *dev_priv, - const struct skl_ddb_entry **entries, - const struct skl_ddb_entry *ddb, - int ignore); +bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb, + const struct skl_ddb_entry entries[], + int num_entries, int ignore_idx); +void skl_write_plane_wm(struct intel_plane *plane, + const struct intel_crtc_state *crtc_state); +void skl_write_cursor_wm(struct intel_plane *plane, + const struct intel_crtc_state *crtc_state); bool ilk_disable_lp_wm(struct drm_device *dev); int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc, struct intel_crtc_state *cstate); @@ -2128,23 +2310,29 @@ int intel_sprite_set_colorkey_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); void intel_pipe_update_start(const struct intel_crtc_state *new_crtc_state); void intel_pipe_update_end(struct intel_crtc_state *new_crtc_state); -void skl_update_plane(struct intel_plane *plane, - const struct intel_crtc_state *crtc_state, - const struct intel_plane_state *plane_state); -void skl_disable_plane(struct intel_plane *plane, struct intel_crtc *crtc); -bool skl_plane_get_hw_state(struct intel_plane *plane, enum pipe *pipe); -bool skl_plane_has_ccs(struct drm_i915_private *dev_priv, - enum pipe pipe, enum plane_id plane_id); -bool skl_plane_has_planar(struct drm_i915_private *dev_priv, - enum pipe pipe, enum plane_id plane_id); -unsigned int skl_plane_max_stride(struct intel_plane *plane, - u32 pixel_format, u64 modifier, - unsigned int rotation); -int skl_plane_check(struct intel_crtc_state *crtc_state, - struct intel_plane_state *plane_state); int intel_plane_check_stride(const struct intel_plane_state *plane_state); int intel_plane_check_src_coordinates(struct intel_plane_state *plane_state); int chv_plane_check_rotation(const struct intel_plane_state *plane_state); +struct intel_plane * +skl_universal_plane_create(struct drm_i915_private *dev_priv, + enum pipe pipe, enum plane_id plane_id); + +static inline bool icl_is_nv12_y_plane(enum plane_id id) +{ + /* Don't need to do a gen check, these planes are only available on gen11 */ + if (id == PLANE_SPRITE4 || id == PLANE_SPRITE5) + return true; + + return false; +} + +static inline bool icl_is_hdr_plane(struct intel_plane *plane) +{ + if (INTEL_GEN(to_i915(plane->base.dev)) < 11) + return false; + + return plane->id < PLANE_SPRITE2; +} /* intel_tv.c */ void intel_tv_init(struct drm_i915_private *dev_priv); @@ -2153,11 +2341,11 @@ void intel_tv_init(struct drm_i915_private *dev_priv); int intel_digital_connector_atomic_get_property(struct drm_connector *connector, const struct drm_connector_state *state, struct drm_property *property, - uint64_t *val); + u64 *val); int intel_digital_connector_atomic_set_property(struct drm_connector *connector, struct drm_connector_state *state, struct drm_property *property, - uint64_t val); + u64 val); int intel_digital_connector_atomic_check(struct drm_connector *conn, struct drm_connector_state *new_state); struct drm_connector_state * @@ -2186,26 +2374,43 @@ int intel_atomic_setup_scalers(struct drm_i915_private *dev_priv, struct intel_crtc_state *crtc_state); /* intel_atomic_plane.c */ -struct intel_plane_state *intel_create_plane_state(struct drm_plane *plane); +struct intel_plane *intel_plane_alloc(void); +void intel_plane_free(struct intel_plane *plane); struct drm_plane_state *intel_plane_duplicate_state(struct drm_plane *plane); void intel_plane_destroy_state(struct drm_plane *plane, struct drm_plane_state *state); extern const struct drm_plane_helper_funcs intel_plane_helper_funcs; +void skl_update_planes_on_crtc(struct intel_atomic_state *state, + struct intel_crtc *crtc); +void i9xx_update_planes_on_crtc(struct intel_atomic_state *state, + struct intel_crtc *crtc); int intel_plane_atomic_check_with_state(const struct intel_crtc_state *old_crtc_state, struct intel_crtc_state *crtc_state, const struct intel_plane_state *old_plane_state, struct intel_plane_state *intel_state); /* intel_color.c */ -void intel_color_init(struct drm_crtc *crtc); -int intel_color_check(struct drm_crtc *crtc, struct drm_crtc_state *state); -void intel_color_set_csc(struct drm_crtc_state *crtc_state); -void intel_color_load_luts(struct drm_crtc_state *crtc_state); +void intel_color_init(struct intel_crtc *crtc); +int intel_color_check(struct intel_crtc_state *crtc_state); +void intel_color_commit(const struct intel_crtc_state *crtc_state); +void intel_color_load_luts(const struct intel_crtc_state *crtc_state); /* intel_lspcon.c */ bool lspcon_init(struct intel_digital_port *intel_dig_port); void lspcon_resume(struct intel_lspcon *lspcon); void lspcon_wait_pcon_mode(struct intel_lspcon *lspcon); +void lspcon_write_infoframe(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + unsigned int type, + const void *buf, ssize_t len); +void lspcon_set_infoframes(struct intel_encoder *encoder, + bool enable, + const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state); +bool lspcon_infoframe_enabled(struct intel_encoder *encoder, + const struct intel_crtc_state *pipe_config); +void lspcon_ycbcr420_config(struct drm_connector *connector, + struct intel_crtc_state *crtc_state); /* intel_pipe_crc.c */ #ifdef CONFIG_DEBUG_FS diff --git a/drivers/gpu/drm/i915/intel_dsi.c b/drivers/gpu/drm/i915/intel_dsi.c new file mode 100644 index 000000000000..5fec02aceaed --- /dev/null +++ b/drivers/gpu/drm/i915/intel_dsi.c @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2018 Intel Corporation + */ + +#include <drm/drm_mipi_dsi.h> +#include "intel_dsi.h" + +int intel_dsi_bitrate(const struct intel_dsi *intel_dsi) +{ + int bpp = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format); + + if (WARN_ON(bpp < 0)) + bpp = 16; + + return intel_dsi->pclk * bpp / intel_dsi->lane_count; +} + +int intel_dsi_tlpx_ns(const struct intel_dsi *intel_dsi) +{ + switch (intel_dsi->escape_clk_div) { + default: + case 0: + return 50; + case 1: + return 100; + case 2: + return 200; + } +} + +int intel_dsi_get_modes(struct drm_connector *connector) +{ + struct intel_connector *intel_connector = to_intel_connector(connector); + struct drm_display_mode *mode; + + DRM_DEBUG_KMS("\n"); + + if (!intel_connector->panel.fixed_mode) { + DRM_DEBUG_KMS("no fixed mode\n"); + return 0; + } + + mode = drm_mode_duplicate(connector->dev, + intel_connector->panel.fixed_mode); + if (!mode) { + DRM_DEBUG_KMS("drm_mode_duplicate failed\n"); + return 0; + } + + drm_mode_probed_add(connector, mode); + return 1; +} + +enum drm_mode_status intel_dsi_mode_valid(struct drm_connector *connector, + struct drm_display_mode *mode) +{ + struct intel_connector *intel_connector = to_intel_connector(connector); + const struct drm_display_mode *fixed_mode = intel_connector->panel.fixed_mode; + int max_dotclk = to_i915(connector->dev)->max_dotclk_freq; + + DRM_DEBUG_KMS("\n"); + + if (mode->flags & DRM_MODE_FLAG_DBLSCAN) + return MODE_NO_DBLESCAN; + + if (fixed_mode) { + if (mode->hdisplay > fixed_mode->hdisplay) + return MODE_PANEL; + if (mode->vdisplay > fixed_mode->vdisplay) + return MODE_PANEL; + if (fixed_mode->clock > max_dotclk) + return MODE_CLOCK_HIGH; + } + + return MODE_OK; +} + +struct intel_dsi_host *intel_dsi_host_init(struct intel_dsi *intel_dsi, + const struct mipi_dsi_host_ops *funcs, + enum port port) +{ + struct intel_dsi_host *host; + struct mipi_dsi_device *device; + + host = kzalloc(sizeof(*host), GFP_KERNEL); + if (!host) + return NULL; + + host->base.ops = funcs; + host->intel_dsi = intel_dsi; + host->port = port; + + /* + * We should call mipi_dsi_host_register(&host->base) here, but we don't + * have a host->dev, and we don't have OF stuff either. So just use the + * dsi framework as a library and hope for the best. Create the dsi + * devices by ourselves here too. Need to be careful though, because we + * don't initialize any of the driver model devices here. + */ + device = kzalloc(sizeof(*device), GFP_KERNEL); + if (!device) { + kfree(host); + return NULL; + } + + device->host = &host->base; + host->device = device; + + return host; +} + +enum drm_panel_orientation +intel_dsi_get_panel_orientation(struct intel_connector *connector) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + enum drm_panel_orientation orientation; + + orientation = dev_priv->vbt.dsi.orientation; + if (orientation != DRM_MODE_PANEL_ORIENTATION_UNKNOWN) + return orientation; + + orientation = dev_priv->vbt.orientation; + if (orientation != DRM_MODE_PANEL_ORIENTATION_UNKNOWN) + return orientation; + + return DRM_MODE_PANEL_ORIENTATION_NORMAL; +} diff --git a/drivers/gpu/drm/i915/intel_dsi.h b/drivers/gpu/drm/i915/intel_dsi.h index ad7c1cb32983..a9a19778dc7f 100644 --- a/drivers/gpu/drm/i915/intel_dsi.h +++ b/drivers/gpu/drm/i915/intel_dsi.h @@ -24,7 +24,6 @@ #ifndef _INTEL_DSI_H #define _INTEL_DSI_H -#include <drm/drmP.h> #include <drm/drm_crtc.h> #include <drm/drm_mipi_dsi.h> #include "intel_drv.h" @@ -40,6 +39,7 @@ struct intel_dsi { struct intel_encoder base; struct intel_dsi_host *dsi_hosts[I915_MAX_PORTS]; + intel_wakeref_t io_wakeref[I915_MAX_PORTS]; /* GPIO Desc for CRC based Panel control */ struct gpio_desc *gpio_panel; @@ -81,14 +81,21 @@ struct intel_dsi { u16 dcs_backlight_ports; u16 dcs_cabc_ports; + /* RGB or BGR */ + bool bgr_enabled; + u8 pixel_overlap; u32 port_bits; u32 bw_timer; u32 dphy_reg; + + /* data lanes dphy timing */ + u32 dphy_data_lane_reg; u32 video_frmt_cfg_bits; u16 lp_byte_clk; /* timeouts in byte clocks */ + u16 hs_tx_timeout; u16 lp_rx_timeout; u16 turn_arnd_val; u16 rst_timer_val; @@ -129,9 +136,36 @@ static inline struct intel_dsi *enc_to_intel_dsi(struct drm_encoder *encoder) return container_of(encoder, struct intel_dsi, base.base); } +static inline bool is_vid_mode(struct intel_dsi *intel_dsi) +{ + return intel_dsi->operation_mode == INTEL_DSI_VIDEO_MODE; +} + +static inline bool is_cmd_mode(struct intel_dsi *intel_dsi) +{ + return intel_dsi->operation_mode == INTEL_DSI_COMMAND_MODE; +} + +static inline u16 intel_dsi_encoder_ports(struct intel_encoder *encoder) +{ + return enc_to_intel_dsi(&encoder->base)->ports; +} + +/* intel_dsi.c */ +int intel_dsi_bitrate(const struct intel_dsi *intel_dsi); +int intel_dsi_tlpx_ns(const struct intel_dsi *intel_dsi); +enum drm_panel_orientation +intel_dsi_get_panel_orientation(struct intel_connector *connector); + /* vlv_dsi.c */ void vlv_dsi_wait_for_fifo_empty(struct intel_dsi *intel_dsi, enum port port); enum mipi_dsi_pixel_format pixel_format_from_register_bits(u32 fmt); +int intel_dsi_get_modes(struct drm_connector *connector); +enum drm_mode_status intel_dsi_mode_valid(struct drm_connector *connector, + struct drm_display_mode *mode); +struct intel_dsi_host *intel_dsi_host_init(struct intel_dsi *intel_dsi, + const struct mipi_dsi_host_ops *funcs, + enum port port); /* vlv_dsi_pll.c */ int vlv_dsi_pll_compute(struct intel_encoder *encoder, @@ -139,7 +173,7 @@ int vlv_dsi_pll_compute(struct intel_encoder *encoder, void vlv_dsi_pll_enable(struct intel_encoder *encoder, const struct intel_crtc_state *config); void vlv_dsi_pll_disable(struct intel_encoder *encoder); -u32 vlv_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp, +u32 vlv_dsi_get_pclk(struct intel_encoder *encoder, struct intel_crtc_state *config); void vlv_dsi_reset_clocks(struct intel_encoder *encoder, enum port port); @@ -149,7 +183,7 @@ int bxt_dsi_pll_compute(struct intel_encoder *encoder, void bxt_dsi_pll_enable(struct intel_encoder *encoder, const struct intel_crtc_state *config); void bxt_dsi_pll_disable(struct intel_encoder *encoder); -u32 bxt_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp, +u32 bxt_dsi_get_pclk(struct intel_encoder *encoder, struct intel_crtc_state *config); void bxt_dsi_reset_clocks(struct intel_encoder *encoder, enum port port); @@ -158,5 +192,6 @@ bool intel_dsi_vbt_init(struct intel_dsi *intel_dsi, u16 panel_id); int intel_dsi_vbt_get_modes(struct intel_dsi *intel_dsi); void intel_dsi_vbt_exec_sequence(struct intel_dsi *intel_dsi, enum mipi_seq seq_id); +void intel_dsi_msleep(struct intel_dsi *intel_dsi, int msec); #endif /* _INTEL_DSI_H */ diff --git a/drivers/gpu/drm/i915/intel_dsi_vbt.c b/drivers/gpu/drm/i915/intel_dsi_vbt.c index ac83d6b89ae0..06a11c35a784 100644 --- a/drivers/gpu/drm/i915/intel_dsi_vbt.c +++ b/drivers/gpu/drm/i915/intel_dsi_vbt.c @@ -24,15 +24,15 @@ * */ -#include <drm/drmP.h> #include <drm/drm_crtc.h> #include <drm/drm_edid.h> #include <drm/i915_drm.h> #include <linux/gpio/consumer.h> +#include <linux/mfd/intel_soc_pmic.h> #include <linux/slab.h> #include <video/mipi_display.h> #include <asm/intel-mid.h> -#include <video/mipi_display.h> +#include <asm/unaligned.h> #include "i915_drv.h" #include "intel_drv.h" #include "intel_dsi.h" @@ -103,6 +103,18 @@ static struct gpio_map vlv_gpio_table[] = { #define CHV_GPIO_PAD_CFG1(f, i) (0x4400 + (f) * 0x400 + (i) * 8 + 4) #define CHV_GPIO_CFGLOCK (1 << 31) +/* ICL DSI Display GPIO Pins */ +#define ICL_GPIO_DDSP_HPD_A 0 +#define ICL_GPIO_L_VDDEN_1 1 +#define ICL_GPIO_L_BKLTEN_1 2 +#define ICL_GPIO_DDPA_CTRLCLK_1 3 +#define ICL_GPIO_DDPA_CTRLDATA_1 4 +#define ICL_GPIO_DDSP_HPD_B 5 +#define ICL_GPIO_L_VDDEN_2 6 +#define ICL_GPIO_L_BKLTEN_2 7 +#define ICL_GPIO_DDPA_CTRLCLK_2 8 +#define ICL_GPIO_DDPA_CTRLDATA_2 9 + static inline enum port intel_dsi_seq_port_to_port(u8 port) { return port ? PORT_C : PORT_A; @@ -111,6 +123,7 @@ static inline enum port intel_dsi_seq_port_to_port(u8 port) static const u8 *mipi_exec_send_packet(struct intel_dsi *intel_dsi, const u8 *data) { + struct drm_i915_private *dev_priv = to_i915(intel_dsi->base.base.dev); struct mipi_dsi_device *dsi_device; u8 type, flags, seq_port; u16 len; @@ -181,7 +194,8 @@ static const u8 *mipi_exec_send_packet(struct intel_dsi *intel_dsi, break; } - vlv_dsi_wait_for_fifo_empty(intel_dsi, port); + if (!IS_ICELAKE(dev_priv)) + vlv_dsi_wait_for_fifo_empty(intel_dsi, port); out: data += len; @@ -322,6 +336,12 @@ static void bxt_exec_gpio(struct drm_i915_private *dev_priv, gpiod_set_value(gpio_desc, value); } +static void icl_exec_gpio(struct drm_i915_private *dev_priv, + u8 gpio_source, u8 gpio_index, bool value) +{ + DRM_DEBUG_KMS("Skipping ICL GPIO element execution\n"); +} + static const u8 *mipi_exec_gpio(struct intel_dsi *intel_dsi, const u8 *data) { struct drm_device *dev = intel_dsi->base.base.dev; @@ -345,7 +365,9 @@ static const u8 *mipi_exec_gpio(struct intel_dsi *intel_dsi, const u8 *data) /* pull up/down */ value = *data++ & 1; - if (IS_VALLEYVIEW(dev_priv)) + if (IS_ICELAKE(dev_priv)) + icl_exec_gpio(dev_priv, gpio_source, gpio_index, value); + else if (IS_VALLEYVIEW(dev_priv)) vlv_exec_gpio(dev_priv, gpio_source, gpio_number, value); else if (IS_CHERRYVIEW(dev_priv)) chv_exec_gpio(dev_priv, gpio_source, gpio_number, value); @@ -371,7 +393,25 @@ static const u8 *mipi_exec_spi(struct intel_dsi *intel_dsi, const u8 *data) static const u8 *mipi_exec_pmic(struct intel_dsi *intel_dsi, const u8 *data) { - DRM_DEBUG_KMS("Skipping PMIC element execution\n"); +#ifdef CONFIG_PMIC_OPREGION + u32 value, mask, reg_address; + u16 i2c_address; + int ret; + + /* byte 0 aka PMIC Flag is reserved */ + i2c_address = get_unaligned_le16(data + 1); + reg_address = get_unaligned_le32(data + 3); + value = get_unaligned_le32(data + 7); + mask = get_unaligned_le32(data + 11); + + ret = intel_soc_pmic_exec_mipi_pmic_seq_element(i2c_address, + reg_address, + value, mask); + if (ret) + DRM_ERROR("%s failed, error: %d\n", __func__, ret); +#else + DRM_ERROR("Your hardware requires CONFIG_PMIC_OPREGION and it is not set\n"); +#endif return data + 15; } @@ -481,6 +521,17 @@ void intel_dsi_vbt_exec_sequence(struct intel_dsi *intel_dsi, } } +void intel_dsi_msleep(struct intel_dsi *intel_dsi, int msec) +{ + struct drm_i915_private *dev_priv = to_i915(intel_dsi->base.base.dev); + + /* For v3 VBTs in vid-mode the delays are part of the VBT sequences */ + if (is_vid_mode(intel_dsi) && dev_priv->vbt.dsi.seq_version >= 3) + return; + + msleep(msec); +} + int intel_dsi_vbt_get_modes(struct intel_dsi *intel_dsi) { struct intel_connector *connector = intel_dsi->attached_connector; @@ -499,110 +550,125 @@ int intel_dsi_vbt_get_modes(struct intel_dsi *intel_dsi) return 1; } -bool intel_dsi_vbt_init(struct intel_dsi *intel_dsi, u16 panel_id) +#define ICL_PREPARE_CNT_MAX 0x7 +#define ICL_CLK_ZERO_CNT_MAX 0xf +#define ICL_TRAIL_CNT_MAX 0x7 +#define ICL_TCLK_PRE_CNT_MAX 0x3 +#define ICL_TCLK_POST_CNT_MAX 0x7 +#define ICL_HS_ZERO_CNT_MAX 0xf +#define ICL_EXIT_ZERO_CNT_MAX 0x7 + +static void icl_dphy_param_init(struct intel_dsi *intel_dsi) { struct drm_device *dev = intel_dsi->base.base.dev; struct drm_i915_private *dev_priv = to_i915(dev); struct mipi_config *mipi_config = dev_priv->vbt.dsi.config; - struct mipi_pps_data *pps = dev_priv->vbt.dsi.pps; - struct drm_display_mode *mode = dev_priv->vbt.lfp_lvds_vbt_mode; - u32 bpp; - u32 tlpx_ns, extra_byte_count, bitrate, tlpx_ui; - u32 ui_num, ui_den; + u32 tlpx_ns; u32 prepare_cnt, exit_zero_cnt, clk_zero_cnt, trail_cnt; u32 ths_prepare_ns, tclk_trail_ns; - u32 tclk_prepare_clkzero, ths_prepare_hszero; - u32 lp_to_hs_switch, hs_to_lp_switch; - u32 pclk, computed_ddr; - u32 mul; - u16 burst_mode_ratio; - enum port port; + u32 hs_zero_cnt; + u32 tclk_pre_cnt, tclk_post_cnt; - DRM_DEBUG_KMS("\n"); + tlpx_ns = intel_dsi_tlpx_ns(intel_dsi); - intel_dsi->eotp_pkt = mipi_config->eot_pkt_disabled ? 0 : 1; - intel_dsi->clock_stop = mipi_config->enable_clk_stop ? 1 : 0; - intel_dsi->lane_count = mipi_config->lane_cnt + 1; - intel_dsi->pixel_format = - pixel_format_from_register_bits( - mipi_config->videomode_color_format << 7); - bpp = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format); - - intel_dsi->dual_link = mipi_config->dual_link; - intel_dsi->pixel_overlap = mipi_config->pixel_overlap; - intel_dsi->operation_mode = mipi_config->is_cmd_mode; - intel_dsi->video_mode_format = mipi_config->video_transfer_mode; - intel_dsi->escape_clk_div = mipi_config->byte_clk_sel; - intel_dsi->lp_rx_timeout = mipi_config->lp_rx_timeout; - intel_dsi->turn_arnd_val = mipi_config->turn_around_timeout; - intel_dsi->rst_timer_val = mipi_config->device_reset_timer; - intel_dsi->init_count = mipi_config->master_init_timer; - intel_dsi->bw_timer = mipi_config->dbi_bw_timer; - intel_dsi->video_frmt_cfg_bits = - mipi_config->bta_enabled ? DISABLE_VIDEO_BTA : 0; - - pclk = mode->clock; + tclk_trail_ns = max(mipi_config->tclk_trail, mipi_config->ths_trail); + ths_prepare_ns = max(mipi_config->ths_prepare, + mipi_config->tclk_prepare); - /* In dual link mode each port needs half of pixel clock */ - if (intel_dsi->dual_link) { - pclk = pclk / 2; + /* + * prepare cnt in escape clocks + * this field represents a hexadecimal value with a precision + * of 1.2 – i.e. the most significant bit is the integer + * and the least significant 2 bits are fraction bits. + * so, the field can represent a range of 0.25 to 1.75 + */ + prepare_cnt = DIV_ROUND_UP(ths_prepare_ns * 4, tlpx_ns); + if (prepare_cnt > ICL_PREPARE_CNT_MAX) { + DRM_DEBUG_KMS("prepare_cnt out of range (%d)\n", prepare_cnt); + prepare_cnt = ICL_PREPARE_CNT_MAX; + } - /* we can enable pixel_overlap if needed by panel. In this - * case we need to increase the pixelclock for extra pixels - */ - if (intel_dsi->dual_link == DSI_DUAL_LINK_FRONT_BACK) { - pclk += DIV_ROUND_UP(mode->vtotal * - intel_dsi->pixel_overlap * - 60, 1000); - } + /* clk zero count in escape clocks */ + clk_zero_cnt = DIV_ROUND_UP(mipi_config->tclk_prepare_clkzero - + ths_prepare_ns, tlpx_ns); + if (clk_zero_cnt > ICL_CLK_ZERO_CNT_MAX) { + DRM_DEBUG_KMS("clk_zero_cnt out of range (%d)\n", clk_zero_cnt); + clk_zero_cnt = ICL_CLK_ZERO_CNT_MAX; } - /* Burst Mode Ratio - * Target ddr frequency from VBT / non burst ddr freq - * multiply by 100 to preserve remainder - */ - if (intel_dsi->video_mode_format == VIDEO_MODE_BURST) { - if (mipi_config->target_burst_mode_freq) { - computed_ddr = (pclk * bpp) / intel_dsi->lane_count; + /* trail cnt in escape clocks*/ + trail_cnt = DIV_ROUND_UP(tclk_trail_ns, tlpx_ns); + if (trail_cnt > ICL_TRAIL_CNT_MAX) { + DRM_DEBUG_KMS("trail_cnt out of range (%d)\n", trail_cnt); + trail_cnt = ICL_TRAIL_CNT_MAX; + } - if (mipi_config->target_burst_mode_freq < - computed_ddr) { - DRM_ERROR("Burst mode freq is less than computed\n"); - return false; - } + /* tclk pre count in escape clocks */ + tclk_pre_cnt = DIV_ROUND_UP(mipi_config->tclk_pre, tlpx_ns); + if (tclk_pre_cnt > ICL_TCLK_PRE_CNT_MAX) { + DRM_DEBUG_KMS("tclk_pre_cnt out of range (%d)\n", tclk_pre_cnt); + tclk_pre_cnt = ICL_TCLK_PRE_CNT_MAX; + } - burst_mode_ratio = DIV_ROUND_UP( - mipi_config->target_burst_mode_freq * 100, - computed_ddr); + /* tclk post count in escape clocks */ + tclk_post_cnt = DIV_ROUND_UP(mipi_config->tclk_post, tlpx_ns); + if (tclk_post_cnt > ICL_TCLK_POST_CNT_MAX) { + DRM_DEBUG_KMS("tclk_post_cnt out of range (%d)\n", tclk_post_cnt); + tclk_post_cnt = ICL_TCLK_POST_CNT_MAX; + } - pclk = DIV_ROUND_UP(pclk * burst_mode_ratio, 100); - } else { - DRM_ERROR("Burst mode target is not set\n"); - return false; - } - } else - burst_mode_ratio = 100; + /* hs zero cnt in escape clocks */ + hs_zero_cnt = DIV_ROUND_UP(mipi_config->ths_prepare_hszero - + ths_prepare_ns, tlpx_ns); + if (hs_zero_cnt > ICL_HS_ZERO_CNT_MAX) { + DRM_DEBUG_KMS("hs_zero_cnt out of range (%d)\n", hs_zero_cnt); + hs_zero_cnt = ICL_HS_ZERO_CNT_MAX; + } - intel_dsi->burst_mode_ratio = burst_mode_ratio; - intel_dsi->pclk = pclk; + /* hs exit zero cnt in escape clocks */ + exit_zero_cnt = DIV_ROUND_UP(mipi_config->ths_exit, tlpx_ns); + if (exit_zero_cnt > ICL_EXIT_ZERO_CNT_MAX) { + DRM_DEBUG_KMS("exit_zero_cnt out of range (%d)\n", exit_zero_cnt); + exit_zero_cnt = ICL_EXIT_ZERO_CNT_MAX; + } - bitrate = (pclk * bpp) / intel_dsi->lane_count; + /* clock lane dphy timings */ + intel_dsi->dphy_reg = (CLK_PREPARE_OVERRIDE | + CLK_PREPARE(prepare_cnt) | + CLK_ZERO_OVERRIDE | + CLK_ZERO(clk_zero_cnt) | + CLK_PRE_OVERRIDE | + CLK_PRE(tclk_pre_cnt) | + CLK_POST_OVERRIDE | + CLK_POST(tclk_post_cnt) | + CLK_TRAIL_OVERRIDE | + CLK_TRAIL(trail_cnt)); + + /* data lanes dphy timings */ + intel_dsi->dphy_data_lane_reg = (HS_PREPARE_OVERRIDE | + HS_PREPARE(prepare_cnt) | + HS_ZERO_OVERRIDE | + HS_ZERO(hs_zero_cnt) | + HS_TRAIL_OVERRIDE | + HS_TRAIL(trail_cnt) | + HS_EXIT_OVERRIDE | + HS_EXIT(exit_zero_cnt)); +} - switch (intel_dsi->escape_clk_div) { - case 0: - tlpx_ns = 50; - break; - case 1: - tlpx_ns = 100; - break; +static void vlv_dphy_param_init(struct intel_dsi *intel_dsi) +{ + struct drm_device *dev = intel_dsi->base.base.dev; + struct drm_i915_private *dev_priv = to_i915(dev); + struct mipi_config *mipi_config = dev_priv->vbt.dsi.config; + u32 tlpx_ns, extra_byte_count, tlpx_ui; + u32 ui_num, ui_den; + u32 prepare_cnt, exit_zero_cnt, clk_zero_cnt, trail_cnt; + u32 ths_prepare_ns, tclk_trail_ns; + u32 tclk_prepare_clkzero, ths_prepare_hszero; + u32 lp_to_hs_switch, hs_to_lp_switch; + u32 mul; - case 2: - tlpx_ns = 200; - break; - default: - tlpx_ns = 50; - break; - } + tlpx_ns = intel_dsi_tlpx_ns(intel_dsi); switch (intel_dsi->lane_count) { case 1: @@ -620,7 +686,7 @@ bool intel_dsi_vbt_init(struct intel_dsi *intel_dsi, u16 panel_id) /* in Kbps */ ui_num = NS_KHZ_RATIO; - ui_den = bitrate; + ui_den = intel_dsi_bitrate(intel_dsi); tclk_prepare_clkzero = mipi_config->tclk_prepare_clkzero; ths_prepare_hszero = mipi_config->ths_prepare_hszero; @@ -746,6 +812,88 @@ bool intel_dsi_vbt_init(struct intel_dsi *intel_dsi, u16 panel_id) DIV_ROUND_UP(2 * tlpx_ui + trail_cnt * 2 + 8, 8); intel_dsi->clk_hs_to_lp_count += extra_byte_count; +} + +bool intel_dsi_vbt_init(struct intel_dsi *intel_dsi, u16 panel_id) +{ + struct drm_device *dev = intel_dsi->base.base.dev; + struct drm_i915_private *dev_priv = to_i915(dev); + struct mipi_config *mipi_config = dev_priv->vbt.dsi.config; + struct mipi_pps_data *pps = dev_priv->vbt.dsi.pps; + struct drm_display_mode *mode = dev_priv->vbt.lfp_lvds_vbt_mode; + u16 burst_mode_ratio; + enum port port; + + DRM_DEBUG_KMS("\n"); + + intel_dsi->eotp_pkt = mipi_config->eot_pkt_disabled ? 0 : 1; + intel_dsi->clock_stop = mipi_config->enable_clk_stop ? 1 : 0; + intel_dsi->lane_count = mipi_config->lane_cnt + 1; + intel_dsi->pixel_format = + pixel_format_from_register_bits( + mipi_config->videomode_color_format << 7); + + intel_dsi->dual_link = mipi_config->dual_link; + intel_dsi->pixel_overlap = mipi_config->pixel_overlap; + intel_dsi->operation_mode = mipi_config->is_cmd_mode; + intel_dsi->video_mode_format = mipi_config->video_transfer_mode; + intel_dsi->escape_clk_div = mipi_config->byte_clk_sel; + intel_dsi->lp_rx_timeout = mipi_config->lp_rx_timeout; + intel_dsi->hs_tx_timeout = mipi_config->hs_tx_timeout; + intel_dsi->turn_arnd_val = mipi_config->turn_around_timeout; + intel_dsi->rst_timer_val = mipi_config->device_reset_timer; + intel_dsi->init_count = mipi_config->master_init_timer; + intel_dsi->bw_timer = mipi_config->dbi_bw_timer; + intel_dsi->video_frmt_cfg_bits = + mipi_config->bta_enabled ? DISABLE_VIDEO_BTA : 0; + intel_dsi->bgr_enabled = mipi_config->rgb_flip; + + /* Starting point, adjusted depending on dual link and burst mode */ + intel_dsi->pclk = mode->clock; + + /* In dual link mode each port needs half of pixel clock */ + if (intel_dsi->dual_link) { + intel_dsi->pclk /= 2; + + /* we can enable pixel_overlap if needed by panel. In this + * case we need to increase the pixelclock for extra pixels + */ + if (intel_dsi->dual_link == DSI_DUAL_LINK_FRONT_BACK) { + intel_dsi->pclk += DIV_ROUND_UP(mode->vtotal * intel_dsi->pixel_overlap * 60, 1000); + } + } + + /* Burst Mode Ratio + * Target ddr frequency from VBT / non burst ddr freq + * multiply by 100 to preserve remainder + */ + if (intel_dsi->video_mode_format == VIDEO_MODE_BURST) { + if (mipi_config->target_burst_mode_freq) { + u32 bitrate = intel_dsi_bitrate(intel_dsi); + + if (mipi_config->target_burst_mode_freq < bitrate) { + DRM_ERROR("Burst mode freq is less than computed\n"); + return false; + } + + burst_mode_ratio = DIV_ROUND_UP( + mipi_config->target_burst_mode_freq * 100, + bitrate); + + intel_dsi->pclk = DIV_ROUND_UP(intel_dsi->pclk * burst_mode_ratio, 100); + } else { + DRM_ERROR("Burst mode target is not set\n"); + return false; + } + } else + burst_mode_ratio = 100; + + intel_dsi->burst_mode_ratio = burst_mode_ratio; + + if (IS_ICELAKE(dev_priv)) + icl_dphy_param_init(intel_dsi); + else + vlv_dphy_param_init(intel_dsi); DRM_DEBUG_KMS("Pclk %d\n", intel_dsi->pclk); DRM_DEBUG_KMS("Pixel overlap %d\n", intel_dsi->pixel_overlap); diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c index 4e142ff49708..a6c82482a841 100644 --- a/drivers/gpu/drm/i915/intel_dvo.c +++ b/drivers/gpu/drm/i915/intel_dvo.c @@ -26,7 +26,6 @@ */ #include <linux/i2c.h> #include <linux/slab.h> -#include <drm/drmP.h> #include <drm/drm_atomic_helper.h> #include <drm/drm_crtc.h> #include "intel_drv.h" @@ -235,9 +234,9 @@ intel_dvo_mode_valid(struct drm_connector *connector, return intel_dvo->dev.dev_ops->mode_valid(&intel_dvo->dev, mode); } -static bool intel_dvo_compute_config(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config, - struct drm_connector_state *conn_state) +static int intel_dvo_compute_config(struct intel_encoder *encoder, + struct intel_crtc_state *pipe_config, + struct drm_connector_state *conn_state) { struct intel_dvo *intel_dvo = enc_to_dvo(encoder); const struct drm_display_mode *fixed_mode = @@ -254,9 +253,11 @@ static bool intel_dvo_compute_config(struct intel_encoder *encoder, intel_fixed_panel_mode(fixed_mode, adjusted_mode); if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) - return false; + return -EINVAL; + + pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; - return true; + return 0; } static void intel_dvo_pre_enable(struct intel_encoder *encoder, @@ -333,18 +334,11 @@ static int intel_dvo_get_modes(struct drm_connector *connector) return 0; } -static void intel_dvo_destroy(struct drm_connector *connector) -{ - drm_connector_cleanup(connector); - intel_panel_fini(&to_intel_connector(connector)->panel); - kfree(connector); -} - static const struct drm_connector_funcs intel_dvo_connector_funcs = { .detect = intel_dvo_detect, .late_register = intel_connector_register, .early_unregister = intel_connector_unregister, - .destroy = intel_dvo_destroy, + .destroy = intel_connector_destroy, .fill_modes = drm_helper_probe_single_connector_modes, .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c index 76b5f94ea6cb..49fa43ff02ba 100644 --- a/drivers/gpu/drm/i915/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/intel_engine_cs.c @@ -25,6 +25,7 @@ #include <drm/drm_print.h> #include "i915_drv.h" +#include "i915_reset.h" #include "intel_ringbuffer.h" #include "intel_lrc.h" @@ -261,6 +262,31 @@ static void __sprint_engine_name(char *name, const struct engine_info *info) info->instance) >= INTEL_ENGINE_CS_MAX_NAME); } +void intel_engine_set_hwsp_writemask(struct intel_engine_cs *engine, u32 mask) +{ + struct drm_i915_private *dev_priv = engine->i915; + i915_reg_t hwstam; + + /* + * Though they added more rings on g4x/ilk, they did not add + * per-engine HWSTAM until gen6. + */ + if (INTEL_GEN(dev_priv) < 6 && engine->class != RENDER_CLASS) + return; + + hwstam = RING_HWSTAM(engine->mmio_base); + if (INTEL_GEN(dev_priv) >= 3) + I915_WRITE(hwstam, mask); + else + I915_WRITE16(hwstam, mask); +} + +static void intel_engine_sanitize_mmio(struct intel_engine_cs *engine) +{ + /* Mask off all writes into the unknown HWSP */ + intel_engine_set_hwsp_writemask(engine, ~0u); +} + static int intel_engine_setup(struct drm_i915_private *dev_priv, enum intel_engine_id id) @@ -273,13 +299,13 @@ intel_engine_setup(struct drm_i915_private *dev_priv, BUILD_BUG_ON(MAX_ENGINE_CLASS >= BIT(GEN11_ENGINE_CLASS_WIDTH)); BUILD_BUG_ON(MAX_ENGINE_INSTANCE >= BIT(GEN11_ENGINE_INSTANCE_WIDTH)); - if (GEM_WARN_ON(info->class > MAX_ENGINE_CLASS)) + if (GEM_DEBUG_WARN_ON(info->class > MAX_ENGINE_CLASS)) return -EINVAL; - if (GEM_WARN_ON(info->instance > MAX_ENGINE_INSTANCE)) + if (GEM_DEBUG_WARN_ON(info->instance > MAX_ENGINE_INSTANCE)) return -EINVAL; - if (GEM_WARN_ON(dev_priv->engine_class[info->class][info->instance])) + if (GEM_DEBUG_WARN_ON(dev_priv->engine_class[info->class][info->instance])) return -EINVAL; GEM_BUG_ON(dev_priv->engine[id]); @@ -312,6 +338,9 @@ intel_engine_setup(struct drm_i915_private *dev_priv, ATOMIC_INIT_NOTIFIER_HEAD(&engine->context_status_notifier); + /* Scrub mmio state on takeover */ + intel_engine_sanitize_mmio(engine); + dev_priv->engine_class[info->class][info->instance] = engine; dev_priv->engine[id] = engine; return 0; @@ -335,7 +364,10 @@ int intel_engines_init_mmio(struct drm_i915_private *dev_priv) WARN_ON(ring_mask == 0); WARN_ON(ring_mask & - GENMASK(sizeof(mask) * BITS_PER_BYTE - 1, I915_NUM_ENGINES)); + GENMASK(BITS_PER_TYPE(mask) - 1, I915_NUM_ENGINES)); + + if (i915_inject_load_failure()) + return -ENODEV; for (i = 0; i < ARRAY_SIZE(intel_engines); i++) { if (!HAS_ENGINE(dev_priv, i)) @@ -362,7 +394,7 @@ int intel_engines_init_mmio(struct drm_i915_private *dev_priv) goto cleanup; } - device_info->num_rings = hweight32(mask); + RUNTIME_INFO(dev_priv)->num_rings = hweight32(mask); i915_check_and_clear_faults(dev_priv); @@ -399,7 +431,7 @@ int intel_engines_init(struct drm_i915_private *dev_priv) err = -EINVAL; err_id = id; - if (GEM_WARN_ON(!init)) + if (GEM_DEBUG_WARN_ON(!init)) goto cleanup; err = init(engine); @@ -423,33 +455,9 @@ cleanup: return err; } -void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno) +void intel_engine_write_global_seqno(struct intel_engine_cs *engine, u32 seqno) { - struct drm_i915_private *dev_priv = engine->i915; - - /* Our semaphore implementation is strictly monotonic (i.e. we proceed - * so long as the semaphore value in the register/page is greater - * than the sync value), so whenever we reset the seqno, - * so long as we reset the tracking semaphore value to 0, it will - * always be before the next request's seqno. If we don't reset - * the semaphore value, then when the seqno moves backwards all - * future waits will complete instantly (causing rendering corruption). - */ - if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) { - I915_WRITE(RING_SYNC_0(engine->mmio_base), 0); - I915_WRITE(RING_SYNC_1(engine->mmio_base), 0); - if (HAS_VEBOX(dev_priv)) - I915_WRITE(RING_SYNC_2(engine->mmio_base), 0); - } - intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno); - clear_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted); - - /* After manually advancing the seqno, fake the interrupt in case - * there are any waiters for that seqno. - */ - intel_engine_wakeup(engine); - GEM_BUG_ON(intel_engine_get_seqno(engine) != seqno); } @@ -463,53 +471,70 @@ static void intel_engine_init_execlist(struct intel_engine_cs *engine) struct intel_engine_execlists * const execlists = &engine->execlists; execlists->port_mask = 1; - BUILD_BUG_ON_NOT_POWER_OF_2(execlists_num_ports(execlists)); + GEM_BUG_ON(!is_power_of_2(execlists_num_ports(execlists))); GEM_BUG_ON(execlists_num_ports(execlists) > EXECLIST_MAX_PORTS); - execlists->queue_priority = INT_MIN; + execlists->queue_priority_hint = INT_MIN; execlists->queue = RB_ROOT_CACHED; } -/** - * intel_engines_setup_common - setup engine state not requiring hw access - * @engine: Engine to setup. - * - * Initializes @engine@ structure members shared between legacy and execlists - * submission modes which do not require hardware access. - * - * Typically done early in the submission mode specific engine setup stage. - */ -void intel_engine_setup_common(struct intel_engine_cs *engine) +static void cleanup_status_page(struct intel_engine_cs *engine) { - i915_timeline_init(engine->i915, &engine->timeline, engine->name); - lockdep_set_subclass(&engine->timeline.lock, TIMELINE_ENGINE); + struct i915_vma *vma; - intel_engine_init_execlist(engine); - intel_engine_init_hangcheck(engine); - intel_engine_init_batch_pool(engine); - intel_engine_init_cmd_parser(engine); + /* Prevent writes into HWSP after returning the page to the system */ + intel_engine_set_hwsp_writemask(engine, ~0u); + + vma = fetch_and_zero(&engine->status_page.vma); + if (!vma) + return; + + if (!HWS_NEEDS_PHYSICAL(engine->i915)) + i915_vma_unpin(vma); + + i915_gem_object_unpin_map(vma->obj); + __i915_gem_object_release_unless_active(vma->obj); } -static void cleanup_status_page(struct intel_engine_cs *engine) +static int pin_ggtt_status_page(struct intel_engine_cs *engine, + struct i915_vma *vma) { - if (HWS_NEEDS_PHYSICAL(engine->i915)) { - void *addr = fetch_and_zero(&engine->status_page.page_addr); + unsigned int flags; - __free_page(virt_to_page(addr)); - } + flags = PIN_GLOBAL; + if (!HAS_LLC(engine->i915)) + /* + * On g33, we cannot place HWS above 256MiB, so + * restrict its pinning to the low mappable arena. + * Though this restriction is not documented for + * gen4, gen5, or byt, they also behave similarly + * and hang if the HWS is placed at the top of the + * GTT. To generalise, it appears that all !llc + * platforms have issues with us placing the HWS + * above the mappable region (even though we never + * actually map it). + */ + flags |= PIN_MAPPABLE; + else + flags |= PIN_HIGH; - i915_vma_unpin_and_release(&engine->status_page.vma, - I915_VMA_RELEASE_MAP); + return i915_vma_pin(vma, 0, 0, flags); } static int init_status_page(struct intel_engine_cs *engine) { struct drm_i915_gem_object *obj; struct i915_vma *vma; - unsigned int flags; void *vaddr; int ret; + /* + * Though the HWS register does support 36bit addresses, historically + * we have had hangs and corruption reported due to wild writes if + * the HWS is placed above 4G. We only allow objects to be allocated + * in GFP_DMA32 for i965, and no earlier physical address users had + * access to more than 4G. + */ obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE); if (IS_ERR(obj)) { DRM_ERROR("Failed to allocate status page\n"); @@ -526,59 +551,67 @@ static int init_status_page(struct intel_engine_cs *engine) goto err; } - flags = PIN_GLOBAL; - if (!HAS_LLC(engine->i915)) - /* On g33, we cannot place HWS above 256MiB, so - * restrict its pinning to the low mappable arena. - * Though this restriction is not documented for - * gen4, gen5, or byt, they also behave similarly - * and hang if the HWS is placed at the top of the - * GTT. To generalise, it appears that all !llc - * platforms have issues with us placing the HWS - * above the mappable region (even though we never - * actually map it). - */ - flags |= PIN_MAPPABLE; - else - flags |= PIN_HIGH; - ret = i915_vma_pin(vma, 0, 0, flags); - if (ret) - goto err; - vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB); if (IS_ERR(vaddr)) { ret = PTR_ERR(vaddr); - goto err_unpin; + goto err; } + engine->status_page.addr = memset(vaddr, 0, PAGE_SIZE); engine->status_page.vma = vma; - engine->status_page.ggtt_offset = i915_ggtt_offset(vma); - engine->status_page.page_addr = memset(vaddr, 0, PAGE_SIZE); + + if (!HWS_NEEDS_PHYSICAL(engine->i915)) { + ret = pin_ggtt_status_page(engine, vma); + if (ret) + goto err_unpin; + } + return 0; err_unpin: - i915_vma_unpin(vma); + i915_gem_object_unpin_map(obj); err: i915_gem_object_put(obj); return ret; } -static int init_phys_status_page(struct intel_engine_cs *engine) +/** + * intel_engines_setup_common - setup engine state not requiring hw access + * @engine: Engine to setup. + * + * Initializes @engine@ structure members shared between legacy and execlists + * submission modes which do not require hardware access. + * + * Typically done early in the submission mode specific engine setup stage. + */ +int intel_engine_setup_common(struct intel_engine_cs *engine) { - struct page *page; + int err; - /* - * Though the HWS register does support 36bit addresses, historically - * we have had hangs and corruption reported due to wild writes if - * the HWS is placed above 4G. - */ - page = alloc_page(GFP_KERNEL | __GFP_DMA32 | __GFP_ZERO); - if (!page) - return -ENOMEM; + err = init_status_page(engine); + if (err) + return err; + + err = i915_timeline_init(engine->i915, + &engine->timeline, + engine->name, + engine->status_page.vma); + if (err) + goto err_hwsp; - engine->status_page.page_addr = page_address(page); + i915_timeline_set_subclass(&engine->timeline, TIMELINE_ENGINE); + + intel_engine_init_breadcrumbs(engine); + intel_engine_init_execlist(engine); + intel_engine_init_hangcheck(engine); + intel_engine_init_batch_pool(engine); + intel_engine_init_cmd_parser(engine); return 0; + +err_hwsp: + cleanup_status_page(engine); + return err; } static void __intel_context_unpin(struct i915_gem_context *ctx, @@ -587,6 +620,56 @@ static void __intel_context_unpin(struct i915_gem_context *ctx, intel_context_unpin(to_intel_context(ctx, engine)); } +struct measure_breadcrumb { + struct i915_request rq; + struct i915_timeline timeline; + struct intel_ring ring; + u32 cs[1024]; +}; + +static int measure_breadcrumb_dw(struct intel_engine_cs *engine) +{ + struct measure_breadcrumb *frame; + int dw = -ENOMEM; + + GEM_BUG_ON(!engine->i915->gt.scratch); + + frame = kzalloc(sizeof(*frame), GFP_KERNEL); + if (!frame) + return -ENOMEM; + + if (i915_timeline_init(engine->i915, + &frame->timeline, "measure", + engine->status_page.vma)) + goto out_frame; + + INIT_LIST_HEAD(&frame->ring.request_list); + frame->ring.timeline = &frame->timeline; + frame->ring.vaddr = frame->cs; + frame->ring.size = sizeof(frame->cs); + frame->ring.effective_size = frame->ring.size; + intel_ring_update_space(&frame->ring); + + frame->rq.i915 = engine->i915; + frame->rq.engine = engine; + frame->rq.ring = &frame->ring; + frame->rq.timeline = &frame->timeline; + + dw = i915_timeline_pin(&frame->timeline); + if (dw < 0) + goto out_timeline; + + dw = engine->emit_fini_breadcrumb(&frame->rq, frame->cs) - frame->cs; + + i915_timeline_unpin(&frame->timeline); + +out_timeline: + i915_timeline_fini(&frame->timeline); +out_frame: + kfree(frame); + return dw; +} + /** * intel_engines_init_common - initialize cengine state which might require hw access * @engine: Engine to initialize. @@ -629,21 +712,14 @@ int intel_engine_init_common(struct intel_engine_cs *engine) } } - ret = intel_engine_init_breadcrumbs(engine); - if (ret) + ret = measure_breadcrumb_dw(engine); + if (ret < 0) goto err_unpin_preempt; - if (HWS_NEEDS_PHYSICAL(i915)) - ret = init_phys_status_page(engine); - else - ret = init_status_page(engine); - if (ret) - goto err_breadcrumbs; + engine->emit_fini_breadcrumb_dw = ret; return 0; -err_breadcrumbs: - intel_engine_fini_breadcrumbs(engine); err_unpin_preempt: if (i915->preempt_context) __intel_context_unpin(i915->preempt_context, engine); @@ -679,7 +755,9 @@ void intel_engine_cleanup_common(struct intel_engine_cs *engine) i915_timeline_fini(&engine->timeline); + intel_wa_list_free(&engine->ctx_wa_list); intel_wa_list_free(&engine->wa_list); + intel_wa_list_free(&engine->whitelist); } u64 intel_engine_get_active_head(const struct intel_engine_cs *engine) @@ -764,12 +842,12 @@ const char *i915_cache_level_str(struct drm_i915_private *i915, int type) u32 intel_calculate_mcr_s_ss_select(struct drm_i915_private *dev_priv) { - const struct sseu_dev_info *sseu = &(INTEL_INFO(dev_priv)->sseu); + const struct sseu_dev_info *sseu = &RUNTIME_INFO(dev_priv)->sseu; u32 mcr_s_ss_select; u32 slice = fls(sseu->slice_mask); u32 subslice = fls(sseu->subslice_mask[slice]); - if (INTEL_GEN(dev_priv) == 10) + if (IS_GEN(dev_priv, 10)) mcr_s_ss_select = GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice); else if (INTEL_GEN(dev_priv) >= 11) @@ -781,15 +859,15 @@ u32 intel_calculate_mcr_s_ss_select(struct drm_i915_private *dev_priv) return mcr_s_ss_select; } -static inline uint32_t +static inline u32 read_subslice_reg(struct drm_i915_private *dev_priv, int slice, int subslice, i915_reg_t reg) { - uint32_t mcr_slice_subslice_mask; - uint32_t mcr_slice_subslice_select; - uint32_t default_mcr_s_ss_select; - uint32_t mcr; - uint32_t ret; + u32 mcr_slice_subslice_mask; + u32 mcr_slice_subslice_select; + u32 default_mcr_s_ss_select; + u32 mcr; + u32 ret; enum forcewake_domains fw_domains; if (INTEL_GEN(dev_priv) >= 11) { @@ -895,10 +973,15 @@ void intel_engine_get_instdone(struct intel_engine_cs *engine, static bool ring_is_idle(struct intel_engine_cs *engine) { struct drm_i915_private *dev_priv = engine->i915; + intel_wakeref_t wakeref; bool idle = true; + if (I915_SELFTEST_ONLY(!engine->mmio_base)) + return true; + /* If the whole device is asleep, the engine must be idle */ - if (!intel_runtime_pm_get_if_in_use(dev_priv)) + wakeref = intel_runtime_pm_get_if_in_use(dev_priv); + if (!wakeref) return true; /* First check that no commands are left in the ring */ @@ -910,7 +993,7 @@ static bool ring_is_idle(struct intel_engine_cs *engine) if (INTEL_GEN(dev_priv) > 2 && !(I915_READ_MODE(engine) & MODE_IDLE)) idle = false; - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, wakeref); return idle; } @@ -934,9 +1017,6 @@ bool intel_engine_is_idle(struct intel_engine_cs *engine) if (!intel_engine_signaled(engine, intel_engine_last_submit(engine))) return false; - if (I915_SELFTEST_ONLY(engine->breadcrumbs.mock)) - return true; - /* Waiting to drain ELSP? */ if (READ_ONCE(engine->execlists.active)) { struct tasklet_struct *t = &engine->execlists.tasklet; @@ -962,10 +1042,7 @@ bool intel_engine_is_idle(struct intel_engine_cs *engine) return false; /* Ring stopped? */ - if (!ring_is_idle(engine)) - return false; - - return true; + return ring_is_idle(engine); } bool intel_engines_are_idle(struct drm_i915_private *dev_priv) @@ -1009,7 +1086,7 @@ bool intel_engine_has_kernel_context(const struct intel_engine_cs *engine) * the last request that remains in the timeline. When idle, it is * the last executed context as tracked by retirement. */ - rq = __i915_gem_active_peek(&engine->timeline.last_request); + rq = __i915_active_request_peek(&engine->timeline.last_request); if (rq) return rq->hw_context == kernel_context; else @@ -1025,26 +1102,36 @@ void intel_engines_reset_default_submission(struct drm_i915_private *i915) engine->set_default_submission(engine); } +static bool reset_engines(struct drm_i915_private *i915) +{ + if (INTEL_INFO(i915)->gpu_reset_clobbers_display) + return false; + + return intel_gpu_reset(i915, ALL_ENGINES) == 0; +} + /** * intel_engines_sanitize: called after the GPU has lost power * @i915: the i915 device + * @force: ignore a failed reset and sanitize engine state anyway * * Anytime we reset the GPU, either with an explicit GPU reset or through a * PCI power cycle, the GPU loses state and we must reset our state tracking * to match. Note that calling intel_engines_sanitize() if the GPU has not * been reset results in much confusion! */ -void intel_engines_sanitize(struct drm_i915_private *i915) +void intel_engines_sanitize(struct drm_i915_private *i915, bool force) { struct intel_engine_cs *engine; enum intel_engine_id id; GEM_TRACE("\n"); - for_each_engine(engine, i915, id) { - if (engine->reset.reset) - engine->reset.reset(engine, NULL); - } + if (!reset_engines(i915) && !force) + return; + + for_each_engine(engine, i915, id) + intel_engine_reset(engine, false); } /** @@ -1080,7 +1167,7 @@ void intel_engines_park(struct drm_i915_private *i915) } /* Must be reset upon idling, or we may miss the busy wakeup. */ - GEM_BUG_ON(engine->execlists.queue_priority != INT_MIN); + GEM_BUG_ON(engine->execlists.queue_priority_hint != INT_MIN); if (engine->park) engine->park(engine); @@ -1196,10 +1283,14 @@ static void print_request(struct drm_printer *m, x = print_sched_attr(rq->i915, &rq->sched.attr, buf, x, sizeof(buf)); - drm_printf(m, "%s%x%s [%llx:%x]%s @ %dms: %s\n", + drm_printf(m, "%s%x%s%s [%llx:%llx]%s @ %dms: %s\n", prefix, rq->global_seqno, - i915_request_completed(rq) ? "!" : "", + i915_request_completed(rq) ? "!" : + i915_request_started(rq) ? "*" : + "", + test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, + &rq->fence.flags) ? "+" : "", rq->fence.context, rq->fence.seqno, buf, jiffies_to_msecs(jiffies - rq->emitted_jiffies), @@ -1243,7 +1334,7 @@ static void intel_engine_print_registers(const struct intel_engine_cs *engine, &engine->execlists; u64 addr; - if (engine->id == RCS && IS_GEN(dev_priv, 4, 7)) + if (engine->id == RCS && IS_GEN_RANGE(dev_priv, 4, 7)) drm_printf(m, "\tCCID: 0x%08x\n", I915_READ(CCID)); drm_printf(m, "\tRING_START: 0x%08x\n", I915_READ(RING_START(engine->mmio_base))); @@ -1264,16 +1355,6 @@ static void intel_engine_print_registers(const struct intel_engine_cs *engine, drm_printf(m, "\tRING_IMR: %08x\n", I915_READ_IMR(engine)); } - if (HAS_LEGACY_SEMAPHORES(dev_priv)) { - drm_printf(m, "\tSYNC_0: 0x%08x\n", - I915_READ(RING_SYNC_0(engine->mmio_base))); - drm_printf(m, "\tSYNC_1: 0x%08x\n", - I915_READ(RING_SYNC_1(engine->mmio_base))); - if (HAS_VEBOX(dev_priv)) - drm_printf(m, "\tSYNC_2: 0x%08x\n", - I915_READ(RING_SYNC_2(engine->mmio_base))); - } - addr = intel_engine_get_active_head(engine); drm_printf(m, "\tACTHD: 0x%08x_%08x\n", upper_32_bits(addr), lower_32_bits(addr)); @@ -1300,7 +1381,8 @@ static void intel_engine_print_registers(const struct intel_engine_cs *engine, } if (HAS_EXECLISTS(dev_priv)) { - const u32 *hws = &engine->status_page.page_addr[I915_HWS_CSB_BUF0_INDEX]; + const u32 *hws = + &engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX]; unsigned int idx; u8 read, write; @@ -1343,9 +1425,10 @@ static void intel_engine_print_registers(const struct intel_engine_cs *engine, char hdr[80]; snprintf(hdr, sizeof(hdr), - "\t\tELSP[%d] count=%d, ring->start=%08x, rq: ", + "\t\tELSP[%d] count=%d, ring:{start:%08x, hwsp:%08x}, rq: ", idx, count, - i915_ggtt_offset(rq->ring->vma)); + i915_ggtt_offset(rq->ring->vma), + rq->timeline->hwsp_offset); print_request(m, rq, hdr); } else { drm_printf(m, "\t\tELSP[%d] idle\n", idx); @@ -1400,14 +1483,9 @@ void intel_engine_dump(struct intel_engine_cs *engine, struct drm_printer *m, const char *header, ...) { - const int MAX_REQUESTS_TO_SHOW = 8; - struct intel_breadcrumbs * const b = &engine->breadcrumbs; - const struct intel_engine_execlists * const execlists = &engine->execlists; struct i915_gpu_error * const error = &engine->i915->gpu_error; - struct i915_request *rq, *last; - unsigned long flags; - struct rb_node *rb; - int count; + struct i915_request *rq; + intel_wakeref_t wakeref; if (header) { va_list ap; @@ -1457,83 +1535,30 @@ void intel_engine_dump(struct intel_engine_cs *engine, rq->ring->emit); drm_printf(m, "\t\tring->space: 0x%08x\n", rq->ring->space); + drm_printf(m, "\t\tring->hwsp: 0x%08x\n", + rq->timeline->hwsp_offset); print_request_ring(m, rq); } rcu_read_unlock(); - if (intel_runtime_pm_get_if_in_use(engine->i915)) { + wakeref = intel_runtime_pm_get_if_in_use(engine->i915); + if (wakeref) { intel_engine_print_registers(engine, m); - intel_runtime_pm_put(engine->i915); + intel_runtime_pm_put(engine->i915, wakeref); } else { drm_printf(m, "\tDevice is asleep; skipping register dump\n"); } - local_irq_save(flags); - spin_lock(&engine->timeline.lock); - - last = NULL; - count = 0; - list_for_each_entry(rq, &engine->timeline.requests, link) { - if (count++ < MAX_REQUESTS_TO_SHOW - 1) - print_request(m, rq, "\t\tE "); - else - last = rq; - } - if (last) { - if (count > MAX_REQUESTS_TO_SHOW) { - drm_printf(m, - "\t\t...skipping %d executing requests...\n", - count - MAX_REQUESTS_TO_SHOW); - } - print_request(m, last, "\t\tE "); - } - - last = NULL; - count = 0; - drm_printf(m, "\t\tQueue priority: %d\n", execlists->queue_priority); - for (rb = rb_first_cached(&execlists->queue); rb; rb = rb_next(rb)) { - struct i915_priolist *p = - rb_entry(rb, typeof(*p), node); - - list_for_each_entry(rq, &p->requests, sched.link) { - if (count++ < MAX_REQUESTS_TO_SHOW - 1) - print_request(m, rq, "\t\tQ "); - else - last = rq; - } - } - if (last) { - if (count > MAX_REQUESTS_TO_SHOW) { - drm_printf(m, - "\t\t...skipping %d queued requests...\n", - count - MAX_REQUESTS_TO_SHOW); - } - print_request(m, last, "\t\tQ "); - } - - spin_unlock(&engine->timeline.lock); - - spin_lock(&b->rb_lock); - for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) { - struct intel_wait *w = rb_entry(rb, typeof(*w), node); - - drm_printf(m, "\t%s [%d] waiting for %x\n", - w->tsk->comm, w->tsk->pid, w->seqno); - } - spin_unlock(&b->rb_lock); - local_irq_restore(flags); - - drm_printf(m, "IRQ? 0x%lx (breadcrumbs? %s)\n", - engine->irq_posted, - yesno(test_bit(ENGINE_IRQ_BREADCRUMB, - &engine->irq_posted))); + intel_execlists_show_requests(engine, m, print_request, 8); drm_printf(m, "HWSP:\n"); - hexdump(m, engine->status_page.page_addr, PAGE_SIZE); + hexdump(m, engine->status_page.addr, PAGE_SIZE); drm_printf(m, "Idle? %s\n", yesno(intel_engine_is_idle(engine))); + + intel_engine_print_breadcrumbs(engine, m); } static u8 user_class_map[] = { diff --git a/drivers/gpu/drm/i915/intel_fbc.c b/drivers/gpu/drm/i915/intel_fbc.c index 74d425c700ef..656e684e7c9a 100644 --- a/drivers/gpu/drm/i915/intel_fbc.c +++ b/drivers/gpu/drm/i915/intel_fbc.c @@ -38,6 +38,8 @@ * forcibly disable it to allow proper screen updates. */ +#include <drm/drm_fourcc.h> + #include "intel_drv.h" #include "i915_drv.h" @@ -84,7 +86,7 @@ static int intel_fbc_calculate_cfb_size(struct drm_i915_private *dev_priv, int lines; intel_fbc_get_plane_source_size(cache, NULL, &lines); - if (INTEL_GEN(dev_priv) == 7) + if (IS_GEN(dev_priv, 7)) lines = min(lines, 2048); else if (INTEL_GEN(dev_priv) >= 8) lines = min(lines, 2560); @@ -127,7 +129,7 @@ static void i8xx_fbc_activate(struct drm_i915_private *dev_priv) cfb_pitch = params->fb.stride; /* FBC_CTL wants 32B or 64B units */ - if (IS_GEN2(dev_priv)) + if (IS_GEN(dev_priv, 2)) cfb_pitch = (cfb_pitch / 32) - 1; else cfb_pitch = (cfb_pitch / 64) - 1; @@ -136,7 +138,7 @@ static void i8xx_fbc_activate(struct drm_i915_private *dev_priv) for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++) I915_WRITE(FBC_TAG(i), 0); - if (IS_GEN4(dev_priv)) { + if (IS_GEN(dev_priv, 4)) { u32 fbc_ctl2; /* Set it up... */ @@ -233,9 +235,9 @@ static void ilk_fbc_activate(struct drm_i915_private *dev_priv) if (params->flags & PLANE_HAS_FENCE) { dpfc_ctl |= DPFC_CTL_FENCE_EN; - if (IS_GEN5(dev_priv)) + if (IS_GEN(dev_priv, 5)) dpfc_ctl |= params->vma->fence->id; - if (IS_GEN6(dev_priv)) { + if (IS_GEN(dev_priv, 6)) { I915_WRITE(SNB_DPFC_CTL_SA, SNB_CPU_FENCE_ENABLE | params->vma->fence->id); @@ -243,7 +245,7 @@ static void ilk_fbc_activate(struct drm_i915_private *dev_priv) params->crtc.fence_y_offset); } } else { - if (IS_GEN6(dev_priv)) { + if (IS_GEN(dev_priv, 6)) { I915_WRITE(SNB_DPFC_CTL_SA, 0); I915_WRITE(DPFC_CPU_FENCE_OFFSET, 0); } @@ -282,7 +284,7 @@ static void gen7_fbc_activate(struct drm_i915_private *dev_priv) int threshold = dev_priv->fbc.threshold; /* Display WA #0529: skl, kbl, bxt. */ - if (IS_GEN9(dev_priv) && !IS_GEMINILAKE(dev_priv)) { + if (IS_GEN(dev_priv, 9) && !IS_GEMINILAKE(dev_priv)) { u32 val = I915_READ(CHICKEN_MISC_4); val &= ~(FBC_STRIDE_OVERRIDE | FBC_STRIDE_MASK); @@ -581,10 +583,10 @@ static bool stride_is_valid(struct drm_i915_private *dev_priv, if (stride < 512) return false; - if (IS_GEN2(dev_priv) || IS_GEN3(dev_priv)) + if (IS_GEN(dev_priv, 2) || IS_GEN(dev_priv, 3)) return stride == 4096 || stride == 8192; - if (IS_GEN4(dev_priv) && !IS_G4X(dev_priv) && stride < 2048) + if (IS_GEN(dev_priv, 4) && !IS_G4X(dev_priv) && stride < 2048) return false; if (stride > 16384) @@ -594,7 +596,7 @@ static bool stride_is_valid(struct drm_i915_private *dev_priv, } static bool pixel_format_is_valid(struct drm_i915_private *dev_priv, - uint32_t pixel_format) + u32 pixel_format) { switch (pixel_format) { case DRM_FORMAT_XRGB8888: @@ -603,7 +605,7 @@ static bool pixel_format_is_valid(struct drm_i915_private *dev_priv, case DRM_FORMAT_XRGB1555: case DRM_FORMAT_RGB565: /* 16bpp not supported on gen2 */ - if (IS_GEN2(dev_priv)) + if (IS_GEN(dev_priv, 2)) return false; /* WaFbcOnly1to1Ratio:ctg */ if (IS_G4X(dev_priv)) @@ -626,7 +628,10 @@ static bool intel_fbc_hw_tracking_covers_screen(struct intel_crtc *crtc) struct intel_fbc *fbc = &dev_priv->fbc; unsigned int effective_w, effective_h, max_w, max_h; - if (INTEL_GEN(dev_priv) >= 8 || IS_HASWELL(dev_priv)) { + if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) { + max_w = 5120; + max_h = 4096; + } else if (INTEL_GEN(dev_priv) >= 8 || IS_HASWELL(dev_priv)) { max_w = 4096; max_h = 4096; } else if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) { @@ -674,6 +679,8 @@ static void intel_fbc_update_state_cache(struct intel_crtc *crtc, cache->plane.adjusted_y = plane_state->color_plane[0].y; cache->plane.y = plane_state->base.src.y1 >> 16; + cache->plane.pixel_blend_mode = plane_state->base.pixel_blend_mode; + if (!cache->plane.visible) return; @@ -748,6 +755,12 @@ static bool intel_fbc_can_activate(struct intel_crtc *crtc) return false; } + if (cache->plane.pixel_blend_mode != DRM_MODE_BLEND_PIXEL_NONE && + cache->fb.format->has_alpha) { + fbc->no_fbc_reason = "per-pixel alpha blending is incompatible with FBC"; + return false; + } + /* WaFbcExceedCdClockThreshold:hsw,bdw */ if ((IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) && cache->crtc.hsw_bdw_pixel_rate >= dev_priv->cdclk.hw.cdclk * 95 / 100) { @@ -776,7 +789,7 @@ static bool intel_fbc_can_activate(struct intel_crtc *crtc) * having a Y offset that isn't divisible by 4 causes FIFO underrun * and screen flicker. */ - if (IS_GEN(dev_priv, 9, 10) && + if (IS_GEN_RANGE(dev_priv, 9, 10) && (fbc->state_cache.plane.adjusted_y & 3)) { fbc->no_fbc_reason = "plane Y offset is misaligned"; return false; @@ -831,7 +844,7 @@ static void intel_fbc_get_reg_params(struct intel_crtc *crtc, params->cfb_size = intel_fbc_calculate_cfb_size(dev_priv, cache); - if (IS_GEN9(dev_priv) && !IS_GEMINILAKE(dev_priv)) + if (IS_GEN(dev_priv, 9) && !IS_GEMINILAKE(dev_priv)) params->gen9_wa_cfb_stride = DIV_ROUND_UP(cache->plane.src_w, 32 * fbc->threshold) * 8; } @@ -1118,8 +1131,6 @@ void intel_fbc_disable(struct intel_crtc *crtc) if (!fbc_supported(dev_priv)) return; - WARN_ON(crtc->active); - mutex_lock(&fbc->lock); if (fbc->crtc == crtc) __intel_fbc_disable(dev_priv); @@ -1301,7 +1312,7 @@ void intel_fbc_init(struct drm_i915_private *dev_priv) fbc->active = false; if (need_fbc_vtd_wa(dev_priv)) - mkwrite_device_info(dev_priv)->has_fbc = false; + mkwrite_device_info(dev_priv)->display.has_fbc = false; i915_modparams.enable_fbc = intel_sanitize_fbc_option(dev_priv); DRM_DEBUG_KMS("Sanitized enable_fbc value: %d\n", diff --git a/drivers/gpu/drm/i915/intel_fbdev.c b/drivers/gpu/drm/i915/intel_fbdev.c index f99332972b7a..376ffe842e26 100644 --- a/drivers/gpu/drm/i915/intel_fbdev.c +++ b/drivers/gpu/drm/i915/intel_fbdev.c @@ -37,9 +37,10 @@ #include <linux/init.h> #include <linux/vga_switcheroo.h> -#include <drm/drmP.h> #include <drm/drm_crtc.h> #include <drm/drm_fb_helper.h> +#include <drm/drm_fourcc.h> + #include "intel_drv.h" #include "intel_frontbuffer.h" #include <drm/i915_drm.h> @@ -178,8 +179,9 @@ static int intelfb_create(struct drm_fb_helper *helper, const struct i915_ggtt_view view = { .type = I915_GGTT_VIEW_NORMAL, }; - struct fb_info *info; struct drm_framebuffer *fb; + intel_wakeref_t wakeref; + struct fb_info *info; struct i915_vma *vma; unsigned long flags = 0; bool prealloc = false; @@ -210,7 +212,7 @@ static int intelfb_create(struct drm_fb_helper *helper, } mutex_lock(&dev->struct_mutex); - intel_runtime_pm_get(dev_priv); + wakeref = intel_runtime_pm_get(dev_priv); /* Pin the GGTT vma for our access via info->screen_base. * This also validates that any existing fb inherited from the @@ -277,7 +279,7 @@ static int intelfb_create(struct drm_fb_helper *helper, ifbdev->vma = vma; ifbdev->vma_flags = flags; - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, wakeref); mutex_unlock(&dev->struct_mutex); vga_switcheroo_client_fb_set(pdev, info); return 0; @@ -285,7 +287,7 @@ static int intelfb_create(struct drm_fb_helper *helper, out_unpin: intel_unpin_fb_vma(vma, flags); out_unlock: - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, wakeref); mutex_unlock(&dev->struct_mutex); return ret; } @@ -593,7 +595,7 @@ static bool intel_fbdev_init_bios(struct drm_device *dev, * pipe. Note we need to use the selected fb's pitch and bpp * rather than the current pipe's, since they differ. */ - cur_size = intel_crtc->config->base.adjusted_mode.crtc_hdisplay; + cur_size = crtc->state->adjusted_mode.crtc_hdisplay; cur_size = cur_size * fb->base.format->cpp[0]; if (fb->base.pitches[0] < cur_size) { DRM_DEBUG_KMS("fb not wide enough for plane %c (%d vs %d)\n", @@ -603,13 +605,13 @@ static bool intel_fbdev_init_bios(struct drm_device *dev, break; } - cur_size = intel_crtc->config->base.adjusted_mode.crtc_vdisplay; + cur_size = crtc->state->adjusted_mode.crtc_vdisplay; cur_size = intel_fb_align_height(&fb->base, 0, cur_size); cur_size *= fb->base.pitches[0]; DRM_DEBUG_KMS("pipe %c area: %dx%d, bpp: %d, size: %d\n", pipe_name(intel_crtc->pipe), - intel_crtc->config->base.adjusted_mode.crtc_hdisplay, - intel_crtc->config->base.adjusted_mode.crtc_vdisplay, + crtc->state->adjusted_mode.crtc_hdisplay, + crtc->state->adjusted_mode.crtc_vdisplay, fb->base.format->cpp[0] * 8, cur_size); @@ -672,13 +674,14 @@ int intel_fbdev_init(struct drm_device *dev) struct intel_fbdev *ifbdev; int ret; - if (WARN_ON(INTEL_INFO(dev_priv)->num_pipes == 0)) + if (WARN_ON(!HAS_DISPLAY(dev_priv))) return -ENODEV; ifbdev = kzalloc(sizeof(struct intel_fbdev), GFP_KERNEL); if (ifbdev == NULL) return -ENOMEM; + mutex_init(&ifbdev->hpd_lock); drm_fb_helper_prepare(dev, &ifbdev->helper, &intel_fb_helper_funcs); if (!intel_fbdev_init_bios(dev, ifbdev)) @@ -752,6 +755,26 @@ void intel_fbdev_fini(struct drm_i915_private *dev_priv) intel_fbdev_destroy(ifbdev); } +/* Suspends/resumes fbdev processing of incoming HPD events. When resuming HPD + * processing, fbdev will perform a full connector reprobe if a hotplug event + * was received while HPD was suspended. + */ +static void intel_fbdev_hpd_set_suspend(struct intel_fbdev *ifbdev, int state) +{ + bool send_hpd = false; + + mutex_lock(&ifbdev->hpd_lock); + ifbdev->hpd_suspended = state == FBINFO_STATE_SUSPENDED; + send_hpd = !ifbdev->hpd_suspended && ifbdev->hpd_waiting; + ifbdev->hpd_waiting = false; + mutex_unlock(&ifbdev->hpd_lock); + + if (send_hpd) { + DRM_DEBUG_KMS("Handling delayed fbcon HPD event\n"); + drm_fb_helper_hotplug_event(&ifbdev->helper); + } +} + void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous) { struct drm_i915_private *dev_priv = to_i915(dev); @@ -773,6 +796,7 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous */ if (state != FBINFO_STATE_RUNNING) flush_work(&dev_priv->fbdev_suspend_work); + console_lock(); } else { /* @@ -800,17 +824,26 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous drm_fb_helper_set_suspend(&ifbdev->helper, state); console_unlock(); + + intel_fbdev_hpd_set_suspend(ifbdev, state); } void intel_fbdev_output_poll_changed(struct drm_device *dev) { struct intel_fbdev *ifbdev = to_i915(dev)->fbdev; + bool send_hpd; if (!ifbdev) return; intel_fbdev_sync(ifbdev); - if (ifbdev->vma || ifbdev->helper.deferred_setup) + + mutex_lock(&ifbdev->hpd_lock); + send_hpd = !ifbdev->hpd_suspended; + ifbdev->hpd_waiting = true; + mutex_unlock(&ifbdev->hpd_lock); + + if (send_hpd && (ifbdev->vma || ifbdev->helper.deferred_setup)) drm_fb_helper_hotplug_event(&ifbdev->helper); } diff --git a/drivers/gpu/drm/i915/intel_fifo_underrun.c b/drivers/gpu/drm/i915/intel_fifo_underrun.c index 77c123cc8817..f33de4be4b89 100644 --- a/drivers/gpu/drm/i915/intel_fifo_underrun.c +++ b/drivers/gpu/drm/i915/intel_fifo_underrun.c @@ -127,8 +127,8 @@ static void ironlake_set_fifo_underrun_reporting(struct drm_device *dev, enum pipe pipe, bool enable) { struct drm_i915_private *dev_priv = to_i915(dev); - uint32_t bit = (pipe == PIPE_A) ? DE_PIPEA_FIFO_UNDERRUN : - DE_PIPEB_FIFO_UNDERRUN; + u32 bit = (pipe == PIPE_A) ? + DE_PIPEA_FIFO_UNDERRUN : DE_PIPEB_FIFO_UNDERRUN; if (enable) ilk_enable_display_irq(dev_priv, bit); @@ -140,7 +140,7 @@ static void ivybridge_check_fifo_underruns(struct intel_crtc *crtc) { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; - uint32_t err_int = I915_READ(GEN7_ERR_INT); + u32 err_int = I915_READ(GEN7_ERR_INT); lockdep_assert_held(&dev_priv->irq_lock); @@ -193,8 +193,8 @@ static void ibx_set_fifo_underrun_reporting(struct drm_device *dev, bool enable) { struct drm_i915_private *dev_priv = to_i915(dev); - uint32_t bit = (pch_transcoder == PIPE_A) ? - SDE_TRANSA_FIFO_UNDER : SDE_TRANSB_FIFO_UNDER; + u32 bit = (pch_transcoder == PIPE_A) ? + SDE_TRANSA_FIFO_UNDER : SDE_TRANSB_FIFO_UNDER; if (enable) ibx_enable_display_interrupt(dev_priv, bit); @@ -206,7 +206,7 @@ static void cpt_check_pch_fifo_underruns(struct intel_crtc *crtc) { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pch_transcoder = crtc->pipe; - uint32_t serr_int = I915_READ(SERR_INT); + u32 serr_int = I915_READ(SERR_INT); lockdep_assert_held(&dev_priv->irq_lock); @@ -258,11 +258,11 @@ static bool __intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev, old = !crtc->cpu_fifo_underrun_disabled; crtc->cpu_fifo_underrun_disabled = !enable; - if (HAS_GMCH_DISPLAY(dev_priv)) + if (HAS_GMCH(dev_priv)) i9xx_set_fifo_underrun_reporting(dev, pipe, enable, old); - else if (IS_GEN5(dev_priv) || IS_GEN6(dev_priv)) + else if (IS_GEN_RANGE(dev_priv, 5, 6)) ironlake_set_fifo_underrun_reporting(dev, pipe, enable); - else if (IS_GEN7(dev_priv)) + else if (IS_GEN(dev_priv, 7)) ivybridge_set_fifo_underrun_reporting(dev, pipe, enable, old); else if (INTEL_GEN(dev_priv) >= 8) broadwell_set_fifo_underrun_reporting(dev, pipe, enable); @@ -369,7 +369,7 @@ void intel_cpu_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv, return; /* GMCH can't disable fifo underruns, filter them. */ - if (HAS_GMCH_DISPLAY(dev_priv) && + if (HAS_GMCH(dev_priv) && crtc->cpu_fifo_underrun_disabled) return; @@ -421,9 +421,9 @@ void intel_check_cpu_fifo_underruns(struct drm_i915_private *dev_priv) if (crtc->cpu_fifo_underrun_disabled) continue; - if (HAS_GMCH_DISPLAY(dev_priv)) + if (HAS_GMCH(dev_priv)) i9xx_check_fifo_underruns(crtc); - else if (IS_GEN7(dev_priv)) + else if (IS_GEN(dev_priv, 7)) ivybridge_check_fifo_underruns(crtc); } diff --git a/drivers/gpu/drm/i915/intel_frontbuffer.c b/drivers/gpu/drm/i915/intel_frontbuffer.c index c3379bde266f..16f253deaf8d 100644 --- a/drivers/gpu/drm/i915/intel_frontbuffer.c +++ b/drivers/gpu/drm/i915/intel_frontbuffer.c @@ -60,7 +60,6 @@ * functions is deprecated and should be avoided. */ -#include <drm/drmP.h> #include "intel_drv.h" #include "intel_frontbuffer.h" diff --git a/drivers/gpu/drm/i915/intel_gpu_commands.h b/drivers/gpu/drm/i915/intel_gpu_commands.h index 105e2a9e874a..b96a31bc1080 100644 --- a/drivers/gpu/drm/i915/intel_gpu_commands.h +++ b/drivers/gpu/drm/i915/intel_gpu_commands.h @@ -112,7 +112,6 @@ #define MI_MEM_VIRTUAL (1 << 22) /* 945,g33,965 */ #define MI_USE_GGTT (1 << 22) /* g4x+ */ #define MI_STORE_DWORD_INDEX MI_INSTR(0x21, 1) -#define MI_STORE_DWORD_INDEX_SHIFT 2 /* * Official intel docs are somewhat sloppy concerning MI_LOAD_REGISTER_IMM: * - Always issue a MI_NOOP _before_ the MI_LOAD_REGISTER_IMM - otherwise hw diff --git a/drivers/gpu/drm/i915/intel_guc.c b/drivers/gpu/drm/i915/intel_guc.c index 230aea69385d..8660af3fd755 100644 --- a/drivers/gpu/drm/i915/intel_guc.c +++ b/drivers/gpu/drm/i915/intel_guc.c @@ -50,7 +50,8 @@ void intel_guc_init_send_regs(struct intel_guc *guc) unsigned int i; guc->send_regs.base = i915_mmio_reg_offset(SOFT_SCRATCH(0)); - guc->send_regs.count = SOFT_SCRATCH_COUNT - 1; + guc->send_regs.count = GUC_MAX_MMIO_MSG_LEN; + BUILD_BUG_ON(GUC_MAX_MMIO_MSG_LEN > SOFT_SCRATCH_COUNT); for (i = 0; i < guc->send_regs.count; i++) { fw_domains |= intel_uncore_forcewake_for_reg(dev_priv, @@ -521,6 +522,44 @@ int intel_guc_auth_huc(struct intel_guc *guc, u32 rsa_offset) return intel_guc_send(guc, action, ARRAY_SIZE(action)); } +/* + * The ENTER/EXIT_S_STATE actions queue the save/restore operation in GuC FW and + * then return, so waiting on the H2G is not enough to guarantee GuC is done. + * When all the processing is done, GuC writes INTEL_GUC_SLEEP_STATE_SUCCESS to + * scratch register 14, so we can poll on that. Note that GuC does not ensure + * that the value in the register is different from + * INTEL_GUC_SLEEP_STATE_SUCCESS while the action is in progress so we need to + * take care of that ourselves as well. + */ +static int guc_sleep_state_action(struct intel_guc *guc, + const u32 *action, u32 len) +{ + struct drm_i915_private *dev_priv = guc_to_i915(guc); + int ret; + u32 status; + + I915_WRITE(SOFT_SCRATCH(14), INTEL_GUC_SLEEP_STATE_INVALID_MASK); + + ret = intel_guc_send(guc, action, len); + if (ret) + return ret; + + ret = __intel_wait_for_register(dev_priv, SOFT_SCRATCH(14), + INTEL_GUC_SLEEP_STATE_INVALID_MASK, + 0, 0, 10, &status); + if (ret) + return ret; + + if (status != INTEL_GUC_SLEEP_STATE_SUCCESS) { + DRM_ERROR("GuC failed to change sleep state. " + "action=0x%x, err=%u\n", + action[0], status); + return -EIO; + } + + return 0; +} + /** * intel_guc_suspend() - notify GuC entering suspend state * @guc: the guc @@ -533,7 +572,7 @@ int intel_guc_suspend(struct intel_guc *guc) intel_guc_ggtt_offset(guc, guc->shared_data) }; - return intel_guc_send(guc, data, ARRAY_SIZE(data)); + return guc_sleep_state_action(guc, data, ARRAY_SIZE(data)); } /** @@ -571,7 +610,7 @@ int intel_guc_resume(struct intel_guc *guc) intel_guc_ggtt_offset(guc, guc->shared_data) }; - return intel_guc_send(guc, data, ARRAY_SIZE(data)); + return guc_sleep_state_action(guc, data, ARRAY_SIZE(data)); } /** diff --git a/drivers/gpu/drm/i915/intel_guc.h b/drivers/gpu/drm/i915/intel_guc.h index ad42faf48c46..744220296653 100644 --- a/drivers/gpu/drm/i915/intel_guc.h +++ b/drivers/gpu/drm/i915/intel_guc.h @@ -95,6 +95,11 @@ struct intel_guc { void (*notify)(struct intel_guc *guc); }; +static inline bool intel_guc_is_alive(struct intel_guc *guc) +{ + return intel_uc_fw_is_loaded(&guc->fw); +} + static inline int intel_guc_send(struct intel_guc *guc, const u32 *action, u32 len) { @@ -187,4 +192,7 @@ static inline void intel_guc_disable_msg(struct intel_guc *guc, u32 mask) spin_unlock_irq(&guc->irq_lock); } +int intel_guc_reset_engine(struct intel_guc *guc, + struct intel_engine_cs *engine); + #endif diff --git a/drivers/gpu/drm/i915/intel_guc_fw.c b/drivers/gpu/drm/i915/intel_guc_fw.c index a9e6fcce467c..13ff7003c6be 100644 --- a/drivers/gpu/drm/i915/intel_guc_fw.c +++ b/drivers/gpu/drm/i915/intel_guc_fw.c @@ -77,9 +77,6 @@ static void guc_fw_select(struct intel_uc_fw *guc_fw) guc_fw->path = I915_KBL_GUC_UCODE; guc_fw->major_ver_wanted = KBL_FW_MAJOR; guc_fw->minor_ver_wanted = KBL_FW_MINOR; - } else { - DRM_WARN("%s: No firmware known for this platform!\n", - intel_uc_fw_type_repr(guc_fw->type)); } } @@ -114,7 +111,7 @@ static void guc_prepare_xfer(struct intel_guc *guc) else I915_WRITE(GEN9_GT_PM_CONFIG, GT_DOORBELL_ENABLE); - if (IS_GEN9(dev_priv)) { + if (IS_GEN(dev_priv, 9)) { /* DOP Clock Gating Enable for GuC clocks */ I915_WRITE(GEN7_MISCCPCTL, (GEN8_DOP_CLOCK_GATE_GUC_ENABLE | I915_READ(GEN7_MISCCPCTL))); @@ -125,66 +122,26 @@ static void guc_prepare_xfer(struct intel_guc *guc) } /* Copy RSA signature from the fw image to HW for verification */ -static int guc_xfer_rsa(struct intel_guc *guc, struct i915_vma *vma) +static void guc_xfer_rsa(struct intel_guc *guc, struct i915_vma *vma) { struct drm_i915_private *dev_priv = guc_to_i915(guc); - struct intel_uc_fw *guc_fw = &guc->fw; - struct sg_table *sg = vma->pages; u32 rsa[UOS_RSA_SCRATCH_COUNT]; int i; - if (sg_pcopy_to_buffer(sg->sgl, sg->nents, rsa, sizeof(rsa), - guc_fw->rsa_offset) != sizeof(rsa)) - return -EINVAL; + sg_pcopy_to_buffer(vma->pages->sgl, vma->pages->nents, + rsa, sizeof(rsa), guc->fw.rsa_offset); for (i = 0; i < UOS_RSA_SCRATCH_COUNT; i++) I915_WRITE(UOS_RSA_SCRATCH(i), rsa[i]); - - return 0; } -/* - * Transfer the firmware image to RAM for execution by the microcontroller. - * - * Architecturally, the DMA engine is bidirectional, and can potentially even - * transfer between GTT locations. This functionality is left out of the API - * for now as there is no need for it. - */ -static int guc_xfer_ucode(struct intel_guc *guc, struct i915_vma *vma) +static bool guc_xfer_completed(struct intel_guc *guc, u32 *status) { struct drm_i915_private *dev_priv = guc_to_i915(guc); - struct intel_uc_fw *guc_fw = &guc->fw; - unsigned long offset; - u32 status; - int ret; - - /* - * The header plus uCode will be copied to WOPCM via DMA, excluding any - * other components - */ - I915_WRITE(DMA_COPY_SIZE, guc_fw->header_size + guc_fw->ucode_size); - - /* Set the source address for the new blob */ - offset = intel_guc_ggtt_offset(guc, vma) + guc_fw->header_offset; - I915_WRITE(DMA_ADDR_0_LOW, lower_32_bits(offset)); - I915_WRITE(DMA_ADDR_0_HIGH, upper_32_bits(offset) & 0xFFFF); - /* - * Set the DMA destination. Current uCode expects the code to be - * loaded at 8k; locations below this are used for the stack. - */ - I915_WRITE(DMA_ADDR_1_LOW, 0x2000); - I915_WRITE(DMA_ADDR_1_HIGH, DMA_ADDRESS_SPACE_WOPCM); - - /* Finally start the DMA */ - I915_WRITE(DMA_CTRL, _MASKED_BIT_ENABLE(UOS_MOVE | START_DMA)); - - /* Wait for DMA to finish */ - ret = __intel_wait_for_register_fw(dev_priv, DMA_CTRL, START_DMA, 0, - 2, 100, &status); - DRM_DEBUG_DRIVER("GuC DMA status %#x\n", status); - - return ret; + /* Did we complete the xfer? */ + *status = I915_READ(DMA_CTRL); + return !(*status & START_DMA); } /* @@ -217,8 +174,8 @@ static int guc_wait_ucode(struct intel_guc *guc) * NB: Docs recommend not using the interrupt for completion. * Measurements indicate this should take no more than 20ms, so a * timeout here indicates that the GuC has failed and is unusable. - * (Higher levels of the driver will attempt to fall back to - * execlist mode if this happens.) + * (Higher levels of the driver may decide to reset the GuC and + * attempt the ucode load again if this happens.) */ ret = wait_for(guc_ready(guc, &status), 100); DRM_DEBUG_DRIVER("GuC status %#x\n", status); @@ -228,10 +185,52 @@ static int guc_wait_ucode(struct intel_guc *guc) ret = -ENOEXEC; } + if (ret == 0 && !guc_xfer_completed(guc, &status)) { + DRM_ERROR("GuC is ready, but the xfer %08x is incomplete\n", + status); + ret = -ENXIO; + } + return ret; } /* + * Transfer the firmware image to RAM for execution by the microcontroller. + * + * Architecturally, the DMA engine is bidirectional, and can potentially even + * transfer between GTT locations. This functionality is left out of the API + * for now as there is no need for it. + */ +static int guc_xfer_ucode(struct intel_guc *guc, struct i915_vma *vma) +{ + struct drm_i915_private *dev_priv = guc_to_i915(guc); + struct intel_uc_fw *guc_fw = &guc->fw; + unsigned long offset; + + /* + * The header plus uCode will be copied to WOPCM via DMA, excluding any + * other components + */ + I915_WRITE(DMA_COPY_SIZE, guc_fw->header_size + guc_fw->ucode_size); + + /* Set the source address for the new blob */ + offset = intel_guc_ggtt_offset(guc, vma) + guc_fw->header_offset; + I915_WRITE(DMA_ADDR_0_LOW, lower_32_bits(offset)); + I915_WRITE(DMA_ADDR_0_HIGH, upper_32_bits(offset) & 0xFFFF); + + /* + * Set the DMA destination. Current uCode expects the code to be + * loaded at 8k; locations below this are used for the stack. + */ + I915_WRITE(DMA_ADDR_1_LOW, 0x2000); + I915_WRITE(DMA_ADDR_1_HIGH, DMA_ADDRESS_SPACE_WOPCM); + + /* Finally start the DMA */ + I915_WRITE(DMA_CTRL, _MASKED_BIT_ENABLE(UOS_MOVE | START_DMA)); + + return guc_wait_ucode(guc); +} +/* * Load the GuC firmware blob into the MinuteIA. */ static int guc_fw_xfer(struct intel_uc_fw *guc_fw, struct i915_vma *vma) @@ -251,17 +250,9 @@ static int guc_fw_xfer(struct intel_uc_fw *guc_fw, struct i915_vma *vma) * by the DMA engine in one operation, whereas the RSA signature is * loaded via MMIO. */ - ret = guc_xfer_rsa(guc, vma); - if (ret) - DRM_WARN("GuC firmware signature xfer error %d\n", ret); + guc_xfer_rsa(guc, vma); ret = guc_xfer_ucode(guc, vma); - if (ret) - DRM_WARN("GuC firmware code xfer error %d\n", ret); - - ret = guc_wait_ucode(guc); - if (ret) - DRM_ERROR("GuC firmware xfer error %d\n", ret); intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); diff --git a/drivers/gpu/drm/i915/intel_guc_fwif.h b/drivers/gpu/drm/i915/intel_guc_fwif.h index 8382d591c784..b2f5148f4f17 100644 --- a/drivers/gpu/drm/i915/intel_guc_fwif.h +++ b/drivers/gpu/drm/i915/intel_guc_fwif.h @@ -39,6 +39,11 @@ #define GUC_VIDEO_ENGINE2 4 #define GUC_MAX_ENGINES_NUM (GUC_VIDEO_ENGINE2 + 1) +#define GUC_DOORBELL_INVALID 256 + +#define GUC_DB_SIZE (PAGE_SIZE) +#define GUC_WQ_SIZE (PAGE_SIZE * 2) + /* Work queue item header definitions */ #define WQ_STATUS_ACTIVE 1 #define WQ_STATUS_SUSPENDED 2 @@ -59,9 +64,6 @@ #define WQ_RING_TAIL_MAX 0x7FF /* 2^11 QWords */ #define WQ_RING_TAIL_MASK (WQ_RING_TAIL_MAX << WQ_RING_TAIL_SHIFT) -#define GUC_DOORBELL_ENABLED 1 -#define GUC_DOORBELL_DISABLED 0 - #define GUC_STAGE_DESC_ATTR_ACTIVE BIT(0) #define GUC_STAGE_DESC_ATTR_PENDING_DB BIT(1) #define GUC_STAGE_DESC_ATTR_KERNEL BIT(2) @@ -219,26 +221,6 @@ struct uc_css_header { u32 header_info; } __packed; -struct guc_doorbell_info { - u32 db_status; - u32 cookie; - u32 reserved[14]; -} __packed; - -union guc_doorbell_qw { - struct { - u32 db_status; - u32 cookie; - }; - u64 value_qw; -} __packed; - -#define GUC_NUM_DOORBELLS 256 -#define GUC_DOORBELL_INVALID (GUC_NUM_DOORBELLS) - -#define GUC_DB_SIZE (PAGE_SIZE) -#define GUC_WQ_SIZE (PAGE_SIZE * 2) - /* Work item for submitting workloads into work queue of GuC. */ struct guc_wq_item { u32 header; @@ -601,7 +583,9 @@ struct guc_shared_ctx_data { * registers, where first register holds data treated as message header, * and other registers are used to hold message payload. * - * For Gen9+, GuC uses software scratch registers 0xC180-0xC1B8 + * For Gen9+, GuC uses software scratch registers 0xC180-0xC1B8, + * but no H2G command takes more than 8 parameters and the GuC FW + * itself uses an 8-element array to store the H2G message. * * +-----------+---------+---------+---------+ * | MMIO[0] | MMIO[1] | ... | MMIO[n] | @@ -633,6 +617,8 @@ struct guc_shared_ctx_data { * field. */ +#define GUC_MAX_MMIO_MSG_LEN 8 + #define INTEL_GUC_MSG_TYPE_SHIFT 28 #define INTEL_GUC_MSG_TYPE_MASK (0xF << INTEL_GUC_MSG_TYPE_SHIFT) #define INTEL_GUC_MSG_DATA_SHIFT 16 @@ -687,6 +673,13 @@ enum intel_guc_report_status { INTEL_GUC_REPORT_STATUS_COMPLETE = 0x4, }; +enum intel_guc_sleep_state_status { + INTEL_GUC_SLEEP_STATE_SUCCESS = 0x0, + INTEL_GUC_SLEEP_STATE_PREEMPT_TO_IDLE_FAILED = 0x1, + INTEL_GUC_SLEEP_STATE_ENGINE_RESET_FAILED = 0x2 +#define INTEL_GUC_SLEEP_STATE_INVALID_MASK 0x80000000 +}; + #define GUC_LOG_CONTROL_LOGGING_ENABLED (1 << 0) #define GUC_LOG_CONTROL_VERBOSITY_SHIFT 4 #define GUC_LOG_CONTROL_VERBOSITY_MASK (0xF << GUC_LOG_CONTROL_VERBOSITY_SHIFT) diff --git a/drivers/gpu/drm/i915/intel_guc_log.c b/drivers/gpu/drm/i915/intel_guc_log.c index d3ebdbc0182e..806fdfd7c78a 100644 --- a/drivers/gpu/drm/i915/intel_guc_log.c +++ b/drivers/gpu/drm/i915/intel_guc_log.c @@ -140,6 +140,9 @@ static struct dentry *create_buf_file_callback(const char *filename, buf_file = debugfs_create_file(filename, mode, parent, buf, &relay_file_operations); + if (IS_ERR(buf_file)) + return NULL; + return buf_file; } @@ -436,6 +439,7 @@ static void guc_log_capture_logs(struct intel_guc_log *log) { struct intel_guc *guc = log_to_guc(log); struct drm_i915_private *dev_priv = guc_to_i915(guc); + intel_wakeref_t wakeref; guc_read_update_log_buffer(log); @@ -443,9 +447,8 @@ static void guc_log_capture_logs(struct intel_guc_log *log) * Generally device is expected to be active only at this * time, so get/put should be really quick. */ - intel_runtime_pm_get(dev_priv); - guc_action_flush_log_complete(guc); - intel_runtime_pm_put(dev_priv); + with_intel_runtime_pm(dev_priv, wakeref) + guc_action_flush_log_complete(guc); } int intel_guc_log_create(struct intel_guc_log *log) @@ -505,7 +508,8 @@ int intel_guc_log_set_level(struct intel_guc_log *log, u32 level) { struct intel_guc *guc = log_to_guc(log); struct drm_i915_private *dev_priv = guc_to_i915(guc); - int ret; + intel_wakeref_t wakeref; + int ret = 0; BUILD_BUG_ON(GUC_LOG_VERBOSITY_MIN != 0); GEM_BUG_ON(!log->vma); @@ -519,16 +523,14 @@ int intel_guc_log_set_level(struct intel_guc_log *log, u32 level) mutex_lock(&dev_priv->drm.struct_mutex); - if (log->level == level) { - ret = 0; + if (log->level == level) goto out_unlock; - } - intel_runtime_pm_get(dev_priv); - ret = guc_action_control_log(guc, GUC_LOG_LEVEL_IS_VERBOSE(level), - GUC_LOG_LEVEL_IS_ENABLED(level), - GUC_LOG_LEVEL_TO_VERBOSITY(level)); - intel_runtime_pm_put(dev_priv); + with_intel_runtime_pm(dev_priv, wakeref) + ret = guc_action_control_log(guc, + GUC_LOG_LEVEL_IS_VERBOSE(level), + GUC_LOG_LEVEL_IS_ENABLED(level), + GUC_LOG_LEVEL_TO_VERBOSITY(level)); if (ret) { DRM_DEBUG_DRIVER("guc_log_control action failed %d\n", ret); goto out_unlock; @@ -601,6 +603,7 @@ void intel_guc_log_relay_flush(struct intel_guc_log *log) { struct intel_guc *guc = log_to_guc(log); struct drm_i915_private *i915 = guc_to_i915(guc); + intel_wakeref_t wakeref; /* * Before initiating the forceful flush, wait for any pending/ongoing @@ -608,9 +611,8 @@ void intel_guc_log_relay_flush(struct intel_guc_log *log) */ flush_work(&log->relay.flush_work); - intel_runtime_pm_get(i915); - guc_action_flush_log(guc); - intel_runtime_pm_put(i915); + with_intel_runtime_pm(i915, wakeref) + guc_action_flush_log(guc); /* GuC would have updated log buffer by now, so capture it */ guc_log_capture_logs(log); diff --git a/drivers/gpu/drm/i915/intel_guc_reg.h b/drivers/gpu/drm/i915/intel_guc_reg.h index d86084742a4a..57e7ad522c2f 100644 --- a/drivers/gpu/drm/i915/intel_guc_reg.h +++ b/drivers/gpu/drm/i915/intel_guc_reg.h @@ -104,6 +104,18 @@ #define GUC_SEND_INTERRUPT _MMIO(0xc4c8) #define GUC_SEND_TRIGGER (1<<0) +#define GUC_NUM_DOORBELLS 256 + +/* format of the HW-monitored doorbell cacheline */ +struct guc_doorbell_info { + u32 db_status; +#define GUC_DOORBELL_DISABLED 0 +#define GUC_DOORBELL_ENABLED 1 + + u32 cookie; + u32 reserved[14]; +} __packed; + #define GEN8_DRBREGL(x) _MMIO(0x1000 + (x) * 8) #define GEN8_DRB_VALID (1<<0) #define GEN8_DRBREGU(x) _MMIO(0x1000 + (x) * 8 + 4) diff --git a/drivers/gpu/drm/i915/intel_guc_submission.c b/drivers/gpu/drm/i915/intel_guc_submission.c index a81f04d46e87..8bc8aa54aa35 100644 --- a/drivers/gpu/drm/i915/intel_guc_submission.c +++ b/drivers/gpu/drm/i915/intel_guc_submission.c @@ -81,6 +81,12 @@ * */ +static inline u32 intel_hws_preempt_done_address(struct intel_engine_cs *engine) +{ + return (i915_ggtt_offset(engine->status_page.vma) + + I915_GEM_HWS_PREEMPT_ADDR); +} + static inline struct i915_priolist *to_priolist(struct rb_node *rb) { return rb_entry(rb, struct i915_priolist, node); @@ -192,7 +198,15 @@ static struct guc_doorbell_info *__get_doorbell(struct intel_guc_client *client) return client->vaddr + client->doorbell_offset; } -static void __create_doorbell(struct intel_guc_client *client) +static bool __doorbell_valid(struct intel_guc *guc, u16 db_id) +{ + struct drm_i915_private *dev_priv = guc_to_i915(guc); + + GEM_BUG_ON(db_id >= GUC_NUM_DOORBELLS); + return I915_READ(GEN8_DRBREGL(db_id)) & GEN8_DRB_VALID; +} + +static void __init_doorbell(struct intel_guc_client *client) { struct guc_doorbell_info *doorbell; @@ -201,21 +215,19 @@ static void __create_doorbell(struct intel_guc_client *client) doorbell->cookie = 0; } -static void __destroy_doorbell(struct intel_guc_client *client) +static void __fini_doorbell(struct intel_guc_client *client) { - struct drm_i915_private *dev_priv = guc_to_i915(client->guc); struct guc_doorbell_info *doorbell; u16 db_id = client->doorbell_id; doorbell = __get_doorbell(client); doorbell->db_status = GUC_DOORBELL_DISABLED; - doorbell->cookie = 0; /* Doorbell release flow requires that we wait for GEN8_DRB_VALID bit * to go to zero after updating db_status before we call the GuC to * release the doorbell */ - if (wait_for_us(!(I915_READ(GEN8_DRBREGL(db_id)) & GEN8_DRB_VALID), 10)) + if (wait_for_us(!__doorbell_valid(client->guc, db_id), 10)) WARN_ONCE(true, "Doorbell never became invalid after disable\n"); } @@ -227,11 +239,11 @@ static int create_doorbell(struct intel_guc_client *client) return -ENODEV; /* internal setup error, should never happen */ __update_doorbell_desc(client, client->doorbell_id); - __create_doorbell(client); + __init_doorbell(client); ret = __guc_allocate_doorbell(client->guc, client->stage_id); if (ret) { - __destroy_doorbell(client); + __fini_doorbell(client); __update_doorbell_desc(client, GUC_DOORBELL_INVALID); DRM_DEBUG_DRIVER("Couldn't create client %u doorbell: %d\n", client->stage_id, ret); @@ -247,7 +259,7 @@ static int destroy_doorbell(struct intel_guc_client *client) GEM_BUG_ON(!has_doorbell(client)); - __destroy_doorbell(client); + __fini_doorbell(client); ret = __guc_deallocate_doorbell(client->guc, client->stage_id); if (ret) DRM_ERROR("Couldn't destroy client %u doorbell: %d\n", @@ -282,8 +294,7 @@ __get_process_desc(struct intel_guc_client *client) /* * Initialise the process descriptor shared with the GuC firmware. */ -static void guc_proc_desc_init(struct intel_guc *guc, - struct intel_guc_client *client) +static void guc_proc_desc_init(struct intel_guc_client *client) { struct guc_process_desc *desc; @@ -304,6 +315,14 @@ static void guc_proc_desc_init(struct intel_guc *guc, desc->priority = client->priority; } +static void guc_proc_desc_fini(struct intel_guc_client *client) +{ + struct guc_process_desc *desc; + + desc = __get_process_desc(client); + memset(desc, 0, sizeof(*desc)); +} + static int guc_stage_desc_pool_create(struct intel_guc *guc) { struct i915_vma *vma; @@ -341,9 +360,9 @@ static void guc_stage_desc_pool_destroy(struct intel_guc *guc) * data structures relating to this client (doorbell, process descriptor, * write queue, etc). */ -static void guc_stage_desc_init(struct intel_guc *guc, - struct intel_guc_client *client) +static void guc_stage_desc_init(struct intel_guc_client *client) { + struct intel_guc *guc = client->guc; struct drm_i915_private *dev_priv = guc_to_i915(guc); struct intel_engine_cs *engine; struct i915_gem_context *ctx = client->owner; @@ -424,8 +443,7 @@ static void guc_stage_desc_init(struct intel_guc *guc, desc->desc_private = ptr_to_u64(client); } -static void guc_stage_desc_fini(struct intel_guc *guc, - struct intel_guc_client *client) +static void guc_stage_desc_fini(struct intel_guc_client *client) { struct guc_stage_desc *desc; @@ -486,14 +504,6 @@ static void guc_wq_item_append(struct intel_guc_client *client, WRITE_ONCE(desc->tail, (wq_off + wqi_size) & (GUC_WQ_SIZE - 1)); } -static void guc_reset_wq(struct intel_guc_client *client) -{ - struct guc_process_desc *desc = __get_process_desc(client); - - desc->head = 0; - desc->tail = 0; -} - static void guc_ring_doorbell(struct intel_guc_client *client) { struct guc_doorbell_info *db; @@ -568,7 +578,8 @@ static void inject_preempt_context(struct work_struct *work) if (engine->id == RCS) { cs = gen8_emit_ggtt_write_rcs(cs, GUC_PREEMPT_FINISHED, - addr); + addr, + PIPE_CONTROL_CS_STALL); } else { cs = gen8_emit_ggtt_write(cs, GUC_PREEMPT_FINISHED, @@ -618,6 +629,8 @@ static void inject_preempt_context(struct work_struct *work) EXECLISTS_ACTIVE_PREEMPT); tasklet_schedule(&engine->execlists.tasklet); } + + (void)I915_SELFTEST_ONLY(engine->execlists.preempt_hang.count++); } /* @@ -661,7 +674,7 @@ static void complete_preempt_context(struct intel_engine_cs *engine) execlists_unwind_incomplete_requests(execlists); wait_for_guc_preempt_report(engine); - intel_write_status_page(engine, I915_GEM_HWS_PREEMPT_INDEX, 0); + intel_write_status_page(engine, I915_GEM_HWS_PREEMPT, 0); } /** @@ -726,7 +739,7 @@ static bool __guc_dequeue(struct intel_engine_cs *engine) if (intel_engine_has_preemption(engine)) { struct guc_preempt_work *preempt_work = &engine->i915->guc.preempt_work[engine->id]; - int prio = execlists->queue_priority; + int prio = execlists->queue_priority_hint; if (__execlists_need_preempt(prio, port_prio(port))) { execlists_set_active(execlists, @@ -746,35 +759,34 @@ static bool __guc_dequeue(struct intel_engine_cs *engine) while ((rb = rb_first_cached(&execlists->queue))) { struct i915_priolist *p = to_priolist(rb); struct i915_request *rq, *rn; + int i; - list_for_each_entry_safe(rq, rn, &p->requests, sched.link) { + priolist_for_each_request_consume(rq, rn, p, i) { if (last && rq->hw_context != last->hw_context) { - if (port == last_port) { - __list_del_many(&p->requests, - &rq->sched.link); + if (port == last_port) goto done; - } if (submit) port_assign(port, last); port++; } - INIT_LIST_HEAD(&rq->sched.link); + list_del_init(&rq->sched.link); __i915_request_submit(rq); trace_i915_request_in(rq, port_index(port, execlists)); + last = rq; submit = true; } rb_erase_cached(&p->node, &execlists->queue); - INIT_LIST_HEAD(&p->requests); if (p->priority != I915_PRIORITY_NORMAL) kmem_cache_free(engine->i915->priorities, p); } done: - execlists->queue_priority = rb ? to_priolist(rb)->priority : INT_MIN; + execlists->queue_priority_hint = + rb ? to_priolist(rb)->priority : INT_MIN; if (submit) port_assign(port, last); if (last) @@ -791,19 +803,8 @@ done: static void guc_dequeue(struct intel_engine_cs *engine) { - unsigned long flags; - bool submit; - - local_irq_save(flags); - - spin_lock(&engine->timeline.lock); - submit = __guc_dequeue(engine); - spin_unlock(&engine->timeline.lock); - - if (submit) + if (__guc_dequeue(engine)) guc_submit(engine); - - local_irq_restore(flags); } static void guc_submission_tasklet(unsigned long data) @@ -812,6 +813,9 @@ static void guc_submission_tasklet(unsigned long data) struct intel_engine_execlists * const execlists = &engine->execlists; struct execlist_port *port = execlists->port; struct i915_request *rq; + unsigned long flags; + + spin_lock_irqsave(&engine->timeline.lock, flags); rq = port_request(port); while (rq && i915_request_completed(rq)) { @@ -829,16 +833,17 @@ static void guc_submission_tasklet(unsigned long data) } if (execlists_is_active(execlists, EXECLISTS_ACTIVE_PREEMPT) && - intel_read_status_page(engine, I915_GEM_HWS_PREEMPT_INDEX) == + intel_read_status_page(engine, I915_GEM_HWS_PREEMPT) == GUC_PREEMPT_FINISHED) complete_preempt_context(engine); if (!execlists_is_active(execlists, EXECLISTS_ACTIVE_PREEMPT)) guc_dequeue(engine); + + spin_unlock_irqrestore(&engine->timeline.lock, flags); } -static struct i915_request * -guc_reset_prepare(struct intel_engine_cs *engine) +static void guc_reset_prepare(struct intel_engine_cs *engine) { struct intel_engine_execlists * const execlists = &engine->execlists; @@ -864,8 +869,6 @@ guc_reset_prepare(struct intel_engine_cs *engine) */ if (engine->i915->guc.preempt_wq) flush_workqueue(engine->i915->guc.preempt_wq); - - return i915_gem_find_active_request(engine); } /* @@ -877,72 +880,31 @@ guc_reset_prepare(struct intel_engine_cs *engine) /* Check that a doorbell register is in the expected state */ static bool doorbell_ok(struct intel_guc *guc, u16 db_id) { - struct drm_i915_private *dev_priv = guc_to_i915(guc); - u32 drbregl; bool valid; - GEM_BUG_ON(db_id >= GUC_DOORBELL_INVALID); + GEM_BUG_ON(db_id >= GUC_NUM_DOORBELLS); - drbregl = I915_READ(GEN8_DRBREGL(db_id)); - valid = drbregl & GEN8_DRB_VALID; + valid = __doorbell_valid(guc, db_id); if (test_bit(db_id, guc->doorbell_bitmap) == valid) return true; - DRM_DEBUG_DRIVER("Doorbell %d has unexpected state (0x%x): valid=%s\n", - db_id, drbregl, yesno(valid)); + DRM_DEBUG_DRIVER("Doorbell %u has unexpected state: valid=%s\n", + db_id, yesno(valid)); return false; } static bool guc_verify_doorbells(struct intel_guc *guc) { + bool doorbells_ok = true; u16 db_id; for (db_id = 0; db_id < GUC_NUM_DOORBELLS; ++db_id) if (!doorbell_ok(guc, db_id)) - return false; + doorbells_ok = false; - return true; -} - -static int guc_clients_doorbell_init(struct intel_guc *guc) -{ - int ret; - - ret = create_doorbell(guc->execbuf_client); - if (ret) - return ret; - - if (guc->preempt_client) { - ret = create_doorbell(guc->preempt_client); - if (ret) { - destroy_doorbell(guc->execbuf_client); - return ret; - } - } - - return 0; -} - -static void guc_clients_doorbell_fini(struct intel_guc *guc) -{ - /* - * By the time we're here, GuC has already been reset. - * Instead of trying (in vain) to communicate with it, let's just - * cleanup the doorbell HW and our internal state. - */ - if (guc->preempt_client) { - __destroy_doorbell(guc->preempt_client); - __update_doorbell_desc(guc->preempt_client, - GUC_DOORBELL_INVALID); - } - - if (guc->execbuf_client) { - __destroy_doorbell(guc->execbuf_client); - __update_doorbell_desc(guc->execbuf_client, - GUC_DOORBELL_INVALID); - } + return doorbells_ok; } /** @@ -1005,6 +967,10 @@ guc_client_alloc(struct drm_i915_private *dev_priv, } client->vaddr = vaddr; + ret = reserve_doorbell(client); + if (ret) + goto err_vaddr; + client->doorbell_offset = __select_cacheline(guc); /* @@ -1017,13 +983,6 @@ guc_client_alloc(struct drm_i915_private *dev_priv, else client->proc_desc_offset = (GUC_DB_SIZE / 2); - guc_proc_desc_init(guc, client); - guc_stage_desc_init(guc, client); - - ret = reserve_doorbell(client); - if (ret) - goto err_vaddr; - DRM_DEBUG_DRIVER("new priority %u client %p for engine(s) 0x%x: stage_id %u\n", priority, client, client->engines, client->stage_id); DRM_DEBUG_DRIVER("doorbell id %u, cacheline offset 0x%lx\n", @@ -1045,7 +1004,6 @@ err_client: static void guc_client_free(struct intel_guc_client *client) { unreserve_doorbell(client); - guc_stage_desc_fini(client->guc, client); i915_vma_unpin_and_release(&client->vma, I915_VMA_RELEASE_MAP); ida_simple_remove(&client->guc->stage_ids, client->stage_id); kfree(client); @@ -1112,6 +1070,69 @@ static void guc_clients_destroy(struct intel_guc *guc) guc_client_free(client); } +static int __guc_client_enable(struct intel_guc_client *client) +{ + int ret; + + guc_proc_desc_init(client); + guc_stage_desc_init(client); + + ret = create_doorbell(client); + if (ret) + goto fail; + + return 0; + +fail: + guc_stage_desc_fini(client); + guc_proc_desc_fini(client); + return ret; +} + +static void __guc_client_disable(struct intel_guc_client *client) +{ + /* + * By the time we're here, GuC may have already been reset. if that is + * the case, instead of trying (in vain) to communicate with it, let's + * just cleanup the doorbell HW and our internal state. + */ + if (intel_guc_is_alive(client->guc)) + destroy_doorbell(client); + else + __fini_doorbell(client); + + guc_stage_desc_fini(client); + guc_proc_desc_fini(client); +} + +static int guc_clients_enable(struct intel_guc *guc) +{ + int ret; + + ret = __guc_client_enable(guc->execbuf_client); + if (ret) + return ret; + + if (guc->preempt_client) { + ret = __guc_client_enable(guc->preempt_client); + if (ret) { + __guc_client_disable(guc->execbuf_client); + return ret; + } + } + + return 0; +} + +static void guc_clients_disable(struct intel_guc *guc) +{ + if (guc->preempt_client) + __guc_client_disable(guc->preempt_client); + + if (guc->execbuf_client) + __guc_client_disable(guc->execbuf_client); +} + /* * Set up the memory resources to be shared with the GuC (via the GGTT) * at firmware loading time. @@ -1295,15 +1316,11 @@ int intel_guc_submission_enable(struct intel_guc *guc) GEM_BUG_ON(!guc->execbuf_client); - guc_reset_wq(guc->execbuf_client); - if (guc->preempt_client) - guc_reset_wq(guc->preempt_client); - err = intel_guc_sample_forcewake(guc); if (err) return err; - err = guc_clients_doorbell_init(guc); + err = guc_clients_enable(guc); if (err) return err; @@ -1325,7 +1342,7 @@ void intel_guc_submission_disable(struct intel_guc *guc) GEM_BUG_ON(dev_priv->gt.awake); /* GT should be parked first */ guc_interrupts_release(dev_priv); - guc_clients_doorbell_fini(guc); + guc_clients_disable(guc); } #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) diff --git a/drivers/gpu/drm/i915/intel_gvt.c b/drivers/gpu/drm/i915/intel_gvt.c index c22b3e18a0f5..1d7d26e4cf14 100644 --- a/drivers/gpu/drm/i915/intel_gvt.c +++ b/drivers/gpu/drm/i915/intel_gvt.c @@ -49,6 +49,9 @@ static bool is_supported_device(struct drm_i915_private *dev_priv) return true; if (IS_BROXTON(dev_priv)) return true; + if (IS_COFFEELAKE(dev_priv)) + return true; + return false; } @@ -105,15 +108,6 @@ int intel_gvt_init(struct drm_i915_private *dev_priv) return -EIO; } - /* - * We're not in host or fail to find a MPT module, disable GVT-g - */ - ret = intel_gvt_init_host(); - if (ret) { - DRM_DEBUG_DRIVER("Not in host or MPT modules not found\n"); - goto bail; - } - ret = intel_gvt_init_device(dev_priv); if (ret) { DRM_DEBUG_DRIVER("Fail to init GVT device\n"); diff --git a/drivers/gpu/drm/i915/intel_hangcheck.c b/drivers/gpu/drm/i915/intel_hangcheck.c index e26d05a46451..a219c796e56d 100644 --- a/drivers/gpu/drm/i915/intel_hangcheck.c +++ b/drivers/gpu/drm/i915/intel_hangcheck.c @@ -23,144 +23,18 @@ */ #include "i915_drv.h" +#include "i915_reset.h" -static bool -ipehr_is_semaphore_wait(struct intel_engine_cs *engine, u32 ipehr) -{ - ipehr &= ~MI_SEMAPHORE_SYNC_MASK; - return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE | - MI_SEMAPHORE_REGISTER); -} - -static struct intel_engine_cs * -semaphore_wait_to_signaller_ring(struct intel_engine_cs *engine, u32 ipehr, - u64 offset) -{ - struct drm_i915_private *dev_priv = engine->i915; - u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK; - struct intel_engine_cs *signaller; - enum intel_engine_id id; - - for_each_engine(signaller, dev_priv, id) { - if (engine == signaller) - continue; - - if (sync_bits == signaller->semaphore.mbox.wait[engine->hw_id]) - return signaller; - } - - DRM_DEBUG_DRIVER("No signaller ring found for %s, ipehr 0x%08x\n", - engine->name, ipehr); - - return ERR_PTR(-ENODEV); -} - -static struct intel_engine_cs * -semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno) -{ - struct drm_i915_private *dev_priv = engine->i915; - void __iomem *vaddr; - u32 cmd, ipehr, head; - u64 offset = 0; - int i, backwards; - - /* - * This function does not support execlist mode - any attempt to - * proceed further into this function will result in a kernel panic - * when dereferencing ring->buffer, which is not set up in execlist - * mode. - * - * The correct way of doing it would be to derive the currently - * executing ring buffer from the current context, which is derived - * from the currently running request. Unfortunately, to get the - * current request we would have to grab the struct_mutex before doing - * anything else, which would be ill-advised since some other thread - * might have grabbed it already and managed to hang itself, causing - * the hang checker to deadlock. - * - * Therefore, this function does not support execlist mode in its - * current form. Just return NULL and move on. - */ - if (engine->buffer == NULL) - return NULL; - - ipehr = I915_READ(RING_IPEHR(engine->mmio_base)); - if (!ipehr_is_semaphore_wait(engine, ipehr)) - return NULL; - - /* - * HEAD is likely pointing to the dword after the actual command, - * so scan backwards until we find the MBOX. But limit it to just 3 - * or 4 dwords depending on the semaphore wait command size. - * Note that we don't care about ACTHD here since that might - * point at at batch, and semaphores are always emitted into the - * ringbuffer itself. - */ - head = I915_READ_HEAD(engine) & HEAD_ADDR; - backwards = (INTEL_GEN(dev_priv) >= 8) ? 5 : 4; - vaddr = (void __iomem *)engine->buffer->vaddr; - - for (i = backwards; i; --i) { - /* - * Be paranoid and presume the hw has gone off into the wild - - * our ring is smaller than what the hardware (and hence - * HEAD_ADDR) allows. Also handles wrap-around. - */ - head &= engine->buffer->size - 1; - - /* This here seems to blow up */ - cmd = ioread32(vaddr + head); - if (cmd == ipehr) - break; - - head -= 4; - } - - if (!i) - return NULL; - - *seqno = ioread32(vaddr + head + 4) + 1; - return semaphore_wait_to_signaller_ring(engine, ipehr, offset); -} - -static int semaphore_passed(struct intel_engine_cs *engine) -{ - struct drm_i915_private *dev_priv = engine->i915; - struct intel_engine_cs *signaller; +struct hangcheck { + u64 acthd; u32 seqno; - - engine->hangcheck.deadlock++; - - signaller = semaphore_waits_for(engine, &seqno); - if (signaller == NULL) - return -1; - - if (IS_ERR(signaller)) - return 0; - - /* Prevent pathological recursion due to driver bugs */ - if (signaller->hangcheck.deadlock >= I915_NUM_ENGINES) - return -1; - - if (intel_engine_signaled(signaller, seqno)) - return 1; - - /* cursory check for an unkickable deadlock */ - if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE && - semaphore_passed(signaller) < 0) - return -1; - - return 0; -} - -static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv) -{ - struct intel_engine_cs *engine; - enum intel_engine_id id; - - for_each_engine(engine, dev_priv, id) - engine->hangcheck.deadlock = 0; -} + enum intel_engine_hangcheck_action action; + unsigned long action_timestamp; + int deadlock; + struct intel_instdone instdone; + bool wedged:1; + bool stalled:1; +}; static bool instdone_unchanged(u32 current_instdone, u32 *old_instdone) { @@ -236,7 +110,7 @@ engine_stuck(struct intel_engine_cs *engine, u64 acthd) if (ha != ENGINE_DEAD) return ha; - if (IS_GEN2(dev_priv)) + if (IS_GEN(dev_priv, 2)) return ENGINE_DEAD; /* Is the chip hanging on a WAIT_FOR_EVENT? @@ -252,54 +126,26 @@ engine_stuck(struct intel_engine_cs *engine, u64 acthd) return ENGINE_WAIT_KICK; } - if (IS_GEN(dev_priv, 6, 7) && tmp & RING_WAIT_SEMAPHORE) { - switch (semaphore_passed(engine)) { - default: - return ENGINE_DEAD; - case 1: - i915_handle_error(dev_priv, ALL_ENGINES, 0, - "stuck semaphore on %s", - engine->name); - I915_WRITE_CTL(engine, tmp); - return ENGINE_WAIT_KICK; - case 0: - return ENGINE_WAIT; - } - } - return ENGINE_DEAD; } static void hangcheck_load_sample(struct intel_engine_cs *engine, - struct intel_engine_hangcheck *hc) + struct hangcheck *hc) { - /* We don't strictly need an irq-barrier here, as we are not - * serving an interrupt request, be paranoid in case the - * barrier has side-effects (such as preventing a broken - * cacheline snoop) and so be sure that we can see the seqno - * advance. If the seqno should stick, due to a stale - * cacheline, we would erroneously declare the GPU hung. - */ - if (engine->irq_seqno_barrier) - engine->irq_seqno_barrier(engine); - hc->acthd = intel_engine_get_active_head(engine); hc->seqno = intel_engine_get_seqno(engine); } static void hangcheck_store_sample(struct intel_engine_cs *engine, - const struct intel_engine_hangcheck *hc) + const struct hangcheck *hc) { engine->hangcheck.acthd = hc->acthd; engine->hangcheck.seqno = hc->seqno; - engine->hangcheck.action = hc->action; - engine->hangcheck.stalled = hc->stalled; - engine->hangcheck.wedged = hc->wedged; } static enum intel_engine_hangcheck_action hangcheck_get_action(struct intel_engine_cs *engine, - const struct intel_engine_hangcheck *hc) + const struct hangcheck *hc) { if (engine->hangcheck.seqno != hc->seqno) return ENGINE_ACTIVE_SEQNO; @@ -311,7 +157,7 @@ hangcheck_get_action(struct intel_engine_cs *engine, } static void hangcheck_accumulate_sample(struct intel_engine_cs *engine, - struct intel_engine_hangcheck *hc) + struct hangcheck *hc) { unsigned long timeout = I915_ENGINE_DEAD_TIMEOUT; @@ -357,10 +203,6 @@ static void hangcheck_accumulate_sample(struct intel_engine_cs *engine, break; case ENGINE_DEAD: - if (GEM_SHOW_DEBUG()) { - struct drm_printer p = drm_debug_printer("hangcheck"); - intel_engine_dump(engine, &p, "%s\n", engine->name); - } break; default: @@ -431,24 +273,35 @@ static void i915_hangcheck_elapsed(struct work_struct *work) intel_uncore_arm_unclaimed_mmio_detection(dev_priv); for_each_engine(engine, dev_priv, id) { - struct intel_engine_hangcheck hc; + struct hangcheck hc; - semaphore_clear_deadlocks(dev_priv); + intel_engine_signal_breadcrumbs(engine); hangcheck_load_sample(engine, &hc); hangcheck_accumulate_sample(engine, &hc); hangcheck_store_sample(engine, &hc); - if (engine->hangcheck.stalled) { + if (hc.stalled) { hung |= intel_engine_flag(engine); if (hc.action != ENGINE_DEAD) stuck |= intel_engine_flag(engine); } - if (engine->hangcheck.wedged) + if (hc.wedged) wedged |= intel_engine_flag(engine); } + if (GEM_SHOW_DEBUG() && (hung | stuck)) { + struct drm_printer p = drm_debug_printer("hangcheck"); + + for_each_engine(engine, dev_priv, id) { + if (intel_engine_is_idle(engine)) + continue; + + intel_engine_dump(engine, &p, "%s\n", engine->name); + } + } + if (wedged) { dev_err(dev_priv->drm.dev, "GPU recovery timed out," diff --git a/drivers/gpu/drm/i915/intel_hdcp.c b/drivers/gpu/drm/i915/intel_hdcp.c index 26e48fc95543..ce7ba3a9c000 100644 --- a/drivers/gpu/drm/i915/intel_hdcp.c +++ b/drivers/gpu/drm/i915/intel_hdcp.c @@ -6,7 +6,6 @@ * Sean Paul <seanpaul@chromium.org> */ -#include <drm/drmP.h> #include <drm/drm_hdcp.h> #include <linux/i2c.h> #include <linux/random.h> @@ -15,6 +14,63 @@ #include "i915_reg.h" #define KEY_LOAD_TRIES 5 +#define ENCRYPT_STATUS_CHANGE_TIMEOUT_MS 50 + +static +bool intel_hdcp_is_ksv_valid(u8 *ksv) +{ + int i, ones = 0; + /* KSV has 20 1's and 20 0's */ + for (i = 0; i < DRM_HDCP_KSV_LEN; i++) + ones += hweight8(ksv[i]); + if (ones != 20) + return false; + + return true; +} + +static +int intel_hdcp_read_valid_bksv(struct intel_digital_port *intel_dig_port, + const struct intel_hdcp_shim *shim, u8 *bksv) +{ + int ret, i, tries = 2; + + /* HDCP spec states that we must retry the bksv if it is invalid */ + for (i = 0; i < tries; i++) { + ret = shim->read_bksv(intel_dig_port, bksv); + if (ret) + return ret; + if (intel_hdcp_is_ksv_valid(bksv)) + break; + } + if (i == tries) { + DRM_DEBUG_KMS("Bksv is invalid\n"); + return -ENODEV; + } + + return 0; +} + +/* Is HDCP1.4 capable on Platform and Sink */ +bool intel_hdcp_capable(struct intel_connector *connector) +{ + struct intel_digital_port *intel_dig_port = conn_to_dig_port(connector); + const struct intel_hdcp_shim *shim = connector->hdcp.shim; + bool capable = false; + u8 bksv[5]; + + if (!shim) + return capable; + + if (shim->hdcp_capable) { + shim->hdcp_capable(intel_dig_port, &capable); + } else { + if (!intel_hdcp_read_valid_bksv(intel_dig_port, shim, bksv)) + capable = true; + } + + return capable; +} static int intel_hdcp_poll_ksv_fifo(struct intel_digital_port *intel_dig_port, const struct intel_hdcp_shim *shim) @@ -101,10 +157,11 @@ static int intel_hdcp_load_keys(struct drm_i915_private *dev_priv) /* * Initiate loading the HDCP key from fuses. * - * BXT+ platforms, HDCP key needs to be loaded by SW. Only SKL and KBL - * differ in the key load trigger process from other platforms. + * BXT+ platforms, HDCP key needs to be loaded by SW. Only Gen 9 + * platforms except BXT and GLK, differ in the key load trigger process + * from other platforms. So GEN9_BC uses the GT Driver Mailbox i/f. */ - if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { + if (IS_GEN9_BC(dev_priv)) { mutex_lock(&dev_priv->pcu_lock); ret = sandybridge_pcode_write(dev_priv, SKL_PCODE_LOAD_HDCP_KEYS, 1); @@ -168,18 +225,6 @@ u32 intel_hdcp_get_repeater_ctl(struct intel_digital_port *intel_dig_port) } static -bool intel_hdcp_is_ksv_valid(u8 *ksv) -{ - int i, ones = 0; - /* KSV has 20 1's and 20 0's */ - for (i = 0; i < DRM_HDCP_KSV_LEN; i++) - ones += hweight8(ksv[i]); - if (ones != 20) - return false; - return true; -} - -static int intel_hdcp_validate_v_prime(struct intel_digital_port *intel_dig_port, const struct intel_hdcp_shim *shim, u8 *ksv_fifo, u8 num_downstream, u8 *bstatus) @@ -383,7 +428,7 @@ int intel_hdcp_validate_v_prime(struct intel_digital_port *intel_dig_port, if (intel_wait_for_register(dev_priv, HDCP_REP_CTL, HDCP_SHA1_COMPLETE, HDCP_SHA1_COMPLETE, 1)) { - DRM_DEBUG_KMS("Timed out waiting for SHA1 complete\n"); + DRM_ERROR("Timed out waiting for SHA1 complete\n"); return -ETIMEDOUT; } if (!(I915_READ(HDCP_REP_CTL) & HDCP_SHA1_V_MATCH)) { @@ -404,7 +449,7 @@ int intel_hdcp_auth_downstream(struct intel_digital_port *intel_dig_port, ret = intel_hdcp_poll_ksv_fifo(intel_dig_port, shim); if (ret) { - DRM_ERROR("KSV list failed to become ready (%d)\n", ret); + DRM_DEBUG_KMS("KSV list failed to become ready (%d)\n", ret); return ret; } @@ -414,7 +459,7 @@ int intel_hdcp_auth_downstream(struct intel_digital_port *intel_dig_port, if (DRM_HDCP_MAX_DEVICE_EXCEEDED(bstatus[0]) || DRM_HDCP_MAX_CASCADE_EXCEEDED(bstatus[1])) { - DRM_ERROR("Max Topology Limit Exceeded\n"); + DRM_DEBUG_KMS("Max Topology Limit Exceeded\n"); return -EPERM; } @@ -450,7 +495,7 @@ int intel_hdcp_auth_downstream(struct intel_digital_port *intel_dig_port, } if (i == tries) { - DRM_ERROR("V Prime validation failed.(%d)\n", ret); + DRM_DEBUG_KMS("V Prime validation failed.(%d)\n", ret); goto err; } @@ -499,7 +544,7 @@ static int intel_hdcp_auth(struct intel_digital_port *intel_dig_port, if (ret) return ret; if (!hdcp_capable) { - DRM_ERROR("Panel is not HDCP capable\n"); + DRM_DEBUG_KMS("Panel is not HDCP capable\n"); return -EINVAL; } } @@ -527,18 +572,9 @@ static int intel_hdcp_auth(struct intel_digital_port *intel_dig_port, memset(&bksv, 0, sizeof(bksv)); - /* HDCP spec states that we must retry the bksv if it is invalid */ - for (i = 0; i < tries; i++) { - ret = shim->read_bksv(intel_dig_port, bksv.shim); - if (ret) - return ret; - if (intel_hdcp_is_ksv_valid(bksv.shim)) - break; - } - if (i == tries) { - DRM_ERROR("HDCP failed, Bksv is invalid\n"); - return -ENODEV; - } + ret = intel_hdcp_read_valid_bksv(intel_dig_port, shim, bksv.shim); + if (ret < 0) + return ret; I915_WRITE(PORT_HDCP_BKSVLO(port), bksv.reg[0]); I915_WRITE(PORT_HDCP_BKSVHI(port), bksv.reg[1]); @@ -594,14 +630,15 @@ static int intel_hdcp_auth(struct intel_digital_port *intel_dig_port, } if (i == tries) { - DRM_ERROR("Timed out waiting for Ri prime match (%x)\n", - I915_READ(PORT_HDCP_STATUS(port))); + DRM_DEBUG_KMS("Timed out waiting for Ri prime match (%x)\n", + I915_READ(PORT_HDCP_STATUS(port))); return -ETIMEDOUT; } /* Wait for encryption confirmation */ if (intel_wait_for_register(dev_priv, PORT_HDCP_STATUS(port), - HDCP_STATUS_ENC, HDCP_STATUS_ENC, 20)) { + HDCP_STATUS_ENC, HDCP_STATUS_ENC, + ENCRYPT_STATUS_CHANGE_TIMEOUT_MS)) { DRM_ERROR("Timed out waiting for encryption\n"); return -ETIMEDOUT; } @@ -618,14 +655,9 @@ static int intel_hdcp_auth(struct intel_digital_port *intel_dig_port, return 0; } -static -struct intel_digital_port *conn_to_dig_port(struct intel_connector *connector) -{ - return enc_to_dig_port(&intel_attached_encoder(&connector->base)->base); -} - static int _intel_hdcp_disable(struct intel_connector *connector) { + struct intel_hdcp *hdcp = &connector->hdcp; struct drm_i915_private *dev_priv = connector->base.dev->dev_private; struct intel_digital_port *intel_dig_port = conn_to_dig_port(connector); enum port port = intel_dig_port->base.port; @@ -636,12 +668,12 @@ static int _intel_hdcp_disable(struct intel_connector *connector) I915_WRITE(PORT_HDCP_CONF(port), 0); if (intel_wait_for_register(dev_priv, PORT_HDCP_STATUS(port), ~0, 0, - 20)) { + ENCRYPT_STATUS_CHANGE_TIMEOUT_MS)) { DRM_ERROR("Failed to disable HDCP, timeout clearing status\n"); return -ETIMEDOUT; } - ret = connector->hdcp_shim->toggle_signalling(intel_dig_port, false); + ret = hdcp->shim->toggle_signalling(intel_dig_port, false); if (ret) { DRM_ERROR("Failed to disable HDCP signalling\n"); return ret; @@ -653,6 +685,7 @@ static int _intel_hdcp_disable(struct intel_connector *connector) static int _intel_hdcp_enable(struct intel_connector *connector) { + struct intel_hdcp *hdcp = &connector->hdcp; struct drm_i915_private *dev_priv = connector->base.dev->dev_private; int i, ret, tries = 3; @@ -677,8 +710,7 @@ static int _intel_hdcp_enable(struct intel_connector *connector) /* Incase of authentication failures, HDCP spec expects reauth. */ for (i = 0; i < tries; i++) { - ret = intel_hdcp_auth(conn_to_dig_port(connector), - connector->hdcp_shim); + ret = intel_hdcp_auth(conn_to_dig_port(connector), hdcp->shim); if (!ret) return 0; @@ -688,55 +720,63 @@ static int _intel_hdcp_enable(struct intel_connector *connector) _intel_hdcp_disable(connector); } - DRM_ERROR("HDCP authentication failed (%d tries/%d)\n", tries, ret); + DRM_DEBUG_KMS("HDCP authentication failed (%d tries/%d)\n", tries, ret); return ret; } +static inline +struct intel_connector *intel_hdcp_to_connector(struct intel_hdcp *hdcp) +{ + return container_of(hdcp, struct intel_connector, hdcp); +} + static void intel_hdcp_check_work(struct work_struct *work) { - struct intel_connector *connector = container_of(to_delayed_work(work), - struct intel_connector, - hdcp_check_work); + struct intel_hdcp *hdcp = container_of(to_delayed_work(work), + struct intel_hdcp, + check_work); + struct intel_connector *connector = intel_hdcp_to_connector(hdcp); + if (!intel_hdcp_check_link(connector)) - schedule_delayed_work(&connector->hdcp_check_work, + schedule_delayed_work(&hdcp->check_work, DRM_HDCP_CHECK_PERIOD_MS); } static void intel_hdcp_prop_work(struct work_struct *work) { - struct intel_connector *connector = container_of(work, - struct intel_connector, - hdcp_prop_work); + struct intel_hdcp *hdcp = container_of(work, struct intel_hdcp, + prop_work); + struct intel_connector *connector = intel_hdcp_to_connector(hdcp); struct drm_device *dev = connector->base.dev; struct drm_connector_state *state; drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); - mutex_lock(&connector->hdcp_mutex); + mutex_lock(&hdcp->mutex); /* * This worker is only used to flip between ENABLED/DESIRED. Either of - * those to UNDESIRED is handled by core. If hdcp_value == UNDESIRED, + * those to UNDESIRED is handled by core. If value == UNDESIRED, * we're running just after hdcp has been disabled, so just exit */ - if (connector->hdcp_value != DRM_MODE_CONTENT_PROTECTION_UNDESIRED) { + if (hdcp->value != DRM_MODE_CONTENT_PROTECTION_UNDESIRED) { state = connector->base.state; - state->content_protection = connector->hdcp_value; + state->content_protection = hdcp->value; } - mutex_unlock(&connector->hdcp_mutex); + mutex_unlock(&hdcp->mutex); drm_modeset_unlock(&dev->mode_config.connection_mutex); } bool is_hdcp_supported(struct drm_i915_private *dev_priv, enum port port) { /* PORT E doesn't have HDCP, and PORT F is disabled */ - return ((INTEL_GEN(dev_priv) >= 8 || IS_HASWELL(dev_priv)) && - !IS_CHERRYVIEW(dev_priv) && port < PORT_E); + return INTEL_GEN(dev_priv) >= 9 && port < PORT_E; } int intel_hdcp_init(struct intel_connector *connector, - const struct intel_hdcp_shim *hdcp_shim) + const struct intel_hdcp_shim *shim) { + struct intel_hdcp *hdcp = &connector->hdcp; int ret; ret = drm_connector_attach_content_protection_property( @@ -744,51 +784,53 @@ int intel_hdcp_init(struct intel_connector *connector, if (ret) return ret; - connector->hdcp_shim = hdcp_shim; - mutex_init(&connector->hdcp_mutex); - INIT_DELAYED_WORK(&connector->hdcp_check_work, intel_hdcp_check_work); - INIT_WORK(&connector->hdcp_prop_work, intel_hdcp_prop_work); + hdcp->shim = shim; + mutex_init(&hdcp->mutex); + INIT_DELAYED_WORK(&hdcp->check_work, intel_hdcp_check_work); + INIT_WORK(&hdcp->prop_work, intel_hdcp_prop_work); return 0; } int intel_hdcp_enable(struct intel_connector *connector) { + struct intel_hdcp *hdcp = &connector->hdcp; int ret; - if (!connector->hdcp_shim) + if (!hdcp->shim) return -ENOENT; - mutex_lock(&connector->hdcp_mutex); + mutex_lock(&hdcp->mutex); ret = _intel_hdcp_enable(connector); if (ret) goto out; - connector->hdcp_value = DRM_MODE_CONTENT_PROTECTION_ENABLED; - schedule_work(&connector->hdcp_prop_work); - schedule_delayed_work(&connector->hdcp_check_work, + hdcp->value = DRM_MODE_CONTENT_PROTECTION_ENABLED; + schedule_work(&hdcp->prop_work); + schedule_delayed_work(&hdcp->check_work, DRM_HDCP_CHECK_PERIOD_MS); out: - mutex_unlock(&connector->hdcp_mutex); + mutex_unlock(&hdcp->mutex); return ret; } int intel_hdcp_disable(struct intel_connector *connector) { + struct intel_hdcp *hdcp = &connector->hdcp; int ret = 0; - if (!connector->hdcp_shim) + if (!hdcp->shim) return -ENOENT; - mutex_lock(&connector->hdcp_mutex); + mutex_lock(&hdcp->mutex); - if (connector->hdcp_value != DRM_MODE_CONTENT_PROTECTION_UNDESIRED) { - connector->hdcp_value = DRM_MODE_CONTENT_PROTECTION_UNDESIRED; + if (hdcp->value != DRM_MODE_CONTENT_PROTECTION_UNDESIRED) { + hdcp->value = DRM_MODE_CONTENT_PROTECTION_UNDESIRED; ret = _intel_hdcp_disable(connector); } - mutex_unlock(&connector->hdcp_mutex); - cancel_delayed_work_sync(&connector->hdcp_check_work); + mutex_unlock(&hdcp->mutex); + cancel_delayed_work_sync(&hdcp->check_work); return ret; } @@ -796,8 +838,8 @@ void intel_hdcp_atomic_check(struct drm_connector *connector, struct drm_connector_state *old_state, struct drm_connector_state *new_state) { - uint64_t old_cp = old_state->content_protection; - uint64_t new_cp = new_state->content_protection; + u64 old_cp = old_state->content_protection; + u64 new_cp = new_state->content_protection; struct drm_crtc_state *crtc_state; if (!new_state->crtc) { @@ -828,17 +870,18 @@ void intel_hdcp_atomic_check(struct drm_connector *connector, /* Implements Part 3 of the HDCP authorization procedure */ int intel_hdcp_check_link(struct intel_connector *connector) { + struct intel_hdcp *hdcp = &connector->hdcp; struct drm_i915_private *dev_priv = connector->base.dev->dev_private; struct intel_digital_port *intel_dig_port = conn_to_dig_port(connector); enum port port = intel_dig_port->base.port; int ret = 0; - if (!connector->hdcp_shim) + if (!hdcp->shim) return -ENOENT; - mutex_lock(&connector->hdcp_mutex); + mutex_lock(&hdcp->mutex); - if (connector->hdcp_value == DRM_MODE_CONTENT_PROTECTION_UNDESIRED) + if (hdcp->value == DRM_MODE_CONTENT_PROTECTION_UNDESIRED) goto out; if (!(I915_READ(PORT_HDCP_STATUS(port)) & HDCP_STATUS_ENC)) { @@ -846,17 +889,15 @@ int intel_hdcp_check_link(struct intel_connector *connector) connector->base.name, connector->base.base.id, I915_READ(PORT_HDCP_STATUS(port))); ret = -ENXIO; - connector->hdcp_value = DRM_MODE_CONTENT_PROTECTION_DESIRED; - schedule_work(&connector->hdcp_prop_work); + hdcp->value = DRM_MODE_CONTENT_PROTECTION_DESIRED; + schedule_work(&hdcp->prop_work); goto out; } - if (connector->hdcp_shim->check_link(intel_dig_port)) { - if (connector->hdcp_value != - DRM_MODE_CONTENT_PROTECTION_UNDESIRED) { - connector->hdcp_value = - DRM_MODE_CONTENT_PROTECTION_ENABLED; - schedule_work(&connector->hdcp_prop_work); + if (hdcp->shim->check_link(intel_dig_port)) { + if (hdcp->value != DRM_MODE_CONTENT_PROTECTION_UNDESIRED) { + hdcp->value = DRM_MODE_CONTENT_PROTECTION_ENABLED; + schedule_work(&hdcp->prop_work); } goto out; } @@ -867,20 +908,20 @@ int intel_hdcp_check_link(struct intel_connector *connector) ret = _intel_hdcp_disable(connector); if (ret) { DRM_ERROR("Failed to disable hdcp (%d)\n", ret); - connector->hdcp_value = DRM_MODE_CONTENT_PROTECTION_DESIRED; - schedule_work(&connector->hdcp_prop_work); + hdcp->value = DRM_MODE_CONTENT_PROTECTION_DESIRED; + schedule_work(&hdcp->prop_work); goto out; } ret = _intel_hdcp_enable(connector); if (ret) { - DRM_ERROR("Failed to enable hdcp (%d)\n", ret); - connector->hdcp_value = DRM_MODE_CONTENT_PROTECTION_DESIRED; - schedule_work(&connector->hdcp_prop_work); + DRM_DEBUG_KMS("Failed to enable hdcp (%d)\n", ret); + hdcp->value = DRM_MODE_CONTENT_PROTECTION_DESIRED; + schedule_work(&hdcp->prop_work); goto out; } out: - mutex_unlock(&connector->hdcp_mutex); + mutex_unlock(&hdcp->mutex); return ret; } diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c index a2dab0b6bde6..f125a62eba8c 100644 --- a/drivers/gpu/drm/i915/intel_hdmi.c +++ b/drivers/gpu/drm/i915/intel_hdmi.c @@ -30,7 +30,6 @@ #include <linux/slab.h> #include <linux/delay.h> #include <linux/hdmi.h> -#include <drm/drmP.h> #include <drm/drm_atomic_helper.h> #include <drm/drm_crtc.h> #include <drm/drm_edid.h> @@ -115,6 +114,8 @@ static u32 hsw_infoframe_enable(unsigned int type) switch (type) { case DP_SDP_VSC: return VIDEO_DIP_ENABLE_VSC_HSW; + case DP_SDP_PPS: + return VDIP_ENABLE_PPS; case HDMI_INFOFRAME_TYPE_AVI: return VIDEO_DIP_ENABLE_AVI_HSW; case HDMI_INFOFRAME_TYPE_SPD: @@ -136,6 +137,8 @@ hsw_dip_data_reg(struct drm_i915_private *dev_priv, switch (type) { case DP_SDP_VSC: return HSW_TVIDEO_DIP_VSC_DATA(cpu_transcoder, i); + case DP_SDP_PPS: + return ICL_VIDEO_DIP_PPS_DATA(cpu_transcoder, i); case HDMI_INFOFRAME_TYPE_AVI: return HSW_TVIDEO_DIP_AVI_DATA(cpu_transcoder, i); case HDMI_INFOFRAME_TYPE_SPD: @@ -148,14 +151,25 @@ hsw_dip_data_reg(struct drm_i915_private *dev_priv, } } -static void g4x_write_infoframe(struct drm_encoder *encoder, +static int hsw_dip_data_size(unsigned int type) +{ + switch (type) { + case DP_SDP_VSC: + return VIDEO_DIP_VSC_DATA_SIZE; + case DP_SDP_PPS: + return VIDEO_DIP_PPS_DATA_SIZE; + default: + return VIDEO_DIP_DATA_SIZE; + } +} + +static void g4x_write_infoframe(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, unsigned int type, const void *frame, ssize_t len) { const u32 *data = frame; - struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); u32 val = I915_READ(VIDEO_DIP_CTL); int i; @@ -186,31 +200,29 @@ static void g4x_write_infoframe(struct drm_encoder *encoder, POSTING_READ(VIDEO_DIP_CTL); } -static bool g4x_infoframe_enabled(struct drm_encoder *encoder, +static bool g4x_infoframe_enabled(struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config) { - struct drm_i915_private *dev_priv = to_i915(encoder->dev); - struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); u32 val = I915_READ(VIDEO_DIP_CTL); if ((val & VIDEO_DIP_ENABLE) == 0) return false; - if ((val & VIDEO_DIP_PORT_MASK) != VIDEO_DIP_PORT(intel_dig_port->base.port)) + if ((val & VIDEO_DIP_PORT_MASK) != VIDEO_DIP_PORT(encoder->port)) return false; return val & (VIDEO_DIP_ENABLE_AVI | VIDEO_DIP_ENABLE_VENDOR | VIDEO_DIP_ENABLE_SPD); } -static void ibx_write_infoframe(struct drm_encoder *encoder, +static void ibx_write_infoframe(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, unsigned int type, const void *frame, ssize_t len) { const u32 *data = frame; - struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc); i915_reg_t reg = TVIDEO_DIP_CTL(intel_crtc->pipe); u32 val = I915_READ(reg); @@ -243,11 +255,10 @@ static void ibx_write_infoframe(struct drm_encoder *encoder, POSTING_READ(reg); } -static bool ibx_infoframe_enabled(struct drm_encoder *encoder, +static bool ibx_infoframe_enabled(struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config) { - struct drm_i915_private *dev_priv = to_i915(encoder->dev); - struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); enum pipe pipe = to_intel_crtc(pipe_config->base.crtc)->pipe; i915_reg_t reg = TVIDEO_DIP_CTL(pipe); u32 val = I915_READ(reg); @@ -255,7 +266,7 @@ static bool ibx_infoframe_enabled(struct drm_encoder *encoder, if ((val & VIDEO_DIP_ENABLE) == 0) return false; - if ((val & VIDEO_DIP_PORT_MASK) != VIDEO_DIP_PORT(intel_dig_port->base.port)) + if ((val & VIDEO_DIP_PORT_MASK) != VIDEO_DIP_PORT(encoder->port)) return false; return val & (VIDEO_DIP_ENABLE_AVI | @@ -263,14 +274,13 @@ static bool ibx_infoframe_enabled(struct drm_encoder *encoder, VIDEO_DIP_ENABLE_SPD | VIDEO_DIP_ENABLE_GCP); } -static void cpt_write_infoframe(struct drm_encoder *encoder, +static void cpt_write_infoframe(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, unsigned int type, const void *frame, ssize_t len) { const u32 *data = frame; - struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc); i915_reg_t reg = TVIDEO_DIP_CTL(intel_crtc->pipe); u32 val = I915_READ(reg); @@ -306,10 +316,10 @@ static void cpt_write_infoframe(struct drm_encoder *encoder, POSTING_READ(reg); } -static bool cpt_infoframe_enabled(struct drm_encoder *encoder, +static bool cpt_infoframe_enabled(struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config) { - struct drm_i915_private *dev_priv = to_i915(encoder->dev); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); enum pipe pipe = to_intel_crtc(pipe_config->base.crtc)->pipe; u32 val = I915_READ(TVIDEO_DIP_CTL(pipe)); @@ -321,14 +331,13 @@ static bool cpt_infoframe_enabled(struct drm_encoder *encoder, VIDEO_DIP_ENABLE_SPD | VIDEO_DIP_ENABLE_GCP); } -static void vlv_write_infoframe(struct drm_encoder *encoder, +static void vlv_write_infoframe(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, unsigned int type, const void *frame, ssize_t len) { const u32 *data = frame; - struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc); i915_reg_t reg = VLV_TVIDEO_DIP_CTL(intel_crtc->pipe); u32 val = I915_READ(reg); @@ -361,18 +370,17 @@ static void vlv_write_infoframe(struct drm_encoder *encoder, POSTING_READ(reg); } -static bool vlv_infoframe_enabled(struct drm_encoder *encoder, +static bool vlv_infoframe_enabled(struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config) { - struct drm_i915_private *dev_priv = to_i915(encoder->dev); - struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); enum pipe pipe = to_intel_crtc(pipe_config->base.crtc)->pipe; u32 val = I915_READ(VLV_TVIDEO_DIP_CTL(pipe)); if ((val & VIDEO_DIP_ENABLE) == 0) return false; - if ((val & VIDEO_DIP_PORT_MASK) != VIDEO_DIP_PORT(intel_dig_port->base.port)) + if ((val & VIDEO_DIP_PORT_MASK) != VIDEO_DIP_PORT(encoder->port)) return false; return val & (VIDEO_DIP_ENABLE_AVI | @@ -380,21 +388,21 @@ static bool vlv_infoframe_enabled(struct drm_encoder *encoder, VIDEO_DIP_ENABLE_SPD | VIDEO_DIP_ENABLE_GCP); } -static void hsw_write_infoframe(struct drm_encoder *encoder, +static void hsw_write_infoframe(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, unsigned int type, const void *frame, ssize_t len) { const u32 *data = frame; - struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; i915_reg_t ctl_reg = HSW_TVIDEO_DIP_CTL(cpu_transcoder); - int data_size = type == DP_SDP_VSC ? - VIDEO_DIP_VSC_DATA_SIZE : VIDEO_DIP_DATA_SIZE; + int data_size; int i; u32 val = I915_READ(ctl_reg); + data_size = hsw_dip_data_size(type); + val &= ~hsw_infoframe_enable(type); I915_WRITE(ctl_reg, val); @@ -415,10 +423,10 @@ static void hsw_write_infoframe(struct drm_encoder *encoder, POSTING_READ(ctl_reg); } -static bool hsw_infoframe_enabled(struct drm_encoder *encoder, +static bool hsw_infoframe_enabled(struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config) { - struct drm_i915_private *dev_priv = to_i915(encoder->dev); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); u32 val = I915_READ(HSW_TVIDEO_DIP_CTL(pipe_config->cpu_transcoder)); return val & (VIDEO_DIP_ENABLE_VSC_HSW | VIDEO_DIP_ENABLE_AVI_HSW | @@ -443,11 +451,11 @@ static bool hsw_infoframe_enabled(struct drm_encoder *encoder, * trick them by giving an offset into the buffer and moving back the header * bytes by one. */ -static void intel_write_infoframe(struct drm_encoder *encoder, +static void intel_write_infoframe(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, union hdmi_infoframe *frame) { - struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); + struct intel_digital_port *intel_dig_port = enc_to_dig_port(&encoder->base); u8 buffer[VIDEO_DIP_DATA_SIZE]; ssize_t len; @@ -457,55 +465,55 @@ static void intel_write_infoframe(struct drm_encoder *encoder, return; /* Insert the 'hole' (see big comment above) at position 3 */ - buffer[0] = buffer[1]; - buffer[1] = buffer[2]; - buffer[2] = buffer[3]; + memmove(&buffer[0], &buffer[1], 3); buffer[3] = 0; len++; - intel_dig_port->write_infoframe(encoder, crtc_state, frame->any.type, buffer, len); + intel_dig_port->write_infoframe(encoder, + crtc_state, + frame->any.type, buffer, len); } -static void intel_hdmi_set_avi_infoframe(struct drm_encoder *encoder, +static void intel_hdmi_set_avi_infoframe(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { - struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder); const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode; - struct drm_connector *connector = &intel_hdmi->attached_connector->base; - bool is_hdmi2_sink = connector->display_info.hdmi.scdc.supported; union hdmi_infoframe frame; int ret; ret = drm_hdmi_avi_infoframe_from_display_mode(&frame.avi, - adjusted_mode, - is_hdmi2_sink); + conn_state->connector, + adjusted_mode); if (ret < 0) { DRM_ERROR("couldn't fill AVI infoframe\n"); return; } - if (crtc_state->ycbcr420) + if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420) frame.avi.colorspace = HDMI_COLORSPACE_YUV420; + else if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444) + frame.avi.colorspace = HDMI_COLORSPACE_YUV444; else frame.avi.colorspace = HDMI_COLORSPACE_RGB; - drm_hdmi_avi_infoframe_quant_range(&frame.avi, adjusted_mode, + drm_hdmi_avi_infoframe_quant_range(&frame.avi, + conn_state->connector, + adjusted_mode, crtc_state->limited_color_range ? HDMI_QUANTIZATION_RANGE_LIMITED : - HDMI_QUANTIZATION_RANGE_FULL, - intel_hdmi->rgb_quant_range_selectable, - is_hdmi2_sink); + HDMI_QUANTIZATION_RANGE_FULL); drm_hdmi_avi_infoframe_content_type(&frame.avi, conn_state); /* TODO: handle pixel repetition for YCBCR420 outputs */ - intel_write_infoframe(encoder, crtc_state, &frame); + intel_write_infoframe(encoder, crtc_state, + &frame); } -static void intel_hdmi_set_spd_infoframe(struct drm_encoder *encoder, +static void intel_hdmi_set_spd_infoframe(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state) { union hdmi_infoframe frame; @@ -519,11 +527,12 @@ static void intel_hdmi_set_spd_infoframe(struct drm_encoder *encoder, frame.spd.sdi = HDMI_SPD_SDI_PC; - intel_write_infoframe(encoder, crtc_state, &frame); + intel_write_infoframe(encoder, crtc_state, + &frame); } static void -intel_hdmi_set_hdmi_infoframe(struct drm_encoder *encoder, +intel_hdmi_set_hdmi_infoframe(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { @@ -536,20 +545,21 @@ intel_hdmi_set_hdmi_infoframe(struct drm_encoder *encoder, if (ret < 0) return; - intel_write_infoframe(encoder, crtc_state, &frame); + intel_write_infoframe(encoder, crtc_state, + &frame); } -static void g4x_set_infoframes(struct drm_encoder *encoder, +static void g4x_set_infoframes(struct intel_encoder *encoder, bool enable, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { - struct drm_i915_private *dev_priv = to_i915(encoder->dev); - struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_digital_port *intel_dig_port = enc_to_dig_port(&encoder->base); struct intel_hdmi *intel_hdmi = &intel_dig_port->hdmi; i915_reg_t reg = VIDEO_DIP_CTL; u32 val = I915_READ(reg); - u32 port = VIDEO_DIP_PORT(intel_dig_port->base.port); + u32 port = VIDEO_DIP_PORT(encoder->port); assert_hdmi_port_disabled(intel_hdmi); @@ -657,11 +667,11 @@ static bool gcp_default_phase_possible(int pipe_bpp, mode->crtc_htotal/2 % pixels_per_group == 0); } -static bool intel_hdmi_set_gcp_infoframe(struct drm_encoder *encoder, +static bool intel_hdmi_set_gcp_infoframe(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { - struct drm_i915_private *dev_priv = to_i915(encoder->dev); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); i915_reg_t reg; u32 val = 0; @@ -689,18 +699,18 @@ static bool intel_hdmi_set_gcp_infoframe(struct drm_encoder *encoder, return val != 0; } -static void ibx_set_infoframes(struct drm_encoder *encoder, +static void ibx_set_infoframes(struct intel_encoder *encoder, bool enable, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { - struct drm_i915_private *dev_priv = to_i915(encoder->dev); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc); - struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); + struct intel_digital_port *intel_dig_port = enc_to_dig_port(&encoder->base); struct intel_hdmi *intel_hdmi = &intel_dig_port->hdmi; i915_reg_t reg = TVIDEO_DIP_CTL(intel_crtc->pipe); u32 val = I915_READ(reg); - u32 port = VIDEO_DIP_PORT(intel_dig_port->base.port); + u32 port = VIDEO_DIP_PORT(encoder->port); assert_hdmi_port_disabled(intel_hdmi); @@ -742,14 +752,14 @@ static void ibx_set_infoframes(struct drm_encoder *encoder, intel_hdmi_set_hdmi_infoframe(encoder, crtc_state, conn_state); } -static void cpt_set_infoframes(struct drm_encoder *encoder, +static void cpt_set_infoframes(struct intel_encoder *encoder, bool enable, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { - struct drm_i915_private *dev_priv = to_i915(encoder->dev); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc); - struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder); + struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base); i915_reg_t reg = TVIDEO_DIP_CTL(intel_crtc->pipe); u32 val = I915_READ(reg); @@ -785,18 +795,17 @@ static void cpt_set_infoframes(struct drm_encoder *encoder, intel_hdmi_set_hdmi_infoframe(encoder, crtc_state, conn_state); } -static void vlv_set_infoframes(struct drm_encoder *encoder, +static void vlv_set_infoframes(struct intel_encoder *encoder, bool enable, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { - struct drm_i915_private *dev_priv = to_i915(encoder->dev); - struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc); - struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder); + struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base); i915_reg_t reg = VLV_TVIDEO_DIP_CTL(intel_crtc->pipe); u32 val = I915_READ(reg); - u32 port = VIDEO_DIP_PORT(intel_dig_port->base.port); + u32 port = VIDEO_DIP_PORT(encoder->port); assert_hdmi_port_disabled(intel_hdmi); @@ -838,12 +847,12 @@ static void vlv_set_infoframes(struct drm_encoder *encoder, intel_hdmi_set_hdmi_infoframe(encoder, crtc_state, conn_state); } -static void hsw_set_infoframes(struct drm_encoder *encoder, +static void hsw_set_infoframes(struct intel_encoder *encoder, bool enable, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { - struct drm_i915_private *dev_priv = to_i915(encoder->dev); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); i915_reg_t reg = HSW_TVIDEO_DIP_CTL(crtc_state->cpu_transcoder); u32 val = I915_READ(reg); @@ -965,13 +974,13 @@ int intel_hdmi_hdcp_write_an_aksv(struct intel_digital_port *intel_dig_port, ret = intel_hdmi_hdcp_write(intel_dig_port, DRM_HDCP_DDC_AN, an, DRM_HDCP_AN_LEN); if (ret) { - DRM_ERROR("Write An over DDC failed (%d)\n", ret); + DRM_DEBUG_KMS("Write An over DDC failed (%d)\n", ret); return ret; } ret = intel_gmbus_output_aksv(adapter); if (ret < 0) { - DRM_ERROR("Failed to output aksv (%d)\n", ret); + DRM_DEBUG_KMS("Failed to output aksv (%d)\n", ret); return ret; } return 0; @@ -984,7 +993,7 @@ static int intel_hdmi_hdcp_read_bksv(struct intel_digital_port *intel_dig_port, ret = intel_hdmi_hdcp_read(intel_dig_port, DRM_HDCP_DDC_BKSV, bksv, DRM_HDCP_KSV_LEN); if (ret) - DRM_ERROR("Read Bksv over DDC failed (%d)\n", ret); + DRM_DEBUG_KMS("Read Bksv over DDC failed (%d)\n", ret); return ret; } @@ -996,7 +1005,7 @@ int intel_hdmi_hdcp_read_bstatus(struct intel_digital_port *intel_dig_port, ret = intel_hdmi_hdcp_read(intel_dig_port, DRM_HDCP_DDC_BSTATUS, bstatus, DRM_HDCP_BSTATUS_LEN); if (ret) - DRM_ERROR("Read bstatus over DDC failed (%d)\n", ret); + DRM_DEBUG_KMS("Read bstatus over DDC failed (%d)\n", ret); return ret; } @@ -1009,7 +1018,7 @@ int intel_hdmi_hdcp_repeater_present(struct intel_digital_port *intel_dig_port, ret = intel_hdmi_hdcp_read(intel_dig_port, DRM_HDCP_DDC_BCAPS, &val, 1); if (ret) { - DRM_ERROR("Read bcaps over DDC failed (%d)\n", ret); + DRM_DEBUG_KMS("Read bcaps over DDC failed (%d)\n", ret); return ret; } *repeater_present = val & DRM_HDCP_DDC_BCAPS_REPEATER_PRESENT; @@ -1024,7 +1033,7 @@ int intel_hdmi_hdcp_read_ri_prime(struct intel_digital_port *intel_dig_port, ret = intel_hdmi_hdcp_read(intel_dig_port, DRM_HDCP_DDC_RI_PRIME, ri_prime, DRM_HDCP_RI_LEN); if (ret) - DRM_ERROR("Read Ri' over DDC failed (%d)\n", ret); + DRM_DEBUG_KMS("Read Ri' over DDC failed (%d)\n", ret); return ret; } @@ -1037,7 +1046,7 @@ int intel_hdmi_hdcp_read_ksv_ready(struct intel_digital_port *intel_dig_port, ret = intel_hdmi_hdcp_read(intel_dig_port, DRM_HDCP_DDC_BCAPS, &val, 1); if (ret) { - DRM_ERROR("Read bcaps over DDC failed (%d)\n", ret); + DRM_DEBUG_KMS("Read bcaps over DDC failed (%d)\n", ret); return ret; } *ksv_ready = val & DRM_HDCP_DDC_BCAPS_KSV_FIFO_READY; @@ -1052,7 +1061,7 @@ int intel_hdmi_hdcp_read_ksv_fifo(struct intel_digital_port *intel_dig_port, ret = intel_hdmi_hdcp_read(intel_dig_port, DRM_HDCP_DDC_KSV_FIFO, ksv_fifo, num_downstream * DRM_HDCP_KSV_LEN); if (ret) { - DRM_ERROR("Read ksv fifo over DDC failed (%d)\n", ret); + DRM_DEBUG_KMS("Read ksv fifo over DDC failed (%d)\n", ret); return ret; } return 0; @@ -1070,7 +1079,7 @@ int intel_hdmi_hdcp_read_v_prime_part(struct intel_digital_port *intel_dig_port, ret = intel_hdmi_hdcp_read(intel_dig_port, DRM_HDCP_DDC_V_PRIME(i), part, DRM_HDCP_V_PRIME_PART_LEN); if (ret) - DRM_ERROR("Read V'[%d] over DDC failed (%d)\n", i, ret); + DRM_DEBUG_KMS("Read V'[%d] over DDC failed (%d)\n", i, ret); return ret; } @@ -1177,15 +1186,17 @@ static bool intel_hdmi_get_hw_state(struct intel_encoder *encoder, { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base); + intel_wakeref_t wakeref; bool ret; - if (!intel_display_power_get_if_enabled(dev_priv, - encoder->power_domain)) + wakeref = intel_display_power_get_if_enabled(dev_priv, + encoder->power_domain); + if (!wakeref) return false; ret = intel_sdvo_port_enabled(dev_priv, intel_hdmi->hdmi_reg, pipe); - intel_display_power_put(dev_priv, encoder->power_domain); + intel_display_power_put(dev_priv, encoder->power_domain, wakeref); return ret; } @@ -1217,7 +1228,7 @@ static void intel_hdmi_get_config(struct intel_encoder *encoder, if (tmp & HDMI_MODE_SELECT_HDMI) pipe_config->has_hdmi_sink = true; - if (intel_dig_port->infoframe_enabled(&encoder->base, pipe_config)) + if (intel_dig_port->infoframe_enabled(encoder, pipe_config)) pipe_config->has_infoframe = true; if (tmp & SDVO_AUDIO_ENABLE) @@ -1438,7 +1449,8 @@ static void intel_disable_hdmi(struct intel_encoder *encoder, intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true); } - intel_dig_port->set_infoframes(&encoder->base, false, + intel_dig_port->set_infoframes(encoder, + false, old_crtc_state, old_conn_state); intel_dp_dual_mode_set_tmds_output(intel_hdmi, false); @@ -1576,7 +1588,7 @@ intel_hdmi_mode_valid(struct drm_connector *connector, if (hdmi->has_hdmi_sink && !force_dvi) { /* if we can't do 8bpc we may still be able to do 12bpc */ - if (status != MODE_OK && !HAS_GMCH_DISPLAY(dev_priv)) + if (status != MODE_OK && !HAS_GMCH(dev_priv)) status = hdmi_port_clock_valid(hdmi, clock * 3 / 2, true, force_dvi); @@ -1597,9 +1609,11 @@ static bool hdmi_deep_color_possible(const struct intel_crtc_state *crtc_state, struct drm_atomic_state *state = crtc_state->base.state; struct drm_connector_state *connector_state; struct drm_connector *connector; + const struct drm_display_mode *adjusted_mode = + &crtc_state->base.adjusted_mode; int i; - if (HAS_GMCH_DISPLAY(dev_priv)) + if (HAS_GMCH(dev_priv)) return false; if (bpc == 10 && INTEL_GEN(dev_priv) < 11) @@ -1624,7 +1638,7 @@ static bool hdmi_deep_color_possible(const struct intel_crtc_state *crtc_state, if (connector_state->crtc != crtc_state->base.crtc) continue; - if (crtc_state->ycbcr420) { + if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420) { const struct drm_hdmi_info *hdmi = &info->hdmi; if (bpc == 12 && !(hdmi->y420_dc_modes & @@ -1645,7 +1659,14 @@ static bool hdmi_deep_color_possible(const struct intel_crtc_state *crtc_state, /* Display WA #1139: glk */ if (bpc == 12 && IS_GLK_REVID(dev_priv, 0, GLK_REVID_A1) && - crtc_state->base.adjusted_mode.htotal > 5460) + adjusted_mode->htotal > 5460) + return false; + + /* Display Wa_1405510057:icl */ + if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 && + bpc == 10 && IS_ICELAKE(dev_priv) && + (adjusted_mode->crtc_hblank_end - + adjusted_mode->crtc_hblank_start) % 8 == 2) return false; return true; @@ -1669,7 +1690,7 @@ intel_hdmi_ycbcr420_config(struct drm_connector *connector, *clock_12bpc /= 2; *clock_10bpc /= 2; *clock_8bpc /= 2; - config->ycbcr420 = true; + config->output_format = INTEL_OUTPUT_FORMAT_YCBCR420; /* YCBCR 420 output conversion needs a scaler */ if (skl_update_scaler_crtc(config)) { @@ -1683,9 +1704,9 @@ intel_hdmi_ycbcr420_config(struct drm_connector *connector, return true; } -bool intel_hdmi_compute_config(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config, - struct drm_connector_state *conn_state) +int intel_hdmi_compute_config(struct intel_encoder *encoder, + struct intel_crtc_state *pipe_config, + struct drm_connector_state *conn_state) { struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base); struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); @@ -1701,8 +1722,9 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder, bool force_dvi = intel_conn_state->force_audio == HDMI_AUDIO_OFF_DVI; if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) - return false; + return -EINVAL; + pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; pipe_config->has_hdmi_sink = !force_dvi && intel_hdmi->has_hdmi_sink; if (pipe_config->has_hdmi_sink) @@ -1731,7 +1753,7 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder, &clock_12bpc, &clock_10bpc, &clock_8bpc)) { DRM_ERROR("Can't support YCBCR420 output\n"); - return false; + return -EINVAL; } } @@ -1781,7 +1803,7 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder, if (hdmi_port_clock_valid(intel_hdmi, pipe_config->port_clock, false, force_dvi) != MODE_OK) { DRM_DEBUG_KMS("unsupported HDMI clock, rejecting mode\n"); - return false; + return -EINVAL; } /* Set user selected PAR to incoming mode's member */ @@ -1800,7 +1822,7 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder, } } - return true; + return 0; } static void @@ -1810,7 +1832,6 @@ intel_hdmi_unset_edid(struct drm_connector *connector) intel_hdmi->has_hdmi_sink = false; intel_hdmi->has_audio = false; - intel_hdmi->rgb_quant_range_selectable = false; intel_hdmi->dp_dual_mode.type = DRM_DP_DUAL_MODE_NONE; intel_hdmi->dp_dual_mode.max_tmds_clock = 0; @@ -1871,11 +1892,12 @@ intel_hdmi_set_edid(struct drm_connector *connector) { struct drm_i915_private *dev_priv = to_i915(connector->dev); struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector); + intel_wakeref_t wakeref; struct edid *edid; bool connected = false; struct i2c_adapter *i2c; - intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS); + wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS); i2c = intel_gmbus_get_adapter(dev_priv, intel_hdmi->ddc_bus); @@ -1890,13 +1912,10 @@ intel_hdmi_set_edid(struct drm_connector *connector) intel_hdmi_dp_dual_mode_detect(connector, edid != NULL); - intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS); + intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS, wakeref); to_intel_connector(connector)->detect_edid = edid; if (edid && edid->input & DRM_EDID_INPUT_DIGITAL) { - intel_hdmi->rgb_quant_range_selectable = - drm_rgb_quant_range_selectable(edid); - intel_hdmi->has_audio = drm_detect_monitor_audio(edid); intel_hdmi->has_hdmi_sink = drm_detect_hdmi_monitor(edid); @@ -1915,11 +1934,12 @@ intel_hdmi_detect(struct drm_connector *connector, bool force) struct drm_i915_private *dev_priv = to_i915(connector->dev); struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector); struct intel_encoder *encoder = &hdmi_to_dig_port(intel_hdmi)->base; + intel_wakeref_t wakeref; DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", connector->base.id, connector->name); - intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS); + wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS); if (IS_ICELAKE(dev_priv) && !intel_digital_port_connected(encoder)) @@ -1931,7 +1951,7 @@ intel_hdmi_detect(struct drm_connector *connector, bool force) status = connector_status_connected; out: - intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS); + intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS, wakeref); if (status != connector_status_connected) cec_notifier_phys_addr_invalidate(intel_hdmi->cec_notifier); @@ -1973,7 +1993,7 @@ static void intel_hdmi_pre_enable(struct intel_encoder *encoder, intel_hdmi_prepare(encoder, pipe_config); - intel_dig_port->set_infoframes(&encoder->base, + intel_dig_port->set_infoframes(encoder, pipe_config->has_infoframe, pipe_config, conn_state); } @@ -1991,7 +2011,7 @@ static void vlv_hdmi_pre_enable(struct intel_encoder *encoder, vlv_set_phy_signal_level(encoder, 0x2b245f5f, 0x00002000, 0x5578b83a, 0x2b247878); - dport->set_infoframes(&encoder->base, + dport->set_infoframes(encoder, pipe_config->has_infoframe, pipe_config, conn_state); @@ -2062,7 +2082,7 @@ static void chv_hdmi_pre_enable(struct intel_encoder *encoder, /* Use 800mV-0dB */ chv_set_phy_signal_level(encoder, 128, 102, false); - dport->set_infoframes(&encoder->base, + dport->set_infoframes(encoder, pipe_config->has_infoframe, pipe_config, conn_state); @@ -2074,13 +2094,26 @@ static void chv_hdmi_pre_enable(struct intel_encoder *encoder, chv_phy_release_cl2_override(encoder); } +static int +intel_hdmi_connector_register(struct drm_connector *connector) +{ + int ret; + + ret = intel_connector_register(connector); + if (ret) + return ret; + + i915_debugfs_connector_add(connector); + + return ret; +} + static void intel_hdmi_destroy(struct drm_connector *connector) { if (intel_attached_hdmi(connector)->cec_notifier) cec_notifier_put(intel_attached_hdmi(connector)->cec_notifier); - kfree(to_intel_connector(connector)->detect_edid); - drm_connector_cleanup(connector); - kfree(connector); + + intel_connector_destroy(connector); } static const struct drm_connector_funcs intel_hdmi_connector_funcs = { @@ -2089,7 +2122,7 @@ static const struct drm_connector_funcs intel_hdmi_connector_funcs = { .fill_modes = drm_helper_probe_single_connector_modes, .atomic_get_property = intel_digital_connector_atomic_get_property, .atomic_set_property = intel_digital_connector_atomic_set_property, - .late_register = intel_connector_register, + .late_register = intel_hdmi_connector_register, .early_unregister = intel_connector_unregister, .destroy = intel_hdmi_destroy, .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, @@ -2109,11 +2142,16 @@ static const struct drm_encoder_funcs intel_hdmi_enc_funcs = { static void intel_hdmi_add_properties(struct intel_hdmi *intel_hdmi, struct drm_connector *connector) { + struct drm_i915_private *dev_priv = to_i915(connector->dev); + intel_attach_force_audio_property(connector); intel_attach_broadcast_rgb_property(connector); intel_attach_aspect_ratio_property(connector); drm_connector_attach_content_type_property(connector); connector->state->picture_aspect_ratio = HDMI_PICTURE_ASPECT_NONE; + + if (!HAS_GMCH(dev_priv)) + drm_connector_attach_max_bpc_property(connector, 8, 12); } /* @@ -2324,9 +2362,18 @@ void intel_infoframe_init(struct intel_digital_port *intel_dig_port) intel_dig_port->set_infoframes = g4x_set_infoframes; intel_dig_port->infoframe_enabled = g4x_infoframe_enabled; } else if (HAS_DDI(dev_priv)) { - intel_dig_port->write_infoframe = hsw_write_infoframe; - intel_dig_port->set_infoframes = hsw_set_infoframes; - intel_dig_port->infoframe_enabled = hsw_infoframe_enabled; + if (intel_dig_port->lspcon.active) { + intel_dig_port->write_infoframe = + lspcon_write_infoframe; + intel_dig_port->set_infoframes = lspcon_set_infoframes; + intel_dig_port->infoframe_enabled = + lspcon_infoframe_enabled; + } else { + intel_dig_port->set_infoframes = hsw_set_infoframes; + intel_dig_port->infoframe_enabled = + hsw_infoframe_enabled; + intel_dig_port->write_infoframe = hsw_write_infoframe; + } } else if (HAS_PCH_IBX(dev_priv)) { intel_dig_port->write_infoframe = ibx_write_infoframe; intel_dig_port->set_infoframes = ibx_set_infoframes; @@ -2485,5 +2532,6 @@ void intel_hdmi_init(struct drm_i915_private *dev_priv, intel_infoframe_init(intel_dig_port); + intel_dig_port->aux_ch = intel_bios_port_aux_ch(dev_priv, port); intel_hdmi_init_connector(intel_dig_port, intel_connector); } diff --git a/drivers/gpu/drm/i915/intel_hotplug.c b/drivers/gpu/drm/i915/intel_hotplug.c index 9a8018130237..b8937c788f03 100644 --- a/drivers/gpu/drm/i915/intel_hotplug.c +++ b/drivers/gpu/drm/i915/intel_hotplug.c @@ -23,7 +23,6 @@ #include <linux/kernel.h> -#include <drm/drmP.h> #include <drm/i915_drm.h> #include "i915_drv.h" @@ -114,51 +113,68 @@ enum hpd_pin intel_hpd_pin_default(struct drm_i915_private *dev_priv, #define HPD_STORM_REENABLE_DELAY (2 * 60 * 1000) /** - * intel_hpd_irq_storm_detect - gather stats and detect HPD irq storm on a pin + * intel_hpd_irq_storm_detect - gather stats and detect HPD IRQ storm on a pin * @dev_priv: private driver data pointer * @pin: the pin to gather stats on + * @long_hpd: whether the HPD IRQ was long or short * - * Gather stats about HPD irqs from the specified @pin, and detect irq + * Gather stats about HPD IRQs from the specified @pin, and detect IRQ * storms. Only the pin specific stats and state are changed, the caller is * responsible for further action. * - * The number of irqs that are allowed within @HPD_STORM_DETECT_PERIOD is + * The number of IRQs that are allowed within @HPD_STORM_DETECT_PERIOD is * stored in @dev_priv->hotplug.hpd_storm_threshold which defaults to - * @HPD_STORM_DEFAULT_THRESHOLD. If this threshold is exceeded, it's - * considered an irq storm and the irq state is set to @HPD_MARK_DISABLED. + * @HPD_STORM_DEFAULT_THRESHOLD. Long IRQs count as +10 to this threshold, and + * short IRQs count as +1. If this threshold is exceeded, it's considered an + * IRQ storm and the IRQ state is set to @HPD_MARK_DISABLED. + * + * By default, most systems will only count long IRQs towards + * &dev_priv->hotplug.hpd_storm_threshold. However, some older systems also + * suffer from short IRQ storms and must also track these. Because short IRQ + * storms are naturally caused by sideband interactions with DP MST devices, + * short IRQ detection is only enabled for systems without DP MST support. + * Systems which are new enough to support DP MST are far less likely to + * suffer from IRQ storms at all, so this is fine. * * The HPD threshold can be controlled through i915_hpd_storm_ctl in debugfs, * and should only be adjusted for automated hotplug testing. * - * Return true if an irq storm was detected on @pin. + * Return true if an IRQ storm was detected on @pin. */ static bool intel_hpd_irq_storm_detect(struct drm_i915_private *dev_priv, - enum hpd_pin pin) + enum hpd_pin pin, bool long_hpd) { - unsigned long start = dev_priv->hotplug.stats[pin].last_jiffies; + struct i915_hotplug *hpd = &dev_priv->hotplug; + unsigned long start = hpd->stats[pin].last_jiffies; unsigned long end = start + msecs_to_jiffies(HPD_STORM_DETECT_PERIOD); - const int threshold = dev_priv->hotplug.hpd_storm_threshold; + const int increment = long_hpd ? 10 : 1; + const int threshold = hpd->hpd_storm_threshold; bool storm = false; + if (!threshold || + (!long_hpd && !dev_priv->hotplug.hpd_short_storm_enabled)) + return false; + if (!time_in_range(jiffies, start, end)) { - dev_priv->hotplug.stats[pin].last_jiffies = jiffies; - dev_priv->hotplug.stats[pin].count = 0; - DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: 0\n", pin); - } else if (dev_priv->hotplug.stats[pin].count > threshold && - threshold) { - dev_priv->hotplug.stats[pin].state = HPD_MARK_DISABLED; + hpd->stats[pin].last_jiffies = jiffies; + hpd->stats[pin].count = 0; + } + + hpd->stats[pin].count += increment; + if (hpd->stats[pin].count > threshold) { + hpd->stats[pin].state = HPD_MARK_DISABLED; DRM_DEBUG_KMS("HPD interrupt storm detected on PIN %d\n", pin); storm = true; } else { - dev_priv->hotplug.stats[pin].count++; DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: %d\n", pin, - dev_priv->hotplug.stats[pin].count); + hpd->stats[pin].count); } return storm; } -static void intel_hpd_irq_storm_disable(struct drm_i915_private *dev_priv) +static void +intel_hpd_irq_storm_switch_to_polling(struct drm_i915_private *dev_priv) { struct drm_device *dev = &dev_priv->drm; struct intel_connector *intel_connector; @@ -210,9 +226,10 @@ static void intel_hpd_irq_storm_reenable_work(struct work_struct *work) container_of(work, typeof(*dev_priv), hotplug.reenable_work.work); struct drm_device *dev = &dev_priv->drm; + intel_wakeref_t wakeref; enum hpd_pin pin; - intel_runtime_pm_get(dev_priv); + wakeref = intel_runtime_pm_get(dev_priv); spin_lock_irq(&dev_priv->irq_lock); for_each_hpd_pin(pin) { @@ -245,7 +262,7 @@ static void intel_hpd_irq_storm_reenable_work(struct work_struct *work) dev_priv->display.hpd_irq_setup(dev_priv); spin_unlock_irq(&dev_priv->irq_lock); - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, wakeref); } bool intel_encoder_hotplug(struct intel_encoder *encoder, @@ -348,8 +365,8 @@ static void i915_hotplug_work_func(struct work_struct *work) hpd_event_bits = dev_priv->hotplug.event_bits; dev_priv->hotplug.event_bits = 0; - /* Disable hotplug on connectors that hit an irq storm. */ - intel_hpd_irq_storm_disable(dev_priv); + /* Enable polling for connectors which had HPD IRQ storms */ + intel_hpd_irq_storm_switch_to_polling(dev_priv); spin_unlock_irq(&dev_priv->irq_lock); @@ -453,7 +470,7 @@ void intel_hpd_irq_handler(struct drm_i915_private *dev_priv, * hotplug bits itself. So only WARN about unexpected * interrupts on saner platforms. */ - WARN_ONCE(!HAS_GMCH_DISPLAY(dev_priv), + WARN_ONCE(!HAS_GMCH(dev_priv), "Received HPD interrupt on pin %d although disabled\n", pin); continue; } @@ -474,15 +491,17 @@ void intel_hpd_irq_handler(struct drm_i915_private *dev_priv, queue_hp = true; } - if (!long_hpd) - continue; - - if (intel_hpd_irq_storm_detect(dev_priv, pin)) { + if (intel_hpd_irq_storm_detect(dev_priv, pin, long_hpd)) { dev_priv->hotplug.event_bits &= ~BIT(pin); storm_detected = true; + queue_hp = true; } } + /* + * Disable any IRQs that storms were detected on. Polling enablement + * happens later in our hotplug work. + */ if (storm_detected && dev_priv->display_irqs_enabled) dev_priv->display.hpd_irq_setup(dev_priv); spin_unlock(&dev_priv->irq_lock); diff --git a/drivers/gpu/drm/i915/intel_huc.c b/drivers/gpu/drm/i915/intel_huc.c index 37ef540dd280..9bd1c9002c2a 100644 --- a/drivers/gpu/drm/i915/intel_huc.c +++ b/drivers/gpu/drm/i915/intel_huc.c @@ -108,20 +108,21 @@ fail: * This function reads status register to verify if HuC * firmware was successfully loaded. * - * Returns positive value if HuC firmware is loaded and verified - * and -ENODEV if HuC is not present. + * Returns: 1 if HuC firmware is loaded and verified, + * 0 if HuC firmware is not loaded and -ENODEV if HuC + * is not present on this platform. */ int intel_huc_check_status(struct intel_huc *huc) { struct drm_i915_private *dev_priv = huc_to_i915(huc); - u32 status; + intel_wakeref_t wakeref; + bool status = false; if (!HAS_HUC(dev_priv)) return -ENODEV; - intel_runtime_pm_get(dev_priv); - status = I915_READ(HUC_STATUS2) & HUC_FW_VERIFIED; - intel_runtime_pm_put(dev_priv); + with_intel_runtime_pm(dev_priv, wakeref) + status = I915_READ(HUC_STATUS2) & HUC_FW_VERIFIED; return status; } diff --git a/drivers/gpu/drm/i915/intel_huc_fw.c b/drivers/gpu/drm/i915/intel_huc_fw.c index f93d2384d482..7d7bfc7f7ca7 100644 --- a/drivers/gpu/drm/i915/intel_huc_fw.c +++ b/drivers/gpu/drm/i915/intel_huc_fw.c @@ -23,8 +23,8 @@ */ #define BXT_HUC_FW_MAJOR 01 -#define BXT_HUC_FW_MINOR 07 -#define BXT_BLD_NUM 1398 +#define BXT_HUC_FW_MINOR 8 +#define BXT_BLD_NUM 2893 #define SKL_HUC_FW_MAJOR 01 #define SKL_HUC_FW_MINOR 07 @@ -76,9 +76,6 @@ static void huc_fw_select(struct intel_uc_fw *huc_fw) huc_fw->path = I915_KBL_HUC_UCODE; huc_fw->major_ver_wanted = KBL_HUC_FW_MAJOR; huc_fw->minor_ver_wanted = KBL_HUC_FW_MINOR; - } else { - DRM_WARN("%s: No firmware known for this platform!\n", - intel_uc_fw_type_repr(huc_fw->type)); } } diff --git a/drivers/gpu/drm/i915/intel_i2c.c b/drivers/gpu/drm/i915/intel_i2c.c index 33d87ab93fdd..5a733e711355 100644 --- a/drivers/gpu/drm/i915/intel_i2c.c +++ b/drivers/gpu/drm/i915/intel_i2c.c @@ -29,7 +29,6 @@ #include <linux/i2c.h> #include <linux/i2c-algo-bit.h> #include <linux/export.h> -#include <drm/drmP.h> #include <drm/drm_hdcp.h> #include "intel_drv.h" #include <drm/i915_drm.h> @@ -698,12 +697,13 @@ out: static int gmbus_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num) { - struct intel_gmbus *bus = container_of(adapter, struct intel_gmbus, - adapter); + struct intel_gmbus *bus = + container_of(adapter, struct intel_gmbus, adapter); struct drm_i915_private *dev_priv = bus->dev_priv; + intel_wakeref_t wakeref; int ret; - intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS); + wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS); if (bus->force_bit) { ret = i2c_bit_algo.master_xfer(adapter, msgs, num); @@ -715,17 +715,16 @@ gmbus_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num) bus->force_bit |= GMBUS_FORCE_BIT_RETRY; } - intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS); + intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS, wakeref); return ret; } int intel_gmbus_output_aksv(struct i2c_adapter *adapter) { - struct intel_gmbus *bus = container_of(adapter, struct intel_gmbus, - adapter); + struct intel_gmbus *bus = + container_of(adapter, struct intel_gmbus, adapter); struct drm_i915_private *dev_priv = bus->dev_priv; - int ret; u8 cmd = DRM_HDCP_DDC_AKSV; u8 buf[DRM_HDCP_KSV_LEN] = { 0 }; struct i2c_msg msgs[] = { @@ -742,8 +741,10 @@ int intel_gmbus_output_aksv(struct i2c_adapter *adapter) .buf = buf, } }; + intel_wakeref_t wakeref; + int ret; - intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS); + wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS); mutex_lock(&dev_priv->gmbus_mutex); /* @@ -754,7 +755,7 @@ int intel_gmbus_output_aksv(struct i2c_adapter *adapter) ret = do_gmbus_xfer(adapter, msgs, ARRAY_SIZE(msgs), GMBUS_AKSV_SELECT); mutex_unlock(&dev_priv->gmbus_mutex); - intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS); + intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS, wakeref); return ret; } @@ -817,12 +818,12 @@ int intel_setup_gmbus(struct drm_i915_private *dev_priv) unsigned int pin; int ret; - if (INTEL_INFO(dev_priv)->num_pipes == 0) + if (!HAS_DISPLAY(dev_priv)) return 0; if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) dev_priv->gpio_mmio_base = VLV_DISPLAY_BASE; - else if (!HAS_GMCH_DISPLAY(dev_priv)) + else if (!HAS_GMCH(dev_priv)) /* * Broxton uses the same PCH offsets for South Display Engine, * even though it doesn't have a PCH. diff --git a/drivers/gpu/drm/i915/intel_lpe_audio.c b/drivers/gpu/drm/i915/intel_lpe_audio.c index 5d5336fbe7b0..f8239bca3820 100644 --- a/drivers/gpu/drm/i915/intel_lpe_audio.c +++ b/drivers/gpu/drm/i915/intel_lpe_audio.c @@ -65,6 +65,7 @@ #include <linux/irq.h> #include <linux/pci.h> #include <linux/pm_runtime.h> +#include <linux/platform_device.h> #include "i915_drv.h" #include <linux/delay.h> diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c index 58d1d3d47dd3..5e98fd79bd9d 100644 --- a/drivers/gpu/drm/i915/intel_lrc.c +++ b/drivers/gpu/drm/i915/intel_lrc.c @@ -133,10 +133,10 @@ */ #include <linux/interrupt.h> -#include <drm/drmP.h> #include <drm/i915_drm.h> #include "i915_drv.h" #include "i915_gem_render_state.h" +#include "i915_reset.h" #include "i915_vgpu.h" #include "intel_lrc_reg.h" #include "intel_mocs.h" @@ -172,6 +172,12 @@ static void execlists_init_reg_state(u32 *reg_state, struct intel_engine_cs *engine, struct intel_ring *ring); +static inline u32 intel_hws_seqno_address(struct intel_engine_cs *engine) +{ + return (i915_ggtt_offset(engine->status_page.vma) + + I915_GEM_HWS_INDEX_ADDR); +} + static inline struct i915_priolist *to_priolist(struct rb_node *rb) { return rb_entry(rb, struct i915_priolist, node); @@ -182,13 +188,90 @@ static inline int rq_prio(const struct i915_request *rq) return rq->sched.attr.priority; } +static int queue_prio(const struct intel_engine_execlists *execlists) +{ + struct i915_priolist *p; + struct rb_node *rb; + + rb = rb_first_cached(&execlists->queue); + if (!rb) + return INT_MIN; + + /* + * As the priolist[] are inverted, with the highest priority in [0], + * we have to flip the index value to become priority. + */ + p = to_priolist(rb); + return ((p->priority + 1) << I915_USER_PRIORITY_SHIFT) - ffs(p->used); +} + static inline bool need_preempt(const struct intel_engine_cs *engine, - const struct i915_request *last, - int prio) + const struct i915_request *rq) +{ + const int last_prio = rq_prio(rq); + + if (!intel_engine_has_preemption(engine)) + return false; + + if (i915_request_completed(rq)) + return false; + + /* + * Check if the current priority hint merits a preemption attempt. + * + * We record the highest value priority we saw during rescheduling + * prior to this dequeue, therefore we know that if it is strictly + * less than the current tail of ESLP[0], we do not need to force + * a preempt-to-idle cycle. + * + * However, the priority hint is a mere hint that we may need to + * preempt. If that hint is stale or we may be trying to preempt + * ourselves, ignore the request. + */ + if (!__execlists_need_preempt(engine->execlists.queue_priority_hint, + last_prio)) + return false; + + /* + * Check against the first request in ELSP[1], it will, thanks to the + * power of PI, be the highest priority of that context. + */ + if (!list_is_last(&rq->link, &engine->timeline.requests) && + rq_prio(list_next_entry(rq, link)) > last_prio) + return true; + + /* + * If the inflight context did not trigger the preemption, then maybe + * it was the set of queued requests? Pick the highest priority in + * the queue (the first active priolist) and see if it deserves to be + * running instead of ELSP[0]. + * + * The highest priority request in the queue can not be either + * ELSP[0] or ELSP[1] as, thanks again to PI, if it was the same + * context, it's priority would not exceed ELSP[0] aka last_prio. + */ + return queue_prio(&engine->execlists) > last_prio; +} + +__maybe_unused static inline bool +assert_priority_queue(const struct intel_engine_execlists *execlists, + const struct i915_request *prev, + const struct i915_request *next) { - return (intel_engine_has_preemption(engine) && - __execlists_need_preempt(prio, rq_prio(last)) && - !i915_request_completed(last)); + if (!prev) + return true; + + /* + * Without preemption, the prev may refer to the still active element + * which we refuse to let go. + * + * Even with preemption, there are times when we think it is better not + * to preempt and leave an ostensibly lower priority request in flight. + */ + if (port_request(execlists->port) == prev) + return true; + + return rq_prio(prev) >= rq_prio(next); } /* @@ -259,74 +342,18 @@ intel_lr_context_descriptor_update(struct i915_gem_context *ctx, ce->lrc_desc = desc; } -static struct i915_priolist * -lookup_priolist(struct intel_engine_cs *engine, int prio) -{ - struct intel_engine_execlists * const execlists = &engine->execlists; - struct i915_priolist *p; - struct rb_node **parent, *rb; - bool first = true; - - if (unlikely(execlists->no_priolist)) - prio = I915_PRIORITY_NORMAL; - -find_priolist: - /* most positive priority is scheduled first, equal priorities fifo */ - rb = NULL; - parent = &execlists->queue.rb_root.rb_node; - while (*parent) { - rb = *parent; - p = to_priolist(rb); - if (prio > p->priority) { - parent = &rb->rb_left; - } else if (prio < p->priority) { - parent = &rb->rb_right; - first = false; - } else { - return p; - } - } - - if (prio == I915_PRIORITY_NORMAL) { - p = &execlists->default_priolist; - } else { - p = kmem_cache_alloc(engine->i915->priorities, GFP_ATOMIC); - /* Convert an allocation failure to a priority bump */ - if (unlikely(!p)) { - prio = I915_PRIORITY_NORMAL; /* recurses just once */ - - /* To maintain ordering with all rendering, after an - * allocation failure we have to disable all scheduling. - * Requests will then be executed in fifo, and schedule - * will ensure that dependencies are emitted in fifo. - * There will be still some reordering with existing - * requests, so if userspace lied about their - * dependencies that reordering may be visible. - */ - execlists->no_priolist = true; - goto find_priolist; - } - } - - p->priority = prio; - INIT_LIST_HEAD(&p->requests); - rb_link_node(&p->node, rb, parent); - rb_insert_color_cached(&p->node, &execlists->queue, first); - - return p; -} - static void unwind_wa_tail(struct i915_request *rq) { rq->tail = intel_ring_wrap(rq->ring, rq->wa_tail - WA_TAIL_BYTES); assert_ring_tail_valid(rq->ring, rq->tail); } -static void __unwind_incomplete_requests(struct intel_engine_cs *engine) +static struct i915_request * +__unwind_incomplete_requests(struct intel_engine_cs *engine) { - struct i915_request *rq, *rn; - struct i915_priolist *uninitialized_var(p); - int last_prio = I915_PRIORITY_INVALID; + struct i915_request *rq, *rn, *active = NULL; + struct list_head *uninitialized_var(pl); + int prio = I915_PRIORITY_INVALID | I915_PRIORITY_NEWCLIENT; lockdep_assert_held(&engine->timeline.lock); @@ -334,20 +361,38 @@ static void __unwind_incomplete_requests(struct intel_engine_cs *engine) &engine->timeline.requests, link) { if (i915_request_completed(rq)) - return; + break; __i915_request_unsubmit(rq); unwind_wa_tail(rq); + GEM_BUG_ON(rq->hw_context->active); + GEM_BUG_ON(rq_prio(rq) == I915_PRIORITY_INVALID); - if (rq_prio(rq) != last_prio) { - last_prio = rq_prio(rq); - p = lookup_priolist(engine, last_prio); + if (rq_prio(rq) != prio) { + prio = rq_prio(rq); + pl = i915_sched_lookup_priolist(engine, prio); } + GEM_BUG_ON(RB_EMPTY_ROOT(&engine->execlists.queue.rb_root)); + + list_add(&rq->sched.link, pl); - GEM_BUG_ON(p->priority != rq_prio(rq)); - list_add(&rq->sched.link, &p->requests); + active = rq; } + + /* + * The active request is now effectively the start of a new client + * stream, so give it the equivalent small priority bump to prevent + * it being gazumped a second time by another peer. + */ + if (!(prio & I915_PRIORITY_NEWCLIENT)) { + prio |= I915_PRIORITY_NEWCLIENT; + active->sched.attr.priority = prio; + list_move_tail(&active->sched.link, + i915_sched_lookup_priolist(engine, prio)); + } + + return active; } void @@ -355,13 +400,8 @@ execlists_unwind_incomplete_requests(struct intel_engine_execlists *execlists) { struct intel_engine_cs *engine = container_of(execlists, typeof(*engine), execlists); - unsigned long flags; - - spin_lock_irqsave(&engine->timeline.lock, flags); __unwind_incomplete_requests(engine); - - spin_unlock_irqrestore(&engine->timeline.lock, flags); } static inline void @@ -394,44 +434,28 @@ execlists_user_end(struct intel_engine_execlists *execlists) static inline void execlists_context_schedule_in(struct i915_request *rq) { + GEM_BUG_ON(rq->hw_context->active); + execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_IN); intel_engine_context_in(rq->engine); + rq->hw_context->active = rq->engine; } static inline void execlists_context_schedule_out(struct i915_request *rq, unsigned long status) { + rq->hw_context->active = NULL; intel_engine_context_out(rq->engine); execlists_context_status_change(rq, status); trace_i915_request_out(rq); } -static void -execlists_update_context_pdps(struct i915_hw_ppgtt *ppgtt, u32 *reg_state) -{ - ASSIGN_CTX_PDP(ppgtt, reg_state, 3); - ASSIGN_CTX_PDP(ppgtt, reg_state, 2); - ASSIGN_CTX_PDP(ppgtt, reg_state, 1); - ASSIGN_CTX_PDP(ppgtt, reg_state, 0); -} - static u64 execlists_update_context(struct i915_request *rq) { struct intel_context *ce = rq->hw_context; - struct i915_hw_ppgtt *ppgtt = - rq->gem_context->ppgtt ?: rq->i915->mm.aliasing_ppgtt; - u32 *reg_state = ce->lrc_reg_state; - - reg_state[CTX_RING_TAIL+1] = intel_ring_set_tail(rq->ring, rq->tail); - /* - * True 32b PPGTT with dynamic page allocation: update PDP - * registers and point the unallocated PDPs to scratch page. - * PML4 is allocated during ppgtt init, so this is not needed - * in 48-bit mode. - */ - if (ppgtt && !i915_vm_is_48bit(&ppgtt->vm)) - execlists_update_context_pdps(ppgtt, reg_state); + ce->lrc_reg_state[CTX_RING_TAIL + 1] = + intel_ring_set_tail(rq->ring, rq->tail); /* * Make sure the context image is complete before we submit it to HW. @@ -499,11 +523,12 @@ static void execlists_submit_ports(struct intel_engine_cs *engine) desc = execlists_update_context(rq); GEM_DEBUG_EXEC(port[n].context_id = upper_32_bits(desc)); - GEM_TRACE("%s in[%d]: ctx=%d.%d, global=%d (fence %llx:%d) (current %d), prio=%d\n", + GEM_TRACE("%s in[%d]: ctx=%d.%d, global=%d (fence %llx:%lld) (current %d:%d), prio=%d\n", engine->name, n, port[n].context_id, count, rq->global_seqno, rq->fence.context, rq->fence.seqno, + hwsp_seqno(rq), intel_engine_get_seqno(engine), rq_prio(rq)); } else { @@ -575,6 +600,8 @@ static void inject_preempt_context(struct intel_engine_cs *engine) execlists_clear_active(execlists, EXECLISTS_ACTIVE_HWACK); execlists_set_active(execlists, EXECLISTS_ACTIVE_PREEMPT); + + (void)I915_SELFTEST_ONLY(execlists->preempt_hang.count++); } static void complete_preempt_context(struct intel_engine_execlists *execlists) @@ -643,7 +670,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) if (!execlists_is_active(execlists, EXECLISTS_ACTIVE_HWACK)) return; - if (need_preempt(engine, last, execlists->queue_priority)) { + if (need_preempt(engine, last)) { inject_preempt_context(engine); return; } @@ -676,7 +703,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) * WaIdleLiteRestore:bdw,skl * Apply the wa NOOPs to prevent * ring:HEAD == rq:TAIL as we resubmit the - * request. See gen8_emit_breadcrumb() for + * request. See gen8_emit_fini_breadcrumb() for * where we prepare the padding after the * end of the request. */ @@ -686,8 +713,11 @@ static void execlists_dequeue(struct intel_engine_cs *engine) while ((rb = rb_first_cached(&execlists->queue))) { struct i915_priolist *p = to_priolist(rb); struct i915_request *rq, *rn; + int i; + + priolist_for_each_request_consume(rq, rn, p, i) { + GEM_BUG_ON(!assert_priority_queue(execlists, last, rq)); - list_for_each_entry_safe(rq, rn, &p->requests, sched.link) { /* * Can we combine this request with the current port? * It has to be the same context/ringbuffer and not @@ -706,11 +736,8 @@ static void execlists_dequeue(struct intel_engine_cs *engine) * combine this request with the last, then we * are done. */ - if (port == last_port) { - __list_del_many(&p->requests, - &rq->sched.link); + if (port == last_port) goto done; - } /* * If GVT overrides us we only ever submit @@ -720,11 +747,8 @@ static void execlists_dequeue(struct intel_engine_cs *engine) * request) to the second port. */ if (ctx_single_port_submission(last->hw_context) || - ctx_single_port_submission(rq->hw_context)) { - __list_del_many(&p->requests, - &rq->sched.link); + ctx_single_port_submission(rq->hw_context)) goto done; - } GEM_BUG_ON(last->hw_context == rq->hw_context); @@ -735,15 +759,16 @@ static void execlists_dequeue(struct intel_engine_cs *engine) GEM_BUG_ON(port_isset(port)); } - INIT_LIST_HEAD(&rq->sched.link); + list_del_init(&rq->sched.link); + __i915_request_submit(rq); trace_i915_request_in(rq, port_index(port, execlists)); + last = rq; submit = true; } rb_erase_cached(&p->node, &execlists->queue); - INIT_LIST_HEAD(&p->requests); if (p->priority != I915_PRIORITY_NORMAL) kmem_cache_free(engine->i915->priorities, p); } @@ -752,20 +777,20 @@ done: /* * Here be a bit of magic! Or sleight-of-hand, whichever you prefer. * - * We choose queue_priority such that if we add a request of greater + * We choose the priority hint such that if we add a request of greater * priority than this, we kick the submission tasklet to decide on * the right order of submitting the requests to hardware. We must * also be prepared to reorder requests as they are in-flight on the - * HW. We derive the queue_priority then as the first "hole" in + * HW. We derive the priority hint then as the first "hole" in * the HW submission ports and if there are no available slots, * the priority of the lowest executing request, i.e. last. * * When we do receive a higher priority request ready to run from the - * user, see queue_request(), the queue_priority is bumped to that + * user, see queue_request(), the priority hint is bumped to that * request triggering preemption on the next dequeue (or subsequent * interrupt for secondary ports). */ - execlists->queue_priority = + execlists->queue_priority_hint = port != execlists->port ? rq_prio(last) : INT_MIN; if (submit) { @@ -796,11 +821,12 @@ execlists_cancel_port_requests(struct intel_engine_execlists * const execlists) while (num_ports-- && port_isset(port)) { struct i915_request *rq = port_request(port); - GEM_TRACE("%s:port%u global=%d (fence %llx:%d), (current %d)\n", + GEM_TRACE("%s:port%u global=%d (fence %llx:%lld), (current %d:%d)\n", rq->engine->name, (unsigned int)(port - execlists->port), rq->global_seqno, rq->fence.context, rq->fence.seqno, + hwsp_seqno(rq), intel_engine_get_seqno(rq->engine)); GEM_BUG_ON(!execlists->active); @@ -818,8 +844,17 @@ execlists_cancel_port_requests(struct intel_engine_execlists * const execlists) execlists_clear_all_active(execlists); } +static inline void +invalidate_csb_entries(const u32 *first, const u32 *last) +{ + clflush((void *)first); + clflush((void *)last); +} + static void reset_csb_pointers(struct intel_engine_execlists *execlists) { + const unsigned int reset_value = GEN8_CSB_ENTRIES - 1; + /* * After a reset, the HW starts writing into CSB entry [0]. We * therefore have to set our HEAD pointer back one entry so that @@ -829,8 +864,11 @@ static void reset_csb_pointers(struct intel_engine_execlists *execlists) * inline comparison of our cached head position against the last HW * write works even before the first interrupt. */ - execlists->csb_head = execlists->csb_write_reset; - WRITE_ONCE(*execlists->csb_write, execlists->csb_write_reset); + execlists->csb_head = reset_value; + WRITE_ONCE(*execlists->csb_write, reset_value); + + invalidate_csb_entries(&execlists->csb_status[0], + &execlists->csb_status[GEN8_CSB_ENTRIES - 1]); } static void nop_submission_tasklet(unsigned long data) @@ -871,30 +909,37 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine) /* Mark all executing requests as skipped. */ list_for_each_entry(rq, &engine->timeline.requests, link) { GEM_BUG_ON(!rq->global_seqno); - if (!i915_request_completed(rq)) + + if (!i915_request_signaled(rq)) dma_fence_set_error(&rq->fence, -EIO); + + i915_request_mark_complete(rq); } /* Flush the queued requests to the timeline list (for retiring). */ while ((rb = rb_first_cached(&execlists->queue))) { struct i915_priolist *p = to_priolist(rb); + int i; - list_for_each_entry_safe(rq, rn, &p->requests, sched.link) { - INIT_LIST_HEAD(&rq->sched.link); - - dma_fence_set_error(&rq->fence, -EIO); + priolist_for_each_request_consume(rq, rn, p, i) { + list_del_init(&rq->sched.link); __i915_request_submit(rq); + dma_fence_set_error(&rq->fence, -EIO); + i915_request_mark_complete(rq); } rb_erase_cached(&p->node, &execlists->queue); - INIT_LIST_HEAD(&p->requests); if (p->priority != I915_PRIORITY_NORMAL) kmem_cache_free(engine->i915->priorities, p); } + intel_write_status_page(engine, + I915_GEM_HWS_INDEX, + intel_engine_last_submit(engine)); + /* Remaining _unready_ requests will be nop'ed when submitted */ - execlists->queue_priority = INT_MIN; + execlists->queue_priority_hint = INT_MIN; execlists->queue = RB_ROOT_CACHED; GEM_BUG_ON(port_isset(execlists->port)); @@ -917,6 +962,8 @@ static void process_csb(struct intel_engine_cs *engine) const u32 * const buf = execlists->csb_status; u8 head, tail; + lockdep_assert_held(&engine->timeline.lock); + /* * Note that csb_write, csb_status may be either in HWSP or mmio. * When reading from the csb_write mmio register, we have to be @@ -1005,12 +1052,13 @@ static void process_csb(struct intel_engine_cs *engine) EXECLISTS_ACTIVE_USER)); rq = port_unpack(port, &count); - GEM_TRACE("%s out[0]: ctx=%d.%d, global=%d (fence %llx:%d) (current %d), prio=%d\n", + GEM_TRACE("%s out[0]: ctx=%d.%d, global=%d (fence %llx:%lld) (current %d:%d), prio=%d\n", engine->name, port->context_id, count, rq ? rq->global_seqno : 0, rq ? rq->fence.context : 0, rq ? rq->fence.seqno : 0, + rq ? hwsp_seqno(rq) : 0, intel_engine_get_seqno(engine), rq ? rq_prio(rq) : 0); @@ -1059,6 +1107,19 @@ static void process_csb(struct intel_engine_cs *engine) } while (head != tail); execlists->csb_head = head; + + /* + * Gen11 has proven to fail wrt global observation point between + * entry and tail update, failing on the ordering and thus + * we see an old entry in the context status buffer. + * + * Forcibly evict out entries for the next gpu csb update, + * to increase the odds that we get a fresh entries with non + * working hardware. The cost for doing so comes out mostly with + * the wash as hardware, working or not, will need to do the + * invalidation before. + */ + invalidate_csb_entries(&buf[0], &buf[GEN8_CSB_ENTRIES - 1]); } static void __execlists_submission_tasklet(struct intel_engine_cs *const engine) @@ -1081,7 +1142,7 @@ static void execlists_submission_tasklet(unsigned long data) GEM_TRACE("%s awake?=%d, active=%x\n", engine->name, - engine->i915->gt.awake, + !!engine->i915->gt.awake, engine->execlists.active); spin_lock_irqsave(&engine->timeline.lock, flags); @@ -1093,13 +1154,7 @@ static void queue_request(struct intel_engine_cs *engine, struct i915_sched_node *node, int prio) { - list_add_tail(&node->link, - &lookup_priolist(engine, prio)->requests); -} - -static void __update_queue(struct intel_engine_cs *engine, int prio) -{ - engine->execlists.queue_priority = prio; + list_add_tail(&node->link, i915_sched_lookup_priolist(engine, prio)); } static void __submit_queue_imm(struct intel_engine_cs *engine) @@ -1117,8 +1172,8 @@ static void __submit_queue_imm(struct intel_engine_cs *engine) static void submit_queue(struct intel_engine_cs *engine, int prio) { - if (prio > engine->execlists.queue_priority) { - __update_queue(engine, prio); + if (prio > engine->execlists.queue_priority_hint) { + engine->execlists.queue_priority_hint = prio; __submit_queue_imm(engine); } } @@ -1141,139 +1196,6 @@ static void execlists_submit_request(struct i915_request *request) spin_unlock_irqrestore(&engine->timeline.lock, flags); } -static struct i915_request *sched_to_request(struct i915_sched_node *node) -{ - return container_of(node, struct i915_request, sched); -} - -static struct intel_engine_cs * -sched_lock_engine(struct i915_sched_node *node, struct intel_engine_cs *locked) -{ - struct intel_engine_cs *engine = sched_to_request(node)->engine; - - GEM_BUG_ON(!locked); - - if (engine != locked) { - spin_unlock(&locked->timeline.lock); - spin_lock(&engine->timeline.lock); - } - - return engine; -} - -static void execlists_schedule(struct i915_request *request, - const struct i915_sched_attr *attr) -{ - struct i915_priolist *uninitialized_var(pl); - struct intel_engine_cs *engine, *last; - struct i915_dependency *dep, *p; - struct i915_dependency stack; - const int prio = attr->priority; - LIST_HEAD(dfs); - - GEM_BUG_ON(prio == I915_PRIORITY_INVALID); - - if (i915_request_completed(request)) - return; - - if (prio <= READ_ONCE(request->sched.attr.priority)) - return; - - /* Need BKL in order to use the temporary link inside i915_dependency */ - lockdep_assert_held(&request->i915->drm.struct_mutex); - - stack.signaler = &request->sched; - list_add(&stack.dfs_link, &dfs); - - /* - * Recursively bump all dependent priorities to match the new request. - * - * A naive approach would be to use recursion: - * static void update_priorities(struct i915_sched_node *node, prio) { - * list_for_each_entry(dep, &node->signalers_list, signal_link) - * update_priorities(dep->signal, prio) - * queue_request(node); - * } - * but that may have unlimited recursion depth and so runs a very - * real risk of overunning the kernel stack. Instead, we build - * a flat list of all dependencies starting with the current request. - * As we walk the list of dependencies, we add all of its dependencies - * to the end of the list (this may include an already visited - * request) and continue to walk onwards onto the new dependencies. The - * end result is a topological list of requests in reverse order, the - * last element in the list is the request we must execute first. - */ - list_for_each_entry(dep, &dfs, dfs_link) { - struct i915_sched_node *node = dep->signaler; - - /* - * Within an engine, there can be no cycle, but we may - * refer to the same dependency chain multiple times - * (redundant dependencies are not eliminated) and across - * engines. - */ - list_for_each_entry(p, &node->signalers_list, signal_link) { - GEM_BUG_ON(p == dep); /* no cycles! */ - - if (i915_sched_node_signaled(p->signaler)) - continue; - - GEM_BUG_ON(p->signaler->attr.priority < node->attr.priority); - if (prio > READ_ONCE(p->signaler->attr.priority)) - list_move_tail(&p->dfs_link, &dfs); - } - } - - /* - * If we didn't need to bump any existing priorities, and we haven't - * yet submitted this request (i.e. there is no potential race with - * execlists_submit_request()), we can set our own priority and skip - * acquiring the engine locks. - */ - if (request->sched.attr.priority == I915_PRIORITY_INVALID) { - GEM_BUG_ON(!list_empty(&request->sched.link)); - request->sched.attr = *attr; - if (stack.dfs_link.next == stack.dfs_link.prev) - return; - __list_del_entry(&stack.dfs_link); - } - - last = NULL; - engine = request->engine; - spin_lock_irq(&engine->timeline.lock); - - /* Fifo and depth-first replacement ensure our deps execute before us */ - list_for_each_entry_safe_reverse(dep, p, &dfs, dfs_link) { - struct i915_sched_node *node = dep->signaler; - - INIT_LIST_HEAD(&dep->dfs_link); - - engine = sched_lock_engine(node, engine); - - if (prio <= node->attr.priority) - continue; - - node->attr.priority = prio; - if (!list_empty(&node->link)) { - if (last != engine) { - pl = lookup_priolist(engine, prio); - last = engine; - } - GEM_BUG_ON(pl->priority != prio); - list_move_tail(&node->link, &pl->requests); - } - - if (prio > engine->execlists.queue_priority && - i915_sw_fence_done(&sched_to_request(node)->submit)) { - /* defer submission until after all of our updates */ - __update_queue(engine, prio); - tasklet_hi_schedule(&engine->execlists.tasklet); - } - } - - spin_unlock_irq(&engine->timeline.lock); -} - static void execlists_context_destroy(struct intel_context *ce) { GEM_BUG_ON(ce->pin_count); @@ -1289,6 +1211,28 @@ static void execlists_context_destroy(struct intel_context *ce) static void execlists_context_unpin(struct intel_context *ce) { + struct intel_engine_cs *engine; + + /* + * The tasklet may still be using a pointer to our state, via an + * old request. However, since we know we only unpin the context + * on retirement of the following request, we know that the last + * request referencing us will have had a completion CS interrupt. + * If we see that it is still active, it means that the tasklet hasn't + * had the chance to run yet; let it run before we teardown the + * reference it may use. + */ + engine = READ_ONCE(ce->active); + if (unlikely(engine)) { + unsigned long flags; + + spin_lock_irqsave(&engine->timeline.lock, flags); + process_csb(engine); + spin_unlock_irqrestore(&engine->timeline.lock, flags); + + GEM_BUG_ON(READ_ONCE(ce->active)); + } + i915_gem_context_unpin_hw_id(ce->gem_context); intel_ring_unpin(ce->ring); @@ -1322,6 +1266,23 @@ static int __context_pin(struct i915_gem_context *ctx, struct i915_vma *vma) return i915_vma_pin(vma, 0, 0, flags); } +static void +__execlists_update_reg_state(struct intel_engine_cs *engine, + struct intel_context *ce) +{ + u32 *regs = ce->lrc_reg_state; + struct intel_ring *ring = ce->ring; + + regs[CTX_RING_BUFFER_START + 1] = i915_ggtt_offset(ring->vma); + regs[CTX_RING_HEAD + 1] = ring->head; + regs[CTX_RING_TAIL + 1] = ring->tail; + + /* RPCS */ + if (engine->class == RENDER_CLASS) + regs[CTX_R_PWR_CLK_STATE + 1] = gen8_make_rpcs(engine->i915, + &ce->sseu); +} + static struct intel_context * __execlists_context_pin(struct intel_engine_cs *engine, struct i915_gem_context *ctx, @@ -1360,10 +1321,8 @@ __execlists_context_pin(struct intel_engine_cs *engine, GEM_BUG_ON(!intel_ring_offset_valid(ce->ring, ce->ring->head)); ce->lrc_reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE; - ce->lrc_reg_state[CTX_RING_BUFFER_START+1] = - i915_ggtt_offset(ce->ring->vma); - ce->lrc_reg_state[CTX_RING_HEAD + 1] = ce->ring->head; - ce->lrc_reg_state[CTX_RING_TAIL + 1] = ce->ring->tail; + + __execlists_update_reg_state(engine, ce); ce->state->obj->pin_global++; i915_gem_context_get(ctx); @@ -1392,6 +1351,7 @@ execlists_context_pin(struct intel_engine_cs *engine, struct intel_context *ce = to_intel_context(ctx, engine); lockdep_assert_held(&ctx->i915->drm.struct_mutex); + GEM_BUG_ON(!ctx->ppgtt); if (likely(ce->pin_count++)) return ce; @@ -1402,29 +1362,116 @@ execlists_context_pin(struct intel_engine_cs *engine, return __execlists_context_pin(engine, ctx, ce); } +static int gen8_emit_init_breadcrumb(struct i915_request *rq) +{ + u32 *cs; + + GEM_BUG_ON(!rq->timeline->has_initial_breadcrumb); + + cs = intel_ring_begin(rq, 6); + if (IS_ERR(cs)) + return PTR_ERR(cs); + + /* + * Check if we have been preempted before we even get started. + * + * After this point i915_request_started() reports true, even if + * we get preempted and so are no longer running. + */ + *cs++ = MI_ARB_CHECK; + *cs++ = MI_NOOP; + + *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; + *cs++ = rq->timeline->hwsp_offset; + *cs++ = 0; + *cs++ = rq->fence.seqno - 1; + + intel_ring_advance(rq, cs); + return 0; +} + +static int emit_pdps(struct i915_request *rq) +{ + const struct intel_engine_cs * const engine = rq->engine; + struct i915_hw_ppgtt * const ppgtt = rq->gem_context->ppgtt; + int err, i; + u32 *cs; + + GEM_BUG_ON(intel_vgpu_active(rq->i915)); + + /* + * Beware ye of the dragons, this sequence is magic! + * + * Small changes to this sequence can cause anything from + * GPU hangs to forcewake errors and machine lockups! + */ + + /* Flush any residual operations from the context load */ + err = engine->emit_flush(rq, EMIT_FLUSH); + if (err) + return err; + + /* Magic required to prevent forcewake errors! */ + err = engine->emit_flush(rq, EMIT_INVALIDATE); + if (err) + return err; + + cs = intel_ring_begin(rq, 4 * GEN8_3LVL_PDPES + 2); + if (IS_ERR(cs)) + return PTR_ERR(cs); + + /* Ensure the LRI have landed before we invalidate & continue */ + *cs++ = MI_LOAD_REGISTER_IMM(2 * GEN8_3LVL_PDPES) | MI_LRI_FORCE_POSTED; + for (i = GEN8_3LVL_PDPES; i--; ) { + const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i); + + *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(engine, i)); + *cs++ = upper_32_bits(pd_daddr); + *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(engine, i)); + *cs++ = lower_32_bits(pd_daddr); + } + *cs++ = MI_NOOP; + + intel_ring_advance(rq, cs); + + /* Be doubly sure the LRI have landed before proceeding */ + err = engine->emit_flush(rq, EMIT_FLUSH); + if (err) + return err; + + /* Re-invalidate the TLB for luck */ + return engine->emit_flush(rq, EMIT_INVALIDATE); +} + static int execlists_request_alloc(struct i915_request *request) { int ret; GEM_BUG_ON(!request->hw_context->pin_count); - /* Flush enough space to reduce the likelihood of waiting after + /* + * Flush enough space to reduce the likelihood of waiting after * we start building the request - in which case we will just * have to repeat work. */ request->reserved_space += EXECLISTS_REQUEST_SIZE; - ret = intel_ring_wait_for_space(request->ring, request->reserved_space); - if (ret) - return ret; - - /* Note that after this point, we have committed to using + /* + * Note that after this point, we have committed to using * this request as it is being used to both track the * state of engine initialisation and liveness of the * golden renderstate above. Think twice before you try * to cancel/unwind this request now. */ + /* Unconditionally invalidate GPU caches and TLBs. */ + if (i915_vm_is_48bit(&request->gem_context->ppgtt->vm)) + ret = request->engine->emit_flush(request, EMIT_INVALIDATE); + else + ret = emit_pdps(request); + if (ret) + return ret; + request->reserved_space -= EXECLISTS_REQUEST_SIZE; return 0; } @@ -1571,18 +1618,6 @@ static u32 *gen9_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch) batch = emit_lri(batch, lri, ARRAY_SIZE(lri)); - /* WaClearSlmSpaceAtContextSwitch:kbl */ - /* Actual scratch location is at 128 bytes offset */ - if (IS_KBL_REVID(engine->i915, 0, KBL_REVID_A0)) { - batch = gen8_emit_pipe_control(batch, - PIPE_CONTROL_FLUSH_L3 | - PIPE_CONTROL_GLOBAL_GTT_IVB | - PIPE_CONTROL_CS_STALL | - PIPE_CONTROL_QW_WRITE, - i915_scratch_offset(engine->i915) - + 2 * CACHELINE_BYTES); - } - /* WaMediaPoolStateCmdInWABB:bxt,glk */ if (HAS_POOLED_EU(engine->i915)) { /* @@ -1697,7 +1732,7 @@ static int intel_init_workaround_bb(struct intel_engine_cs *engine) unsigned int i; int ret; - if (GEM_WARN_ON(engine->id != RCS)) + if (GEM_DEBUG_WARN_ON(engine->id != RCS)) return -EINVAL; switch (INTEL_GEN(engine->i915)) { @@ -1736,8 +1771,8 @@ static int intel_init_workaround_bb(struct intel_engine_cs *engine) */ for (i = 0; i < ARRAY_SIZE(wa_bb_fn); i++) { wa_bb[i]->offset = batch_ptr - batch; - if (GEM_WARN_ON(!IS_ALIGNED(wa_bb[i]->offset, - CACHELINE_BYTES))) { + if (GEM_DEBUG_WARN_ON(!IS_ALIGNED(wa_bb[i]->offset, + CACHELINE_BYTES))) { ret = -EINVAL; break; } @@ -1759,7 +1794,7 @@ static void enable_execlists(struct intel_engine_cs *engine) { struct drm_i915_private *dev_priv = engine->i915; - I915_WRITE(RING_HWSTAM(engine->mmio_base), 0xffffffff); + intel_engine_set_hwsp_writemask(engine, ~0u); /* HWSTAM */ /* * Make sure we're not enabling the new 12-deep CSB @@ -1780,7 +1815,7 @@ static void enable_execlists(struct intel_engine_cs *engine) _MASKED_BIT_DISABLE(STOP_RING)); I915_WRITE(RING_HWS_PGA(engine->mmio_base), - engine->status_page.ggtt_offset); + i915_ggtt_offset(engine->status_page.vma)); POSTING_READ(RING_HWS_PGA(engine->mmio_base)); } @@ -1800,6 +1835,7 @@ static bool unexpected_starting_state(struct intel_engine_cs *engine) static int gen8_init_common_ring(struct intel_engine_cs *engine) { intel_engine_apply_workarounds(engine); + intel_engine_apply_whitelist(engine); intel_mocs_init_engine(engine); @@ -1816,48 +1852,9 @@ static int gen8_init_common_ring(struct intel_engine_cs *engine) return 0; } -static int gen8_init_render_ring(struct intel_engine_cs *engine) -{ - struct drm_i915_private *dev_priv = engine->i915; - int ret; - - ret = gen8_init_common_ring(engine); - if (ret) - return ret; - - intel_whitelist_workarounds_apply(engine); - - /* We need to disable the AsyncFlip performance optimisations in order - * to use MI_WAIT_FOR_EVENT within the CS. It should already be - * programmed to '1' on all products. - * - * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw,chv - */ - I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE)); - - I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING)); - - return 0; -} - -static int gen9_init_render_ring(struct intel_engine_cs *engine) -{ - int ret; - - ret = gen8_init_common_ring(engine); - if (ret) - return ret; - - intel_whitelist_workarounds_apply(engine); - - return 0; -} - -static struct i915_request * -execlists_reset_prepare(struct intel_engine_cs *engine) +static void execlists_reset_prepare(struct intel_engine_cs *engine) { struct intel_engine_execlists * const execlists = &engine->execlists; - struct i915_request *request, *active; unsigned long flags; GEM_TRACE("%s: depth<-%d\n", engine->name, @@ -1873,59 +1870,21 @@ execlists_reset_prepare(struct intel_engine_cs *engine) * prevents the race. */ __tasklet_disable_sync_once(&execlists->tasklet); + GEM_BUG_ON(!reset_in_progress(execlists)); + /* And flush any current direct submission. */ spin_lock_irqsave(&engine->timeline.lock, flags); - - /* - * We want to flush the pending context switches, having disabled - * the tasklet above, we can assume exclusive access to the execlists. - * For this allows us to catch up with an inflight preemption event, - * and avoid blaming an innocent request if the stall was due to the - * preemption itself. - */ - process_csb(engine); - - /* - * The last active request can then be no later than the last request - * now in ELSP[0]. So search backwards from there, so that if the GPU - * has advanced beyond the last CSB update, it will be pardoned. - */ - active = NULL; - request = port_request(execlists->port); - if (request) { - /* - * Prevent the breadcrumb from advancing before we decide - * which request is currently active. - */ - intel_engine_stop_cs(engine); - - list_for_each_entry_from_reverse(request, - &engine->timeline.requests, - link) { - if (__i915_request_completed(request, - request->global_seqno)) - break; - - active = request; - } - } - + process_csb(engine); /* drain preemption events */ spin_unlock_irqrestore(&engine->timeline.lock, flags); - - return active; } -static void execlists_reset(struct intel_engine_cs *engine, - struct i915_request *request) +static void execlists_reset(struct intel_engine_cs *engine, bool stalled) { struct intel_engine_execlists * const execlists = &engine->execlists; + struct i915_request *rq; unsigned long flags; u32 *regs; - GEM_TRACE("%s request global=%x, current=%d\n", - engine->name, request ? request->global_seqno : 0, - intel_engine_get_seqno(engine)); - spin_lock_irqsave(&engine->timeline.lock, flags); /* @@ -1940,12 +1899,18 @@ static void execlists_reset(struct intel_engine_cs *engine, execlists_cancel_port_requests(execlists); /* Push back any incomplete requests for replay after the reset. */ - __unwind_incomplete_requests(engine); + rq = __unwind_incomplete_requests(engine); /* Following the reset, we need to reload the CSB read/write pointers */ reset_csb_pointers(&engine->execlists); - spin_unlock_irqrestore(&engine->timeline.lock, flags); + GEM_TRACE("%s seqno=%d, current=%d, stalled? %s\n", + engine->name, + rq ? rq->global_seqno : 0, + intel_engine_get_seqno(engine), + yesno(stalled)); + if (!rq) + goto out_unlock; /* * If the request was innocent, we leave the request in the ELSP @@ -1958,8 +1923,9 @@ static void execlists_reset(struct intel_engine_cs *engine, * and have to at least restore the RING register in the context * image back to the expected values to skip over the guilty request. */ - if (!request || request->fence.error != -EIO) - return; + i915_reset_request(rq, stalled); + if (!stalled) + goto out_unlock; /* * We want a simple context + ring to execute the breadcrumb update. @@ -1969,25 +1935,22 @@ static void execlists_reset(struct intel_engine_cs *engine, * future request will be after userspace has had the opportunity * to recreate its own state. */ - regs = request->hw_context->lrc_reg_state; + regs = rq->hw_context->lrc_reg_state; if (engine->pinned_default_state) { memcpy(regs, /* skip restoring the vanilla PPHWSP */ engine->pinned_default_state + LRC_STATE_PN * PAGE_SIZE, engine->context_size - PAGE_SIZE); } - execlists_init_reg_state(regs, - request->gem_context, engine, request->ring); /* Move the RING_HEAD onto the breadcrumb, past the hanging batch */ - regs[CTX_RING_BUFFER_START + 1] = i915_ggtt_offset(request->ring->vma); + rq->ring->head = intel_ring_wrap(rq->ring, rq->postfix); + intel_ring_update_space(rq->ring); - request->ring->head = intel_ring_wrap(request->ring, request->postfix); - regs[CTX_RING_HEAD + 1] = request->ring->head; + execlists_init_reg_state(regs, rq->gem_context, engine, rq->ring); + __execlists_update_reg_state(engine, rq->hw_context); - intel_ring_update_space(request->ring); - - /* Reset WaIdleLiteRestore:bdw,skl as well */ - unwind_wa_tail(request); +out_unlock: + spin_unlock_irqrestore(&engine->timeline.lock, flags); } static void execlists_reset_finish(struct intel_engine_cs *engine) @@ -2000,6 +1963,7 @@ static void execlists_reset_finish(struct intel_engine_cs *engine) * to sleep before we restart and reload a context. * */ + GEM_BUG_ON(!reset_in_progress(execlists)); if (!RB_EMPTY_ROOT(&execlists->queue.rb_root)) execlists->tasklet.func(execlists->tasklet.data); @@ -2008,57 +1972,11 @@ static void execlists_reset_finish(struct intel_engine_cs *engine) atomic_read(&execlists->tasklet.count)); } -static int intel_logical_ring_emit_pdps(struct i915_request *rq) -{ - struct i915_hw_ppgtt *ppgtt = rq->gem_context->ppgtt; - struct intel_engine_cs *engine = rq->engine; - const int num_lri_cmds = GEN8_3LVL_PDPES * 2; - u32 *cs; - int i; - - cs = intel_ring_begin(rq, num_lri_cmds * 2 + 2); - if (IS_ERR(cs)) - return PTR_ERR(cs); - - *cs++ = MI_LOAD_REGISTER_IMM(num_lri_cmds); - for (i = GEN8_3LVL_PDPES - 1; i >= 0; i--) { - const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i); - - *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(engine, i)); - *cs++ = upper_32_bits(pd_daddr); - *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(engine, i)); - *cs++ = lower_32_bits(pd_daddr); - } - - *cs++ = MI_NOOP; - intel_ring_advance(rq, cs); - - return 0; -} - static int gen8_emit_bb_start(struct i915_request *rq, u64 offset, u32 len, const unsigned int flags) { u32 *cs; - int ret; - - /* Don't rely in hw updating PDPs, specially in lite-restore. - * Ideally, we should set Force PD Restore in ctx descriptor, - * but we can't. Force Restore would be a second option, but - * it is unsafe in case of lite-restore (because the ctx is - * not idle). PML4 is allocated during ppgtt init so this is - * not needed in 48-bit.*/ - if (rq->gem_context->ppgtt && - (intel_engine_flag(rq->engine) & rq->gem_context->ppgtt->pd_dirty_rings) && - !i915_vm_is_48bit(&rq->gem_context->ppgtt->vm) && - !intel_vgpu_active(rq->i915)) { - ret = intel_logical_ring_emit_pdps(rq); - if (ret) - return ret; - - rq->gem_context->ppgtt->pd_dirty_rings &= ~intel_engine_flag(rq->engine); - } cs = intel_ring_begin(rq, 6); if (IS_ERR(cs)) @@ -2091,6 +2009,7 @@ static int gen8_emit_bb_start(struct i915_request *rq, *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; *cs++ = MI_NOOP; + intel_ring_advance(rq, cs); return 0; @@ -2129,7 +2048,7 @@ static int gen8_emit_flush(struct i915_request *request, u32 mode) if (mode & EMIT_INVALIDATE) { cmd |= MI_INVALIDATE_TLB; - if (request->engine->id == VCS) + if (request->engine->class == VIDEO_DECODE_CLASS) cmd |= MI_INVALIDATE_BSD; } @@ -2175,7 +2094,7 @@ static int gen8_emit_flush_render(struct i915_request *request, * On GEN9: before VF_CACHE_INVALIDATE we need to emit a NULL * pipe control. */ - if (IS_GEN9(request->i915)) + if (IS_GEN(request->i915, 9)) vf_flush_wa = true; /* WaForGAMHang:kbl */ @@ -2217,51 +2136,68 @@ static int gen8_emit_flush_render(struct i915_request *request, * used as a workaround for not being allowed to do lite * restore with HEAD==TAIL (WaIdleLiteRestore). */ -static void gen8_emit_wa_tail(struct i915_request *request, u32 *cs) +static u32 *gen8_emit_wa_tail(struct i915_request *request, u32 *cs) { /* Ensure there's always at least one preemption point per-request. */ *cs++ = MI_ARB_CHECK; *cs++ = MI_NOOP; request->wa_tail = intel_ring_offset(request, cs); + + return cs; } -static void gen8_emit_breadcrumb(struct i915_request *request, u32 *cs) +static u32 *gen8_emit_fini_breadcrumb(struct i915_request *request, u32 *cs) { /* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */ BUILD_BUG_ON(I915_GEM_HWS_INDEX_ADDR & (1 << 5)); - cs = gen8_emit_ggtt_write(cs, request->global_seqno, + cs = gen8_emit_ggtt_write(cs, + request->fence.seqno, + request->timeline->hwsp_offset); + + cs = gen8_emit_ggtt_write(cs, + request->global_seqno, intel_hws_seqno_address(request->engine)); + *cs++ = MI_USER_INTERRUPT; *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; + request->tail = intel_ring_offset(request, cs); assert_ring_tail_valid(request->ring, request->tail); - gen8_emit_wa_tail(request, cs); + return gen8_emit_wa_tail(request, cs); } -static const int gen8_emit_breadcrumb_sz = 6 + WA_TAIL_DWORDS; -static void gen8_emit_breadcrumb_rcs(struct i915_request *request, u32 *cs) +static u32 *gen8_emit_fini_breadcrumb_rcs(struct i915_request *request, u32 *cs) { - /* We're using qword write, seqno should be aligned to 8 bytes. */ - BUILD_BUG_ON(I915_GEM_HWS_INDEX & 1); + cs = gen8_emit_ggtt_write_rcs(cs, + request->fence.seqno, + request->timeline->hwsp_offset, + PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | + PIPE_CONTROL_DEPTH_CACHE_FLUSH | + PIPE_CONTROL_DC_FLUSH_ENABLE | + PIPE_CONTROL_FLUSH_ENABLE | + PIPE_CONTROL_CS_STALL); + + cs = gen8_emit_ggtt_write_rcs(cs, + request->global_seqno, + intel_hws_seqno_address(request->engine), + PIPE_CONTROL_CS_STALL); - cs = gen8_emit_ggtt_write_rcs(cs, request->global_seqno, - intel_hws_seqno_address(request->engine)); *cs++ = MI_USER_INTERRUPT; *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; + request->tail = intel_ring_offset(request, cs); assert_ring_tail_valid(request->ring, request->tail); - gen8_emit_wa_tail(request, cs); + return gen8_emit_wa_tail(request, cs); } -static const int gen8_emit_breadcrumb_rcs_sz = 8 + WA_TAIL_DWORDS; static int gen8_init_rcs_context(struct i915_request *rq) { int ret; - ret = intel_ctx_workarounds_emit(rq); + ret = intel_engine_emit_ctx_wa(rq); if (ret) return ret; @@ -2314,7 +2250,7 @@ void intel_execlists_set_default_submission(struct intel_engine_cs *engine) { engine->submit_request = execlists_submit_request; engine->cancel_requests = execlists_cancel_requests; - engine->schedule = execlists_schedule; + engine->schedule = i915_schedule; engine->execlists.tasklet.func = execlists_submission_tasklet; engine->reset.prepare = execlists_reset_prepare; @@ -2347,8 +2283,8 @@ logical_ring_default_vfuncs(struct intel_engine_cs *engine) engine->request_alloc = execlists_request_alloc; engine->emit_flush = gen8_emit_flush; - engine->emit_breadcrumb = gen8_emit_breadcrumb; - engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_sz; + engine->emit_init_breadcrumb = gen8_emit_init_breadcrumb; + engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb; engine->set_default_submission = intel_execlists_set_default_submission; @@ -2387,10 +2323,14 @@ logical_ring_default_irqs(struct intel_engine_cs *engine) engine->irq_keep_mask = GT_CONTEXT_SWITCH_INTERRUPT << shift; } -static void +static int logical_ring_setup(struct intel_engine_cs *engine) { - intel_engine_setup_common(engine); + int err; + + err = intel_engine_setup_common(engine); + if (err) + return err; /* Intentionally left blank. */ engine->buffer = NULL; @@ -2400,12 +2340,8 @@ logical_ring_setup(struct intel_engine_cs *engine) logical_ring_default_vfuncs(engine); logical_ring_default_irqs(engine); -} -static bool csb_force_mmio(struct drm_i915_private *i915) -{ - /* Older GVT emulation depends upon intercepting CSB mmio */ - return intel_vgpu_active(i915) && !intel_vgpu_has_hwsp_emulation(i915); + return 0; } static int logical_ring_init(struct intel_engine_cs *engine) @@ -2418,6 +2354,8 @@ static int logical_ring_init(struct intel_engine_cs *engine) if (ret) return ret; + intel_engine_init_workarounds(engine); + if (HAS_LOGICAL_RING_ELSQ(i915)) { execlists->submit_reg = i915->regs + i915_mmio_reg_offset(RING_EXECLIST_SQ_CONTENTS(engine)); @@ -2437,24 +2375,12 @@ static int logical_ring_init(struct intel_engine_cs *engine) upper_32_bits(ce->lrc_desc); } - execlists->csb_read = - i915->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)); - if (csb_force_mmio(i915)) { - execlists->csb_status = (u32 __force *) - (i915->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_BUF_LO(engine, 0))); + execlists->csb_status = + &engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX]; - execlists->csb_write = (u32 __force *)execlists->csb_read; - execlists->csb_write_reset = - _MASKED_FIELD(GEN8_CSB_WRITE_PTR_MASK, - GEN8_CSB_ENTRIES - 1); - } else { - execlists->csb_status = - &engine->status_page.page_addr[I915_HWS_CSB_BUF0_INDEX]; + execlists->csb_write = + &engine->status_page.addr[intel_hws_csb_write_index(i915)]; - execlists->csb_write = - &engine->status_page.page_addr[intel_hws_csb_write_index(i915)]; - execlists->csb_write_reset = GEN8_CSB_ENTRIES - 1; - } reset_csb_pointers(execlists); return 0; @@ -2462,23 +2388,16 @@ static int logical_ring_init(struct intel_engine_cs *engine) int logical_render_ring_init(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = engine->i915; int ret; - logical_ring_setup(engine); - - if (HAS_L3_DPF(dev_priv)) - engine->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT; + ret = logical_ring_setup(engine); + if (ret) + return ret; /* Override some for render ring. */ - if (INTEL_GEN(dev_priv) >= 9) - engine->init_hw = gen9_init_render_ring; - else - engine->init_hw = gen8_init_render_ring; engine->init_context = gen8_init_rcs_context; engine->emit_flush = gen8_emit_flush_render; - engine->emit_breadcrumb = gen8_emit_breadcrumb_rcs; - engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_rcs_sz; + engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb_rcs; ret = logical_ring_init(engine); if (ret) @@ -2495,34 +2414,66 @@ int logical_render_ring_init(struct intel_engine_cs *engine) ret); } - intel_engine_init_workarounds(engine); + intel_engine_init_whitelist(engine); return 0; } int logical_xcs_ring_init(struct intel_engine_cs *engine) { - logical_ring_setup(engine); + int err; + + err = logical_ring_setup(engine); + if (err) + return err; return logical_ring_init(engine); } -static u32 -make_rpcs(struct drm_i915_private *dev_priv) +u32 gen8_make_rpcs(struct drm_i915_private *i915, struct intel_sseu *req_sseu) { - bool subslice_pg = INTEL_INFO(dev_priv)->sseu.has_subslice_pg; - u8 slices = hweight8(INTEL_INFO(dev_priv)->sseu.slice_mask); - u8 subslices = hweight8(INTEL_INFO(dev_priv)->sseu.subslice_mask[0]); + const struct sseu_dev_info *sseu = &RUNTIME_INFO(i915)->sseu; + bool subslice_pg = sseu->has_subslice_pg; + struct intel_sseu ctx_sseu; + u8 slices, subslices; u32 rpcs = 0; /* * No explicit RPCS request is needed to ensure full * slice/subslice/EU enablement prior to Gen9. */ - if (INTEL_GEN(dev_priv) < 9) + if (INTEL_GEN(i915) < 9) return 0; /* + * If i915/perf is active, we want a stable powergating configuration + * on the system. + * + * We could choose full enablement, but on ICL we know there are use + * cases which disable slices for functional, apart for performance + * reasons. So in this case we select a known stable subset. + */ + if (!i915->perf.oa.exclusive_stream) { + ctx_sseu = *req_sseu; + } else { + ctx_sseu = intel_device_default_sseu(i915); + + if (IS_GEN(i915, 11)) { + /* + * We only need subslice count so it doesn't matter + * which ones we select - just turn off low bits in the + * amount of half of all available subslices per slice. + */ + ctx_sseu.subslice_mask = + ~(~0 << (hweight8(ctx_sseu.subslice_mask) / 2)); + ctx_sseu.slice_mask = 0x1; + } + } + + slices = hweight8(ctx_sseu.slice_mask); + subslices = hweight8(ctx_sseu.subslice_mask); + + /* * Since the SScount bitfield in GEN8_R_PWR_CLK_STATE is only three bits * wide and Icelake has up to eight subslices, specfial programming is * needed in order to correctly enable all subslices. @@ -2547,7 +2498,9 @@ make_rpcs(struct drm_i915_private *dev_priv) * subslices are enabled, or a count between one and four on the first * slice. */ - if (IS_GEN11(dev_priv) && slices == 1 && subslices >= 4) { + if (IS_GEN(i915, 11) && + slices == 1 && + subslices > min_t(u8, 4, hweight8(sseu->subslice_mask[0]) / 2)) { GEM_BUG_ON(subslices & 1); subslice_pg = false; @@ -2560,10 +2513,10 @@ make_rpcs(struct drm_i915_private *dev_priv) * must make an explicit request through RPCS for full * enablement. */ - if (INTEL_INFO(dev_priv)->sseu.has_slice_pg) { + if (sseu->has_slice_pg) { u32 mask, val = slices; - if (INTEL_GEN(dev_priv) >= 11) { + if (INTEL_GEN(i915) >= 11) { mask = GEN11_RPCS_S_CNT_MASK; val <<= GEN11_RPCS_S_CNT_SHIFT; } else { @@ -2588,18 +2541,16 @@ make_rpcs(struct drm_i915_private *dev_priv) rpcs |= GEN8_RPCS_ENABLE | GEN8_RPCS_SS_CNT_ENABLE | val; } - if (INTEL_INFO(dev_priv)->sseu.has_eu_pg) { + if (sseu->has_eu_pg) { u32 val; - val = INTEL_INFO(dev_priv)->sseu.eu_per_subslice << - GEN8_RPCS_EU_MIN_SHIFT; + val = ctx_sseu.min_eus_per_subslice << GEN8_RPCS_EU_MIN_SHIFT; GEM_BUG_ON(val & ~GEN8_RPCS_EU_MIN_MASK); val &= GEN8_RPCS_EU_MIN_MASK; rpcs |= val; - val = INTEL_INFO(dev_priv)->sseu.eu_per_subslice << - GEN8_RPCS_EU_MAX_SHIFT; + val = ctx_sseu.max_eus_per_subslice << GEN8_RPCS_EU_MAX_SHIFT; GEM_BUG_ON(val & ~GEN8_RPCS_EU_MAX_MASK); val &= GEN8_RPCS_EU_MAX_MASK; @@ -2646,7 +2597,6 @@ static void execlists_init_reg_state(u32 *regs, struct intel_ring *ring) { struct drm_i915_private *dev_priv = engine->i915; - struct i915_hw_ppgtt *ppgtt = ctx->ppgtt ?: dev_priv->mm.aliasing_ppgtt; u32 base = engine->mmio_base; bool rcs = engine->class == RENDER_CLASS; @@ -2718,18 +2668,22 @@ static void execlists_init_reg_state(u32 *regs, CTX_REG(regs, CTX_PDP0_UDW, GEN8_RING_PDP_UDW(engine, 0), 0); CTX_REG(regs, CTX_PDP0_LDW, GEN8_RING_PDP_LDW(engine, 0), 0); - if (ppgtt && i915_vm_is_48bit(&ppgtt->vm)) { + if (i915_vm_is_48bit(&ctx->ppgtt->vm)) { /* 64b PPGTT (48bit canonical) * PDP0_DESCRIPTOR contains the base address to PML4 and * other PDP Descriptors are ignored. */ - ASSIGN_CTX_PML4(ppgtt, regs); + ASSIGN_CTX_PML4(ctx->ppgtt, regs); + } else { + ASSIGN_CTX_PDP(ctx->ppgtt, regs, 3); + ASSIGN_CTX_PDP(ctx->ppgtt, regs, 2); + ASSIGN_CTX_PDP(ctx->ppgtt, regs, 1); + ASSIGN_CTX_PDP(ctx->ppgtt, regs, 0); } if (rcs) { regs[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1); - CTX_REG(regs, CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE, - make_rpcs(dev_priv)); + CTX_REG(regs, CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE, 0); i915_oa_init_reg_state(engine, ctx, regs); } @@ -2806,7 +2760,7 @@ static int execlists_context_deferred_alloc(struct i915_gem_context *ctx, { struct drm_i915_gem_object *ctx_obj; struct i915_vma *vma; - uint32_t context_size; + u32 context_size; struct intel_ring *ring; struct i915_timeline *timeline; int ret; @@ -2832,7 +2786,7 @@ static int execlists_context_deferred_alloc(struct i915_gem_context *ctx, goto error_deref_obj; } - timeline = i915_timeline_create(ctx->i915, ctx->name); + timeline = i915_timeline_create(ctx->i915, ctx->name, NULL); if (IS_ERR(timeline)) { ret = PTR_ERR(timeline); goto error_deref_obj; @@ -2890,14 +2844,70 @@ void intel_lr_context_resume(struct drm_i915_private *i915) intel_ring_reset(ce->ring, 0); - if (ce->pin_count) { /* otherwise done in context_pin */ - u32 *regs = ce->lrc_reg_state; + if (ce->pin_count) /* otherwise done in context_pin */ + __execlists_update_reg_state(engine, ce); + } + } +} - regs[CTX_RING_HEAD + 1] = ce->ring->head; - regs[CTX_RING_TAIL + 1] = ce->ring->tail; - } +void intel_execlists_show_requests(struct intel_engine_cs *engine, + struct drm_printer *m, + void (*show_request)(struct drm_printer *m, + struct i915_request *rq, + const char *prefix), + unsigned int max) +{ + const struct intel_engine_execlists *execlists = &engine->execlists; + struct i915_request *rq, *last; + unsigned long flags; + unsigned int count; + struct rb_node *rb; + + spin_lock_irqsave(&engine->timeline.lock, flags); + + last = NULL; + count = 0; + list_for_each_entry(rq, &engine->timeline.requests, link) { + if (count++ < max - 1) + show_request(m, rq, "\t\tE "); + else + last = rq; + } + if (last) { + if (count > max) { + drm_printf(m, + "\t\t...skipping %d executing requests...\n", + count - max); + } + show_request(m, last, "\t\tE "); + } + + last = NULL; + count = 0; + if (execlists->queue_priority_hint != INT_MIN) + drm_printf(m, "\t\tQueue priority hint: %d\n", + execlists->queue_priority_hint); + for (rb = rb_first_cached(&execlists->queue); rb; rb = rb_next(rb)) { + struct i915_priolist *p = rb_entry(rb, typeof(*p), node); + int i; + + priolist_for_each_request(rq, p, i) { + if (count++ < max - 1) + show_request(m, rq, "\t\tQ "); + else + last = rq; } } + if (last) { + if (count > max) { + drm_printf(m, + "\t\t...skipping %d queued requests...\n", + count - max); + } + show_request(m, last, "\t\tQ "); + } + + spin_unlock_irqrestore(&engine->timeline.lock, flags); } #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) diff --git a/drivers/gpu/drm/i915/intel_lrc.h b/drivers/gpu/drm/i915/intel_lrc.h index f5a5502ecf70..f1aec8a6986f 100644 --- a/drivers/gpu/drm/i915/intel_lrc.h +++ b/drivers/gpu/drm/i915/intel_lrc.h @@ -97,11 +97,21 @@ int logical_xcs_ring_init(struct intel_engine_cs *engine); */ #define LRC_HEADER_PAGES LRC_PPHWSP_PN +struct drm_printer; + struct drm_i915_private; struct i915_gem_context; void intel_lr_context_resume(struct drm_i915_private *dev_priv); - void intel_execlists_set_default_submission(struct intel_engine_cs *engine); +void intel_execlists_show_requests(struct intel_engine_cs *engine, + struct drm_printer *m, + void (*show_request)(struct drm_printer *m, + struct i915_request *rq, + const char *prefix), + unsigned int max); + +u32 gen8_make_rpcs(struct drm_i915_private *i915, struct intel_sseu *ctx_sseu); + #endif /* _INTEL_LRC_H_ */ diff --git a/drivers/gpu/drm/i915/intel_lspcon.c b/drivers/gpu/drm/i915/intel_lspcon.c index 3e085c5f2b81..322bdddda164 100644 --- a/drivers/gpu/drm/i915/intel_lspcon.c +++ b/drivers/gpu/drm/i915/intel_lspcon.c @@ -27,6 +27,22 @@ #include <drm/drm_dp_dual_mode_helper.h> #include "intel_drv.h" +/* LSPCON OUI Vendor ID(signatures) */ +#define LSPCON_VENDOR_PARADE_OUI 0x001CF8 +#define LSPCON_VENDOR_MCA_OUI 0x0060AD + +/* AUX addresses to write MCA AVI IF */ +#define LSPCON_MCA_AVI_IF_WRITE_OFFSET 0x5C0 +#define LSPCON_MCA_AVI_IF_CTRL 0x5DF +#define LSPCON_MCA_AVI_IF_KICKOFF (1 << 0) +#define LSPCON_MCA_AVI_IF_HANDLED (1 << 1) + +/* AUX addresses to write Parade AVI IF */ +#define LSPCON_PARADE_AVI_IF_WRITE_OFFSET 0x516 +#define LSPCON_PARADE_AVI_IF_CTRL 0x51E +#define LSPCON_PARADE_AVI_IF_KICKOFF (1 << 7) +#define LSPCON_PARADE_AVI_IF_DATA_SIZE 32 + static struct intel_dp *lspcon_to_intel_dp(struct intel_lspcon *lspcon) { struct intel_digital_port *dig_port = @@ -50,6 +66,40 @@ static const char *lspcon_mode_name(enum drm_lspcon_mode mode) } } +static bool lspcon_detect_vendor(struct intel_lspcon *lspcon) +{ + struct intel_dp *dp = lspcon_to_intel_dp(lspcon); + struct drm_dp_dpcd_ident *ident; + u32 vendor_oui; + + if (drm_dp_read_desc(&dp->aux, &dp->desc, drm_dp_is_branch(dp->dpcd))) { + DRM_ERROR("Can't read description\n"); + return false; + } + + ident = &dp->desc.ident; + vendor_oui = (ident->oui[0] << 16) | (ident->oui[1] << 8) | + ident->oui[2]; + + switch (vendor_oui) { + case LSPCON_VENDOR_MCA_OUI: + lspcon->vendor = LSPCON_VENDOR_MCA; + DRM_DEBUG_KMS("Vendor: Mega Chips\n"); + break; + + case LSPCON_VENDOR_PARADE_OUI: + lspcon->vendor = LSPCON_VENDOR_PARADE; + DRM_DEBUG_KMS("Vendor: Parade Tech\n"); + break; + + default: + DRM_ERROR("Invalid/Unknown vendor OUI\n"); + return false; + } + + return true; +} + static enum drm_lspcon_mode lspcon_get_current_mode(struct intel_lspcon *lspcon) { enum drm_lspcon_mode current_mode; @@ -130,6 +180,21 @@ static bool lspcon_wake_native_aux_ch(struct intel_lspcon *lspcon) return true; } +void lspcon_ycbcr420_config(struct drm_connector *connector, + struct intel_crtc_state *crtc_state) +{ + const struct drm_display_info *info = &connector->display_info; + const struct drm_display_mode *adjusted_mode = + &crtc_state->base.adjusted_mode; + + if (drm_mode_is_420_only(info, adjusted_mode) && + connector->ycbcr_420_allowed) { + crtc_state->port_clock /= 2; + crtc_state->output_format = INTEL_OUTPUT_FORMAT_YCBCR444; + crtc_state->lspcon_downsampling = true; + } +} + static bool lspcon_probe(struct intel_lspcon *lspcon) { int retry; @@ -159,7 +224,18 @@ static bool lspcon_probe(struct intel_lspcon *lspcon) /* Yay ... got a LSPCON device */ DRM_DEBUG_KMS("LSPCON detected\n"); lspcon->mode = lspcon_wait_mode(lspcon, expected_mode); - lspcon->active = true; + + /* + * In the SW state machine, lets Put LSPCON in PCON mode only. + * In this way, it will work with both HDMI 1.4 sinks as well as HDMI + * 2.0 sinks. + */ + if (lspcon->mode != DRM_LSPCON_MODE_PCON) { + if (lspcon_change_mode(lspcon, DRM_LSPCON_MODE_PCON) < 0) { + DRM_ERROR("LSPCON mode change to PCON failed\n"); + return false; + } + } return true; } @@ -185,6 +261,255 @@ static void lspcon_resume_in_pcon_wa(struct intel_lspcon *lspcon) DRM_DEBUG_KMS("LSPCON DP descriptor mismatch after resume\n"); } +static bool lspcon_parade_fw_ready(struct drm_dp_aux *aux) +{ + u8 avi_if_ctrl; + u8 retry; + ssize_t ret; + + /* Check if LSPCON FW is ready for data */ + for (retry = 0; retry < 5; retry++) { + if (retry) + usleep_range(200, 300); + + ret = drm_dp_dpcd_read(aux, LSPCON_PARADE_AVI_IF_CTRL, + &avi_if_ctrl, 1); + if (ret < 0) { + DRM_ERROR("Failed to read AVI IF control\n"); + return false; + } + + if ((avi_if_ctrl & LSPCON_PARADE_AVI_IF_KICKOFF) == 0) + return true; + } + + DRM_ERROR("Parade FW not ready to accept AVI IF\n"); + return false; +} + +static bool _lspcon_parade_write_infoframe_blocks(struct drm_dp_aux *aux, + u8 *avi_buf) +{ + u8 avi_if_ctrl; + u8 block_count = 0; + u8 *data; + u16 reg; + ssize_t ret; + + while (block_count < 4) { + if (!lspcon_parade_fw_ready(aux)) { + DRM_DEBUG_KMS("LSPCON FW not ready, block %d\n", + block_count); + return false; + } + + reg = LSPCON_PARADE_AVI_IF_WRITE_OFFSET; + data = avi_buf + block_count * 8; + ret = drm_dp_dpcd_write(aux, reg, data, 8); + if (ret < 0) { + DRM_ERROR("Failed to write AVI IF block %d\n", + block_count); + return false; + } + + /* + * Once a block of data is written, we have to inform the FW + * about this by writing into avi infoframe control register: + * - set the kickoff bit[7] to 1 + * - write the block no. to bits[1:0] + */ + reg = LSPCON_PARADE_AVI_IF_CTRL; + avi_if_ctrl = LSPCON_PARADE_AVI_IF_KICKOFF | block_count; + ret = drm_dp_dpcd_write(aux, reg, &avi_if_ctrl, 1); + if (ret < 0) { + DRM_ERROR("Failed to update (0x%x), block %d\n", + reg, block_count); + return false; + } + + block_count++; + } + + DRM_DEBUG_KMS("Wrote AVI IF blocks successfully\n"); + return true; +} + +static bool _lspcon_write_avi_infoframe_parade(struct drm_dp_aux *aux, + const u8 *frame, + ssize_t len) +{ + u8 avi_if[LSPCON_PARADE_AVI_IF_DATA_SIZE] = {1, }; + + /* + * Parade's frames contains 32 bytes of data, divided + * into 4 frames: + * Token byte (first byte of first frame, must be non-zero) + * HB0 to HB2 from AVI IF (3 bytes header) + * PB0 to PB27 from AVI IF (28 bytes data) + * So it should look like this + * first block: | <token> <HB0-HB2> <DB0-DB3> | + * next 3 blocks: |<DB4-DB11>|<DB12-DB19>|<DB20-DB28>| + */ + + if (len > LSPCON_PARADE_AVI_IF_DATA_SIZE - 1) { + DRM_ERROR("Invalid length of infoframes\n"); + return false; + } + + memcpy(&avi_if[1], frame, len); + + if (!_lspcon_parade_write_infoframe_blocks(aux, avi_if)) { + DRM_DEBUG_KMS("Failed to write infoframe blocks\n"); + return false; + } + + return true; +} + +static bool _lspcon_write_avi_infoframe_mca(struct drm_dp_aux *aux, + const u8 *buffer, ssize_t len) +{ + int ret; + u32 val = 0; + u32 retry; + u16 reg; + const u8 *data = buffer; + + reg = LSPCON_MCA_AVI_IF_WRITE_OFFSET; + while (val < len) { + /* DPCD write for AVI IF can fail on a slow FW day, so retry */ + for (retry = 0; retry < 5; retry++) { + ret = drm_dp_dpcd_write(aux, reg, (void *)data, 1); + if (ret == 1) { + break; + } else if (retry < 4) { + mdelay(50); + continue; + } else { + DRM_ERROR("DPCD write failed at:0x%x\n", reg); + return false; + } + } + val++; reg++; data++; + } + + val = 0; + reg = LSPCON_MCA_AVI_IF_CTRL; + ret = drm_dp_dpcd_read(aux, reg, &val, 1); + if (ret < 0) { + DRM_ERROR("DPCD read failed, address 0x%x\n", reg); + return false; + } + + /* Indicate LSPCON chip about infoframe, clear bit 1 and set bit 0 */ + val &= ~LSPCON_MCA_AVI_IF_HANDLED; + val |= LSPCON_MCA_AVI_IF_KICKOFF; + + ret = drm_dp_dpcd_write(aux, reg, &val, 1); + if (ret < 0) { + DRM_ERROR("DPCD read failed, address 0x%x\n", reg); + return false; + } + + val = 0; + ret = drm_dp_dpcd_read(aux, reg, &val, 1); + if (ret < 0) { + DRM_ERROR("DPCD read failed, address 0x%x\n", reg); + return false; + } + + if (val == LSPCON_MCA_AVI_IF_HANDLED) + DRM_DEBUG_KMS("AVI IF handled by FW\n"); + + return true; +} + +void lspcon_write_infoframe(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + unsigned int type, + const void *frame, ssize_t len) +{ + bool ret; + struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); + struct intel_lspcon *lspcon = enc_to_intel_lspcon(&encoder->base); + + /* LSPCON only needs AVI IF */ + if (type != HDMI_INFOFRAME_TYPE_AVI) + return; + + if (lspcon->vendor == LSPCON_VENDOR_MCA) + ret = _lspcon_write_avi_infoframe_mca(&intel_dp->aux, + frame, len); + else + ret = _lspcon_write_avi_infoframe_parade(&intel_dp->aux, + frame, len); + + if (!ret) { + DRM_ERROR("Failed to write AVI infoframes\n"); + return; + } + + DRM_DEBUG_DRIVER("AVI infoframes updated successfully\n"); +} + +void lspcon_set_infoframes(struct intel_encoder *encoder, + bool enable, + const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) +{ + ssize_t ret; + union hdmi_infoframe frame; + u8 buf[VIDEO_DIP_DATA_SIZE]; + struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base); + struct intel_lspcon *lspcon = &dig_port->lspcon; + const struct drm_display_mode *adjusted_mode = + &crtc_state->base.adjusted_mode; + + if (!lspcon->active) { + DRM_ERROR("Writing infoframes while LSPCON disabled ?\n"); + return; + } + + ret = drm_hdmi_avi_infoframe_from_display_mode(&frame.avi, + conn_state->connector, + adjusted_mode); + if (ret < 0) { + DRM_ERROR("couldn't fill AVI infoframe\n"); + return; + } + + if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444) { + if (crtc_state->lspcon_downsampling) + frame.avi.colorspace = HDMI_COLORSPACE_YUV420; + else + frame.avi.colorspace = HDMI_COLORSPACE_YUV444; + } else { + frame.avi.colorspace = HDMI_COLORSPACE_RGB; + } + + drm_hdmi_avi_infoframe_quant_range(&frame.avi, + conn_state->connector, + adjusted_mode, + crtc_state->limited_color_range ? + HDMI_QUANTIZATION_RANGE_LIMITED : + HDMI_QUANTIZATION_RANGE_FULL); + + ret = hdmi_infoframe_pack(&frame, buf, sizeof(buf)); + if (ret < 0) { + DRM_ERROR("Failed to pack AVI IF\n"); + return; + } + + dig_port->write_infoframe(encoder, crtc_state, HDMI_INFOFRAME_TYPE_AVI, + buf, ret); +} + +bool lspcon_infoframe_enabled(struct intel_encoder *encoder, + const struct intel_crtc_state *pipe_config) +{ + return enc_to_intel_lspcon(&encoder->base)->active; +} + void lspcon_resume(struct intel_lspcon *lspcon) { enum drm_lspcon_mode expected_mode; @@ -216,6 +541,7 @@ bool lspcon_init(struct intel_digital_port *intel_dig_port) struct intel_lspcon *lspcon = &intel_dig_port->lspcon; struct drm_device *dev = intel_dig_port->base.base.dev; struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_connector *connector = &dp->attached_connector->base; if (!HAS_LSPCON(dev_priv)) { DRM_ERROR("LSPCON is not supported on this platform\n"); @@ -230,25 +556,18 @@ bool lspcon_init(struct intel_digital_port *intel_dig_port) return false; } - /* - * In the SW state machine, lets Put LSPCON in PCON mode only. - * In this way, it will work with both HDMI 1.4 sinks as well as HDMI - * 2.0 sinks. - */ - if (lspcon->active && lspcon->mode != DRM_LSPCON_MODE_PCON) { - if (lspcon_change_mode(lspcon, DRM_LSPCON_MODE_PCON) < 0) { - DRM_ERROR("LSPCON mode change to PCON failed\n"); - return false; - } - } - if (!intel_dp_read_dpcd(dp)) { DRM_ERROR("LSPCON DPCD read failed\n"); return false; } - drm_dp_read_desc(&dp->aux, &dp->desc, drm_dp_is_branch(dp->dpcd)); + if (!lspcon_detect_vendor(lspcon)) { + DRM_ERROR("LSPCON vendor detection failed\n"); + return false; + } + connector->ycbcr_420_allowed = true; + lspcon->active = true; DRM_DEBUG_KMS("Success: LSPCON init\n"); return true; } diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c index f9f3b0885ba5..b4aa49768e90 100644 --- a/drivers/gpu/drm/i915/intel_lvds.c +++ b/drivers/gpu/drm/i915/intel_lvds.c @@ -32,7 +32,6 @@ #include <linux/i2c.h> #include <linux/slab.h> #include <linux/vga_switcheroo.h> -#include <drm/drmP.h> #include <drm/drm_atomic_helper.h> #include <drm/drm_crtc.h> #include <drm/drm_edid.h> @@ -42,10 +41,6 @@ #include <linux/acpi.h> /* Private structure for the integrated LVDS support */ -struct intel_lvds_connector { - struct intel_connector base; -}; - struct intel_lvds_pps { /* 100us units */ int t1_t2; @@ -70,7 +65,7 @@ struct intel_lvds_encoder { struct intel_lvds_pps init_pps; u32 init_lvds_val; - struct intel_lvds_connector *attached_connector; + struct intel_connector *attached_connector; }; static struct intel_lvds_encoder *to_lvds_encoder(struct drm_encoder *encoder) @@ -78,11 +73,6 @@ static struct intel_lvds_encoder *to_lvds_encoder(struct drm_encoder *encoder) return container_of(encoder, struct intel_lvds_encoder, base.base); } -static struct intel_lvds_connector *to_lvds_connector(struct drm_connector *connector) -{ - return container_of(connector, struct intel_lvds_connector, base.base); -} - bool intel_lvds_port_enabled(struct drm_i915_private *dev_priv, i915_reg_t lvds_reg, enum pipe *pipe) { @@ -104,15 +94,17 @@ static bool intel_lvds_get_hw_state(struct intel_encoder *encoder, { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base); + intel_wakeref_t wakeref; bool ret; - if (!intel_display_power_get_if_enabled(dev_priv, - encoder->power_domain)) + wakeref = intel_display_power_get_if_enabled(dev_priv, + encoder->power_domain); + if (!wakeref) return false; ret = intel_lvds_port_enabled(dev_priv, lvds_encoder->reg, pipe); - intel_display_power_put(dev_priv, encoder->power_domain); + intel_display_power_put(dev_priv, encoder->power_domain, wakeref); return ret; } @@ -288,7 +280,7 @@ static void intel_pre_enable_lvds(struct intel_encoder *encoder, * special lvds dither control bit on pch-split platforms, dithering is * only controlled through the PIPECONF reg. */ - if (IS_GEN4(dev_priv)) { + if (IS_GEN(dev_priv, 4)) { /* * Bspec wording suggests that LVDS port dithering only exists * for 18bpp panels. @@ -388,15 +380,15 @@ intel_lvds_mode_valid(struct drm_connector *connector, return MODE_OK; } -static bool intel_lvds_compute_config(struct intel_encoder *intel_encoder, - struct intel_crtc_state *pipe_config, - struct drm_connector_state *conn_state) +static int intel_lvds_compute_config(struct intel_encoder *intel_encoder, + struct intel_crtc_state *pipe_config, + struct drm_connector_state *conn_state) { struct drm_i915_private *dev_priv = to_i915(intel_encoder->base.dev); struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&intel_encoder->base); struct intel_connector *intel_connector = - &lvds_encoder->attached_connector->base; + lvds_encoder->attached_connector; struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; struct intel_crtc *intel_crtc = to_intel_crtc(pipe_config->base.crtc); unsigned int lvds_bpp; @@ -404,7 +396,7 @@ static bool intel_lvds_compute_config(struct intel_encoder *intel_encoder, /* Should never happen!! */ if (INTEL_GEN(dev_priv) < 4 && intel_crtc->pipe == 0) { DRM_ERROR("Can't support LVDS on pipe A\n"); - return false; + return -EINVAL; } if (lvds_encoder->a3_power == LVDS_A3_POWER_UP) @@ -418,6 +410,8 @@ static bool intel_lvds_compute_config(struct intel_encoder *intel_encoder, pipe_config->pipe_bpp = lvds_bpp; } + pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; + /* * We have timings from the BIOS for the panel, put them in * to the adjusted mode. The CRTC will be set up for this mode, @@ -428,7 +422,7 @@ static bool intel_lvds_compute_config(struct intel_encoder *intel_encoder, adjusted_mode); if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) - return false; + return -EINVAL; if (HAS_PCH_SPLIT(dev_priv)) { pipe_config->has_pch_encoder = true; @@ -447,7 +441,7 @@ static bool intel_lvds_compute_config(struct intel_encoder *intel_encoder, * user's requested refresh rate. */ - return true; + return 0; } static enum drm_connector_status @@ -461,15 +455,15 @@ intel_lvds_detect(struct drm_connector *connector, bool force) */ static int intel_lvds_get_modes(struct drm_connector *connector) { - struct intel_lvds_connector *lvds_connector = to_lvds_connector(connector); + struct intel_connector *intel_connector = to_intel_connector(connector); struct drm_device *dev = connector->dev; struct drm_display_mode *mode; /* use cached edid if we have one */ - if (!IS_ERR_OR_NULL(lvds_connector->base.edid)) - return drm_add_edid_modes(connector, lvds_connector->base.edid); + if (!IS_ERR_OR_NULL(intel_connector->edid)) + return drm_add_edid_modes(connector, intel_connector->edid); - mode = drm_mode_duplicate(dev, lvds_connector->base.panel.fixed_mode); + mode = drm_mode_duplicate(dev, intel_connector->panel.fixed_mode); if (mode == NULL) return 0; @@ -477,27 +471,6 @@ static int intel_lvds_get_modes(struct drm_connector *connector) return 1; } -/** - * intel_lvds_destroy - unregister and free LVDS structures - * @connector: connector to free - * - * Unregister the DDC bus for this connector then free the driver private - * structure. - */ -static void intel_lvds_destroy(struct drm_connector *connector) -{ - struct intel_lvds_connector *lvds_connector = - to_lvds_connector(connector); - - if (!IS_ERR_OR_NULL(lvds_connector->base.edid)) - kfree(lvds_connector->base.edid); - - intel_panel_fini(&lvds_connector->base.panel); - - drm_connector_cleanup(connector); - kfree(connector); -} - static const struct drm_connector_helper_funcs intel_lvds_connector_helper_funcs = { .get_modes = intel_lvds_get_modes, .mode_valid = intel_lvds_mode_valid, @@ -511,7 +484,7 @@ static const struct drm_connector_funcs intel_lvds_connector_funcs = { .atomic_set_property = intel_digital_connector_atomic_set_property, .late_register = intel_connector_register, .early_unregister = intel_connector_unregister, - .destroy = intel_lvds_destroy, + .destroy = intel_connector_destroy, .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, .atomic_duplicate_state = intel_digital_connector_duplicate_state, }; @@ -802,8 +775,7 @@ static bool compute_is_dual_link_lvds(struct intel_lvds_encoder *lvds_encoder) return i915_modparams.lvds_channel_mode == 2; /* single channel LVDS is limited to 112 MHz */ - if (lvds_encoder->attached_connector->base.panel.fixed_mode->clock - > 112999) + if (lvds_encoder->attached_connector->panel.fixed_mode->clock > 112999) return true; if (dmi_check_system(intel_dual_link_lvds)) @@ -826,26 +798,6 @@ static bool compute_is_dual_link_lvds(struct intel_lvds_encoder *lvds_encoder) return (val & LVDS_CLKB_POWER_MASK) == LVDS_CLKB_POWER_UP; } -static bool intel_lvds_supported(struct drm_i915_private *dev_priv) -{ - /* - * With the introduction of the PCH we gained a dedicated - * LVDS presence pin, use it. - */ - if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) - return true; - - /* - * Otherwise LVDS was only attached to mobile products, - * except for the inglorious 830gm - */ - if (INTEL_GEN(dev_priv) <= 4 && - IS_MOBILE(dev_priv) && !IS_I830(dev_priv)) - return true; - - return false; -} - /** * intel_lvds_init - setup LVDS connectors on this device * @dev_priv: i915 device @@ -858,7 +810,6 @@ void intel_lvds_init(struct drm_i915_private *dev_priv) struct drm_device *dev = &dev_priv->drm; struct intel_lvds_encoder *lvds_encoder; struct intel_encoder *intel_encoder; - struct intel_lvds_connector *lvds_connector; struct intel_connector *intel_connector; struct drm_connector *connector; struct drm_encoder *encoder; @@ -871,9 +822,6 @@ void intel_lvds_init(struct drm_i915_private *dev_priv) u8 pin; u32 allowed_scalers; - if (!intel_lvds_supported(dev_priv)) - return; - /* Skip init on machines we know falsely report LVDS */ if (dmi_check_system(intel_no_lvds)) { WARN(!dev_priv->vbt.int_lvds_support, @@ -911,23 +859,16 @@ void intel_lvds_init(struct drm_i915_private *dev_priv) if (!lvds_encoder) return; - lvds_connector = kzalloc(sizeof(*lvds_connector), GFP_KERNEL); - if (!lvds_connector) { - kfree(lvds_encoder); - return; - } - - if (intel_connector_init(&lvds_connector->base) < 0) { - kfree(lvds_connector); + intel_connector = intel_connector_alloc(); + if (!intel_connector) { kfree(lvds_encoder); return; } - lvds_encoder->attached_connector = lvds_connector; + lvds_encoder->attached_connector = intel_connector; intel_encoder = &lvds_encoder->base; encoder = &intel_encoder->base; - intel_connector = &lvds_connector->base; connector = &intel_connector->base; drm_connector_init(dev, &intel_connector->base, &intel_lvds_connector_funcs, DRM_MODE_CONNECTOR_LVDS); @@ -946,6 +887,7 @@ void intel_lvds_init(struct drm_i915_private *dev_priv) } intel_encoder->get_hw_state = intel_lvds_get_hw_state; intel_encoder->get_config = intel_lvds_get_config; + intel_encoder->update_pipe = intel_panel_update_backlight; intel_connector->get_hw_state = intel_connector_get_hw_state; intel_connector_attach_encoder(intel_connector, intel_encoder); @@ -956,7 +898,7 @@ void intel_lvds_init(struct drm_i915_private *dev_priv) intel_encoder->cloneable = 0; if (HAS_PCH_SPLIT(dev_priv)) intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2); - else if (IS_GEN4(dev_priv)) + else if (IS_GEN(dev_priv, 4)) intel_encoder->crtc_mask = (1 << 0) | (1 << 1); else intel_encoder->crtc_mask = (1 << 1); @@ -1008,7 +950,7 @@ void intel_lvds_init(struct drm_i915_private *dev_priv) } else { edid = ERR_PTR(-ENOENT); } - lvds_connector->base.edid = edid; + intel_connector->edid = edid; list_for_each_entry(scan, &connector->probed_modes, head) { if (scan->type & DRM_MODE_TYPE_PREFERRED) { @@ -1072,6 +1014,6 @@ failed: drm_connector_cleanup(connector); drm_encoder_cleanup(encoder); kfree(lvds_encoder); - kfree(lvds_connector); + intel_connector_free(intel_connector); return; } diff --git a/drivers/gpu/drm/i915/intel_mocs.c b/drivers/gpu/drm/i915/intel_mocs.c index 77e9871a8c9a..331e7a678fb7 100644 --- a/drivers/gpu/drm/i915/intel_mocs.c +++ b/drivers/gpu/drm/i915/intel_mocs.c @@ -28,48 +28,60 @@ struct drm_i915_mocs_entry { u32 control_value; u16 l3cc_value; + u16 used; }; struct drm_i915_mocs_table { - u32 size; + unsigned int size; + unsigned int n_entries; const struct drm_i915_mocs_entry *table; }; /* Defines for the tables (XXX_MOCS_0 - XXX_MOCS_63) */ -#define LE_CACHEABILITY(value) ((value) << 0) -#define LE_TGT_CACHE(value) ((value) << 2) +#define _LE_CACHEABILITY(value) ((value) << 0) +#define _LE_TGT_CACHE(value) ((value) << 2) #define LE_LRUM(value) ((value) << 4) #define LE_AOM(value) ((value) << 6) #define LE_RSC(value) ((value) << 7) #define LE_SCC(value) ((value) << 8) #define LE_PFM(value) ((value) << 11) #define LE_SCF(value) ((value) << 14) +#define LE_COS(value) ((value) << 15) +#define LE_SSE(value) ((value) << 17) /* Defines for the tables (LNCFMOCS0 - LNCFMOCS31) - two entries per word */ #define L3_ESC(value) ((value) << 0) #define L3_SCC(value) ((value) << 1) -#define L3_CACHEABILITY(value) ((value) << 4) +#define _L3_CACHEABILITY(value) ((value) << 4) /* Helper defines */ #define GEN9_NUM_MOCS_ENTRIES 62 /* 62 out of 64 - 63 & 64 are reserved. */ +#define GEN11_NUM_MOCS_ENTRIES 64 /* 63-64 are reserved, but configured. */ /* (e)LLC caching options */ -#define LE_PAGETABLE 0 -#define LE_UC 1 -#define LE_WT 2 -#define LE_WB 3 - -/* L3 caching options */ -#define L3_DIRECT 0 -#define L3_UC 1 -#define L3_RESERVED 2 -#define L3_WB 3 +#define LE_0_PAGETABLE _LE_CACHEABILITY(0) +#define LE_1_UC _LE_CACHEABILITY(1) +#define LE_2_WT _LE_CACHEABILITY(2) +#define LE_3_WB _LE_CACHEABILITY(3) /* Target cache */ -#define LE_TC_PAGETABLE 0 -#define LE_TC_LLC 1 -#define LE_TC_LLC_ELLC 2 -#define LE_TC_LLC_ELLC_ALT 3 +#define LE_TC_0_PAGETABLE _LE_TGT_CACHE(0) +#define LE_TC_1_LLC _LE_TGT_CACHE(1) +#define LE_TC_2_LLC_ELLC _LE_TGT_CACHE(2) +#define LE_TC_3_LLC_ELLC_ALT _LE_TGT_CACHE(3) + +/* L3 caching options */ +#define L3_0_DIRECT _L3_CACHEABILITY(0) +#define L3_1_UC _L3_CACHEABILITY(1) +#define L3_2_RESERVED _L3_CACHEABILITY(2) +#define L3_3_WB _L3_CACHEABILITY(3) + +#define MOCS_ENTRY(__idx, __control_value, __l3cc_value) \ + [__idx] = { \ + .control_value = __control_value, \ + .l3cc_value = __l3cc_value, \ + .used = 1, \ + } /* * MOCS tables @@ -80,85 +92,147 @@ struct drm_i915_mocs_table { * LNCFCMOCS0 - LNCFCMOCS32 registers. * * These tables are intended to be kept reasonably consistent across - * platforms. However some of the fields are not applicable to all of - * them. + * HW platforms, and for ICL+, be identical across OSes. To achieve + * that, for Icelake and above, list of entries is published as part + * of bspec. * * Entries not part of the following tables are undefined as far as * userspace is concerned and shouldn't be relied upon. For the time - * being they will be implicitly initialized to the strictest caching - * configuration (uncached) to guarantee forwards compatibility with - * userspace programs written against more recent kernels providing - * additional MOCS entries. + * being they will be initialized to PTE. * - * NOTE: These tables MUST start with being uncached and the length - * MUST be less than 63 as the last two registers are reserved - * by the hardware. These tables are part of the kernel ABI and - * may only be updated incrementally by adding entries at the - * end. + * The last two entries are reserved by the hardware. For ICL+ they + * should be initialized according to bspec and never used, for older + * platforms they should never be written to. + * + * NOTE: These tables are part of bspec and defined as part of hardware + * interface for ICL+. For older platforms, they are part of kernel + * ABI. It is expected that, for specific hardware platform, existing + * entries will remain constant and the table will only be updated by + * adding new entries, filling unused positions. */ +#define GEN9_MOCS_ENTRIES \ + MOCS_ENTRY(I915_MOCS_UNCACHED, \ + LE_1_UC | LE_TC_2_LLC_ELLC, \ + L3_1_UC), \ + MOCS_ENTRY(I915_MOCS_PTE, \ + LE_0_PAGETABLE | LE_TC_2_LLC_ELLC | LE_LRUM(3), \ + L3_3_WB) + static const struct drm_i915_mocs_entry skylake_mocs_table[] = { - [I915_MOCS_UNCACHED] = { - /* 0x00000009 */ - .control_value = LE_CACHEABILITY(LE_UC) | - LE_TGT_CACHE(LE_TC_LLC_ELLC) | - LE_LRUM(0) | LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | - LE_PFM(0) | LE_SCF(0), - - /* 0x0010 */ - .l3cc_value = L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_UC), - }, - [I915_MOCS_PTE] = { - /* 0x00000038 */ - .control_value = LE_CACHEABILITY(LE_PAGETABLE) | - LE_TGT_CACHE(LE_TC_LLC_ELLC) | - LE_LRUM(3) | LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | - LE_PFM(0) | LE_SCF(0), - /* 0x0030 */ - .l3cc_value = L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB), - }, - [I915_MOCS_CACHED] = { - /* 0x0000003b */ - .control_value = LE_CACHEABILITY(LE_WB) | - LE_TGT_CACHE(LE_TC_LLC_ELLC) | - LE_LRUM(3) | LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | - LE_PFM(0) | LE_SCF(0), - /* 0x0030 */ - .l3cc_value = L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB), - }, + GEN9_MOCS_ENTRIES, + MOCS_ENTRY(I915_MOCS_CACHED, + LE_3_WB | LE_TC_2_LLC_ELLC | LE_LRUM(3), + L3_3_WB) }; /* NOTE: the LE_TGT_CACHE is not used on Broxton */ static const struct drm_i915_mocs_entry broxton_mocs_table[] = { - [I915_MOCS_UNCACHED] = { - /* 0x00000009 */ - .control_value = LE_CACHEABILITY(LE_UC) | - LE_TGT_CACHE(LE_TC_LLC_ELLC) | - LE_LRUM(0) | LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | - LE_PFM(0) | LE_SCF(0), - - /* 0x0010 */ - .l3cc_value = L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_UC), - }, - [I915_MOCS_PTE] = { - /* 0x00000038 */ - .control_value = LE_CACHEABILITY(LE_PAGETABLE) | - LE_TGT_CACHE(LE_TC_LLC_ELLC) | - LE_LRUM(3) | LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | - LE_PFM(0) | LE_SCF(0), - - /* 0x0030 */ - .l3cc_value = L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB), - }, - [I915_MOCS_CACHED] = { - /* 0x00000039 */ - .control_value = LE_CACHEABILITY(LE_UC) | - LE_TGT_CACHE(LE_TC_LLC_ELLC) | - LE_LRUM(3) | LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | - LE_PFM(0) | LE_SCF(0), - - /* 0x0030 */ - .l3cc_value = L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB), - }, + GEN9_MOCS_ENTRIES, + MOCS_ENTRY(I915_MOCS_CACHED, + LE_1_UC | LE_TC_2_LLC_ELLC | LE_LRUM(3), + L3_3_WB) +}; + +#define GEN11_MOCS_ENTRIES \ + /* Base - Uncached (Deprecated) */ \ + MOCS_ENTRY(I915_MOCS_UNCACHED, \ + LE_1_UC | LE_TC_1_LLC, \ + L3_1_UC), \ + /* Base - L3 + LeCC:PAT (Deprecated) */ \ + MOCS_ENTRY(I915_MOCS_PTE, \ + LE_0_PAGETABLE | LE_TC_1_LLC, \ + L3_3_WB), \ + /* Base - L3 + LLC */ \ + MOCS_ENTRY(2, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \ + L3_3_WB), \ + /* Base - Uncached */ \ + MOCS_ENTRY(3, \ + LE_1_UC | LE_TC_1_LLC, \ + L3_1_UC), \ + /* Base - L3 */ \ + MOCS_ENTRY(4, \ + LE_1_UC | LE_TC_1_LLC, \ + L3_3_WB), \ + /* Base - LLC */ \ + MOCS_ENTRY(5, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \ + L3_1_UC), \ + /* Age 0 - LLC */ \ + MOCS_ENTRY(6, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), \ + L3_1_UC), \ + /* Age 0 - L3 + LLC */ \ + MOCS_ENTRY(7, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), \ + L3_3_WB), \ + /* Age: Don't Chg. - LLC */ \ + MOCS_ENTRY(8, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), \ + L3_1_UC), \ + /* Age: Don't Chg. - L3 + LLC */ \ + MOCS_ENTRY(9, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), \ + L3_3_WB), \ + /* No AOM - LLC */ \ + MOCS_ENTRY(10, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), \ + L3_1_UC), \ + /* No AOM - L3 + LLC */ \ + MOCS_ENTRY(11, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), \ + L3_3_WB), \ + /* No AOM; Age 0 - LLC */ \ + MOCS_ENTRY(12, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), \ + L3_1_UC), \ + /* No AOM; Age 0 - L3 + LLC */ \ + MOCS_ENTRY(13, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), \ + L3_3_WB), \ + /* No AOM; Age:DC - LLC */ \ + MOCS_ENTRY(14, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), \ + L3_1_UC), \ + /* No AOM; Age:DC - L3 + LLC */ \ + MOCS_ENTRY(15, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), \ + L3_3_WB), \ + /* Self-Snoop - L3 + LLC */ \ + MOCS_ENTRY(18, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SSE(3), \ + L3_3_WB), \ + /* Skip Caching - L3 + LLC(12.5%) */ \ + MOCS_ENTRY(19, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(7), \ + L3_3_WB), \ + /* Skip Caching - L3 + LLC(25%) */ \ + MOCS_ENTRY(20, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(3), \ + L3_3_WB), \ + /* Skip Caching - L3 + LLC(50%) */ \ + MOCS_ENTRY(21, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(1), \ + L3_3_WB), \ + /* Skip Caching - L3 + LLC(75%) */ \ + MOCS_ENTRY(22, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(3), \ + L3_3_WB), \ + /* Skip Caching - L3 + LLC(87.5%) */ \ + MOCS_ENTRY(23, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(7), \ + L3_3_WB), \ + /* HW Reserved - SW program but never use */ \ + MOCS_ENTRY(62, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \ + L3_1_UC), \ + /* HW Reserved - SW program but never use */ \ + MOCS_ENTRY(63, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \ + L3_1_UC) + +static const struct drm_i915_mocs_entry icelake_mocs_table[] = { + GEN11_MOCS_ENTRIES }; /** @@ -178,13 +252,19 @@ static bool get_mocs_settings(struct drm_i915_private *dev_priv, { bool result = false; - if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv) || - IS_ICELAKE(dev_priv)) { + if (IS_ICELAKE(dev_priv)) { + table->size = ARRAY_SIZE(icelake_mocs_table); + table->table = icelake_mocs_table; + table->n_entries = GEN11_NUM_MOCS_ENTRIES; + result = true; + } else if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) { table->size = ARRAY_SIZE(skylake_mocs_table); + table->n_entries = GEN9_NUM_MOCS_ENTRIES; table->table = skylake_mocs_table; result = true; } else if (IS_GEN9_LP(dev_priv)) { table->size = ARRAY_SIZE(broxton_mocs_table); + table->n_entries = GEN9_NUM_MOCS_ENTRIES; table->table = broxton_mocs_table; result = true; } else { @@ -193,7 +273,7 @@ static bool get_mocs_settings(struct drm_i915_private *dev_priv, } /* WaDisableSkipCaching:skl,bxt,kbl,glk */ - if (IS_GEN9(dev_priv)) { + if (IS_GEN(dev_priv, 9)) { int i; for (i = 0; i < table->size; i++) @@ -226,6 +306,19 @@ static i915_reg_t mocs_register(enum intel_engine_id engine_id, int index) } } +/* + * Get control_value from MOCS entry taking into account when it's not used: + * I915_MOCS_PTE's value is returned in this case. + */ +static u32 get_entry_control(const struct drm_i915_mocs_table *table, + unsigned int index) +{ + if (table->table[index].used) + return table->table[index].control_value; + + return table->table[I915_MOCS_PTE].control_value; +} + /** * intel_mocs_init_engine() - emit the mocs control table * @engine: The engine for whom to emit the registers. @@ -238,27 +331,23 @@ void intel_mocs_init_engine(struct intel_engine_cs *engine) struct drm_i915_private *dev_priv = engine->i915; struct drm_i915_mocs_table table; unsigned int index; + u32 unused_value; if (!get_mocs_settings(dev_priv, &table)) return; - GEM_BUG_ON(table.size > GEN9_NUM_MOCS_ENTRIES); - - for (index = 0; index < table.size; index++) - I915_WRITE(mocs_register(engine->id, index), - table.table[index].control_value); - - /* - * Ok, now set the unused entries to uncached. These entries - * are officially undefined and no contract for the contents - * and settings is given for these entries. - * - * Entry 0 in the table is uncached - so we are just writing - * that value to all the used entries. - */ - for (; index < GEN9_NUM_MOCS_ENTRIES; index++) - I915_WRITE(mocs_register(engine->id, index), - table.table[0].control_value); + /* Set unused values to PTE */ + unused_value = table.table[I915_MOCS_PTE].control_value; + + for (index = 0; index < table.size; index++) { + u32 value = get_entry_control(&table, index); + + I915_WRITE(mocs_register(engine->id, index), value); + } + + /* All remaining entries are also unused */ + for (; index < table.n_entries; index++) + I915_WRITE(mocs_register(engine->id, index), unused_value); } /** @@ -276,33 +365,32 @@ static int emit_mocs_control_table(struct i915_request *rq, { enum intel_engine_id engine = rq->engine->id; unsigned int index; + u32 unused_value; u32 *cs; - if (WARN_ON(table->size > GEN9_NUM_MOCS_ENTRIES)) + if (GEM_WARN_ON(table->size > table->n_entries)) return -ENODEV; - cs = intel_ring_begin(rq, 2 + 2 * GEN9_NUM_MOCS_ENTRIES); + /* Set unused values to PTE */ + unused_value = table->table[I915_MOCS_PTE].control_value; + + cs = intel_ring_begin(rq, 2 + 2 * table->n_entries); if (IS_ERR(cs)) return PTR_ERR(cs); - *cs++ = MI_LOAD_REGISTER_IMM(GEN9_NUM_MOCS_ENTRIES); + *cs++ = MI_LOAD_REGISTER_IMM(table->n_entries); for (index = 0; index < table->size; index++) { + u32 value = get_entry_control(table, index); + *cs++ = i915_mmio_reg_offset(mocs_register(engine, index)); - *cs++ = table->table[index].control_value; + *cs++ = value; } - /* - * Ok, now set the unused entries to uncached. These entries - * are officially undefined and no contract for the contents - * and settings is given for these entries. - * - * Entry 0 in the table is uncached - so we are just writing - * that value to all the used entries. - */ - for (; index < GEN9_NUM_MOCS_ENTRIES; index++) { + /* All remaining entries are also unused */ + for (; index < table->n_entries; index++) { *cs++ = i915_mmio_reg_offset(mocs_register(engine, index)); - *cs++ = table->table[0].control_value; + *cs++ = unused_value; } *cs++ = MI_NOOP; @@ -311,12 +399,24 @@ static int emit_mocs_control_table(struct i915_request *rq, return 0; } +/* + * Get l3cc_value from MOCS entry taking into account when it's not used: + * I915_MOCS_PTE's value is returned in this case. + */ +static u16 get_entry_l3cc(const struct drm_i915_mocs_table *table, + unsigned int index) +{ + if (table->table[index].used) + return table->table[index].l3cc_value; + + return table->table[I915_MOCS_PTE].l3cc_value; +} + static inline u32 l3cc_combine(const struct drm_i915_mocs_table *table, u16 low, u16 high) { - return table->table[low].l3cc_value | - table->table[high].l3cc_value << 16; + return low | high << 16; } /** @@ -333,38 +433,43 @@ static inline u32 l3cc_combine(const struct drm_i915_mocs_table *table, static int emit_mocs_l3cc_table(struct i915_request *rq, const struct drm_i915_mocs_table *table) { + u16 unused_value; unsigned int i; u32 *cs; - if (WARN_ON(table->size > GEN9_NUM_MOCS_ENTRIES)) + if (GEM_WARN_ON(table->size > table->n_entries)) return -ENODEV; - cs = intel_ring_begin(rq, 2 + GEN9_NUM_MOCS_ENTRIES); + /* Set unused values to PTE */ + unused_value = table->table[I915_MOCS_PTE].l3cc_value; + + cs = intel_ring_begin(rq, 2 + table->n_entries); if (IS_ERR(cs)) return PTR_ERR(cs); - *cs++ = MI_LOAD_REGISTER_IMM(GEN9_NUM_MOCS_ENTRIES / 2); + *cs++ = MI_LOAD_REGISTER_IMM(table->n_entries / 2); + + for (i = 0; i < table->size / 2; i++) { + u16 low = get_entry_l3cc(table, 2 * i); + u16 high = get_entry_l3cc(table, 2 * i + 1); - for (i = 0; i < table->size/2; i++) { *cs++ = i915_mmio_reg_offset(GEN9_LNCFCMOCS(i)); - *cs++ = l3cc_combine(table, 2 * i, 2 * i + 1); + *cs++ = l3cc_combine(table, low, high); } + /* Odd table size - 1 left over */ if (table->size & 0x01) { - /* Odd table size - 1 left over */ + u16 low = get_entry_l3cc(table, 2 * i); + *cs++ = i915_mmio_reg_offset(GEN9_LNCFCMOCS(i)); - *cs++ = l3cc_combine(table, 2 * i, 0); + *cs++ = l3cc_combine(table, low, unused_value); i++; } - /* - * Now set the rest of the table to uncached - use entry 0 as - * this will be uncached. Leave the last pair uninitialised as - * they are reserved by the hardware. - */ - for (; i < GEN9_NUM_MOCS_ENTRIES / 2; i++) { + /* All remaining entries are also unused */ + for (; i < table->n_entries / 2; i++) { *cs++ = i915_mmio_reg_offset(GEN9_LNCFCMOCS(i)); - *cs++ = l3cc_combine(table, 0, 0); + *cs++ = l3cc_combine(table, unused_value, unused_value); } *cs++ = MI_NOOP; @@ -391,26 +496,35 @@ void intel_mocs_init_l3cc_table(struct drm_i915_private *dev_priv) { struct drm_i915_mocs_table table; unsigned int i; + u16 unused_value; if (!get_mocs_settings(dev_priv, &table)) return; - for (i = 0; i < table.size/2; i++) - I915_WRITE(GEN9_LNCFCMOCS(i), l3cc_combine(&table, 2*i, 2*i+1)); + /* Set unused values to PTE */ + unused_value = table.table[I915_MOCS_PTE].l3cc_value; + + for (i = 0; i < table.size / 2; i++) { + u16 low = get_entry_l3cc(&table, 2 * i); + u16 high = get_entry_l3cc(&table, 2 * i + 1); + + I915_WRITE(GEN9_LNCFCMOCS(i), + l3cc_combine(&table, low, high)); + } /* Odd table size - 1 left over */ if (table.size & 0x01) { - I915_WRITE(GEN9_LNCFCMOCS(i), l3cc_combine(&table, 2*i, 0)); + u16 low = get_entry_l3cc(&table, 2 * i); + + I915_WRITE(GEN9_LNCFCMOCS(i), + l3cc_combine(&table, low, unused_value)); i++; } - /* - * Now set the rest of the table to uncached - use entry 0 as - * this will be uncached. Leave the last pair as initialised as - * they are reserved by the hardware. - */ - for (; i < (GEN9_NUM_MOCS_ENTRIES / 2); i++) - I915_WRITE(GEN9_LNCFCMOCS(i), l3cc_combine(&table, 0, 0)); + /* All remaining entries are also unused */ + for (; i < table.n_entries / 2; i++) + I915_WRITE(GEN9_LNCFCMOCS(i), + l3cc_combine(&table, unused_value, unused_value)); } /** diff --git a/drivers/gpu/drm/i915/intel_mocs.h b/drivers/gpu/drm/i915/intel_mocs.h index d89080d75b80..3d99d1271b2b 100644 --- a/drivers/gpu/drm/i915/intel_mocs.h +++ b/drivers/gpu/drm/i915/intel_mocs.h @@ -49,7 +49,6 @@ * context handling keep the MOCS in step. */ -#include <drm/drmP.h> #include "i915_drv.h" int intel_rcs_context_init_mocs(struct i915_request *rq); diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c index e034b4166d32..5e00ee9270b5 100644 --- a/drivers/gpu/drm/i915/intel_opregion.c +++ b/drivers/gpu/drm/i915/intel_opregion.c @@ -30,7 +30,6 @@ #include <linux/firmware.h> #include <acpi/video.h> -#include <drm/drmP.h> #include <drm/i915_drm.h> #include "intel_opregion.h" @@ -55,7 +54,12 @@ struct opregion_header { u8 signature[16]; u32 size; - u32 opregion_ver; + struct { + u8 rsvd; + u8 revision; + u8 minor; + u8 major; + } __packed over; u8 bios_ver[32]; u8 vbios_ver[16]; u8 driver_ver[16]; @@ -119,7 +123,8 @@ struct opregion_asle { u64 fdss; u32 fdsp; u32 stat; - u64 rvda; /* Physical address of raw vbt data */ + u64 rvda; /* Physical (2.0) or relative from opregion (2.1+) + * address of raw VBT data. */ u32 rvds; /* Size of raw vbt data */ u8 rsvd[58]; } __packed; @@ -773,70 +778,6 @@ static void intel_setup_cadls(struct drm_i915_private *dev_priv) opregion->acpi->cadl[i] = 0; } -void intel_opregion_register(struct drm_i915_private *dev_priv) -{ - struct intel_opregion *opregion = &dev_priv->opregion; - - if (!opregion->header) - return; - - if (opregion->acpi) { - intel_didl_outputs(dev_priv); - intel_setup_cadls(dev_priv); - - /* Notify BIOS we are ready to handle ACPI video ext notifs. - * Right now, all the events are handled by the ACPI video module. - * We don't actually need to do anything with them. */ - opregion->acpi->csts = 0; - opregion->acpi->drdy = 1; - - opregion->acpi_notifier.notifier_call = intel_opregion_video_event; - register_acpi_notifier(&opregion->acpi_notifier); - } - - if (opregion->asle) { - opregion->asle->tche = ASLE_TCHE_BLC_EN; - opregion->asle->ardy = ASLE_ARDY_READY; - } -} - -void intel_opregion_unregister(struct drm_i915_private *dev_priv) -{ - struct intel_opregion *opregion = &dev_priv->opregion; - - if (!opregion->header) - return; - - if (opregion->asle) - opregion->asle->ardy = ASLE_ARDY_NOT_READY; - - cancel_work_sync(&dev_priv->opregion.asle_work); - - if (opregion->acpi) { - opregion->acpi->drdy = 0; - - unregister_acpi_notifier(&opregion->acpi_notifier); - opregion->acpi_notifier.notifier_call = NULL; - } - - /* just clear all opregion memory pointers now */ - memunmap(opregion->header); - if (opregion->rvda) { - memunmap(opregion->rvda); - opregion->rvda = NULL; - } - if (opregion->vbt_firmware) { - kfree(opregion->vbt_firmware); - opregion->vbt_firmware = NULL; - } - opregion->header = NULL; - opregion->acpi = NULL; - opregion->swsci = NULL; - opregion->asle = NULL; - opregion->vbt = NULL; - opregion->lid_state = NULL; -} - static void swsci_setup(struct drm_i915_private *dev_priv) { struct intel_opregion *opregion = &dev_priv->opregion; @@ -989,6 +930,11 @@ int intel_opregion_setup(struct drm_i915_private *dev_priv) opregion->header = base; opregion->lid_state = base + ACPI_CLID; + DRM_DEBUG_DRIVER("ACPI OpRegion version %u.%u.%u\n", + opregion->header->over.major, + opregion->header->over.minor, + opregion->header->over.revision); + mboxes = opregion->header->mboxes; if (mboxes & MBOX_ACPI) { DRM_DEBUG_DRIVER("Public ACPI methods supported\n"); @@ -1017,11 +963,26 @@ int intel_opregion_setup(struct drm_i915_private *dev_priv) if (dmi_check_system(intel_no_opregion_vbt)) goto out; - if (opregion->header->opregion_ver >= 2 && opregion->asle && + if (opregion->header->over.major >= 2 && opregion->asle && opregion->asle->rvda && opregion->asle->rvds) { - opregion->rvda = memremap(opregion->asle->rvda, - opregion->asle->rvds, + resource_size_t rvda = opregion->asle->rvda; + + /* + * opregion 2.0: rvda is the physical VBT address. + * + * opregion 2.1+: rvda is unsigned, relative offset from + * opregion base, and should never point within opregion. + */ + if (opregion->header->over.major > 2 || + opregion->header->over.minor >= 1) { + WARN_ON(rvda < OPREGION_SIZE); + + rvda += asls; + } + + opregion->rvda = memremap(rvda, opregion->asle->rvds, MEMREMAP_WB); + vbt = opregion->rvda; vbt_size = opregion->asle->rvds; if (intel_bios_is_valid_vbt(vbt, vbt_size)) { @@ -1031,6 +992,8 @@ int intel_opregion_setup(struct drm_i915_private *dev_priv) goto out; } else { DRM_DEBUG_KMS("Invalid VBT in ACPI OpRegion (RVDA)\n"); + memunmap(opregion->rvda); + opregion->rvda = NULL; } } @@ -1115,3 +1078,97 @@ intel_opregion_get_panel_type(struct drm_i915_private *dev_priv) return ret - 1; } + +void intel_opregion_register(struct drm_i915_private *i915) +{ + struct intel_opregion *opregion = &i915->opregion; + + if (!opregion->header) + return; + + if (opregion->acpi) { + opregion->acpi_notifier.notifier_call = + intel_opregion_video_event; + register_acpi_notifier(&opregion->acpi_notifier); + } + + intel_opregion_resume(i915); +} + +void intel_opregion_resume(struct drm_i915_private *i915) +{ + struct intel_opregion *opregion = &i915->opregion; + + if (!opregion->header) + return; + + if (opregion->acpi) { + intel_didl_outputs(i915); + intel_setup_cadls(i915); + + /* + * Notify BIOS we are ready to handle ACPI video ext notifs. + * Right now, all the events are handled by the ACPI video + * module. We don't actually need to do anything with them. + */ + opregion->acpi->csts = 0; + opregion->acpi->drdy = 1; + } + + if (opregion->asle) { + opregion->asle->tche = ASLE_TCHE_BLC_EN; + opregion->asle->ardy = ASLE_ARDY_READY; + } + + intel_opregion_notify_adapter(i915, PCI_D0); +} + +void intel_opregion_suspend(struct drm_i915_private *i915, pci_power_t state) +{ + struct intel_opregion *opregion = &i915->opregion; + + if (!opregion->header) + return; + + intel_opregion_notify_adapter(i915, state); + + if (opregion->asle) + opregion->asle->ardy = ASLE_ARDY_NOT_READY; + + cancel_work_sync(&i915->opregion.asle_work); + + if (opregion->acpi) + opregion->acpi->drdy = 0; +} + +void intel_opregion_unregister(struct drm_i915_private *i915) +{ + struct intel_opregion *opregion = &i915->opregion; + + intel_opregion_suspend(i915, PCI_D1); + + if (!opregion->header) + return; + + if (opregion->acpi_notifier.notifier_call) { + unregister_acpi_notifier(&opregion->acpi_notifier); + opregion->acpi_notifier.notifier_call = NULL; + } + + /* just clear all opregion memory pointers now */ + memunmap(opregion->header); + if (opregion->rvda) { + memunmap(opregion->rvda); + opregion->rvda = NULL; + } + if (opregion->vbt_firmware) { + kfree(opregion->vbt_firmware); + opregion->vbt_firmware = NULL; + } + opregion->header = NULL; + opregion->acpi = NULL; + opregion->swsci = NULL; + opregion->asle = NULL; + opregion->vbt = NULL; + opregion->lid_state = NULL; +} diff --git a/drivers/gpu/drm/i915/intel_opregion.h b/drivers/gpu/drm/i915/intel_opregion.h index e8498a8cda3d..4aa68ffbd30e 100644 --- a/drivers/gpu/drm/i915/intel_opregion.h +++ b/drivers/gpu/drm/i915/intel_opregion.h @@ -57,8 +57,14 @@ struct intel_opregion { #ifdef CONFIG_ACPI int intel_opregion_setup(struct drm_i915_private *dev_priv); + void intel_opregion_register(struct drm_i915_private *dev_priv); void intel_opregion_unregister(struct drm_i915_private *dev_priv); + +void intel_opregion_resume(struct drm_i915_private *dev_priv); +void intel_opregion_suspend(struct drm_i915_private *dev_priv, + pci_power_t state); + void intel_opregion_asle_intr(struct drm_i915_private *dev_priv); int intel_opregion_notify_encoder(struct intel_encoder *intel_encoder, bool enable); @@ -81,6 +87,15 @@ static inline void intel_opregion_unregister(struct drm_i915_private *dev_priv) { } +static inline void intel_opregion_resume(struct drm_i915_private *dev_priv) +{ +} + +static inline void intel_opregion_suspend(struct drm_i915_private *dev_priv, + pci_power_t state) +{ +} + static inline void intel_opregion_asle_intr(struct drm_i915_private *dev_priv) { } diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c index 72eb7e48e8bc..c0df1dbb0069 100644 --- a/drivers/gpu/drm/i915/intel_overlay.c +++ b/drivers/gpu/drm/i915/intel_overlay.c @@ -25,8 +25,9 @@ * * Derived from Xorg ddx, xf86-video-intel, src/i830_video.c */ -#include <drm/drmP.h> #include <drm/i915_drm.h> +#include <drm/drm_fourcc.h> + #include "i915_drv.h" #include "i915_reg.h" #include "intel_drv.h" @@ -185,7 +186,7 @@ struct intel_overlay { struct overlay_registers __iomem *regs; u32 flip_addr; /* flip handling */ - struct i915_gem_active last_flip; + struct i915_active_request last_flip; }; static void i830_overlay_clock_gating(struct drm_i915_private *dev_priv, @@ -213,23 +214,23 @@ static void i830_overlay_clock_gating(struct drm_i915_private *dev_priv, static void intel_overlay_submit_request(struct intel_overlay *overlay, struct i915_request *rq, - i915_gem_retire_fn retire) + i915_active_retire_fn retire) { - GEM_BUG_ON(i915_gem_active_peek(&overlay->last_flip, - &overlay->i915->drm.struct_mutex)); - i915_gem_active_set_retire_fn(&overlay->last_flip, retire, - &overlay->i915->drm.struct_mutex); - i915_gem_active_set(&overlay->last_flip, rq); + GEM_BUG_ON(i915_active_request_peek(&overlay->last_flip, + &overlay->i915->drm.struct_mutex)); + i915_active_request_set_retire_fn(&overlay->last_flip, retire, + &overlay->i915->drm.struct_mutex); + __i915_active_request_set(&overlay->last_flip, rq); i915_request_add(rq); } static int intel_overlay_do_wait_request(struct intel_overlay *overlay, struct i915_request *rq, - i915_gem_retire_fn retire) + i915_active_retire_fn retire) { intel_overlay_submit_request(overlay, rq, retire); - return i915_gem_active_retire(&overlay->last_flip, - &overlay->i915->drm.struct_mutex); + return i915_active_request_retire(&overlay->last_flip, + &overlay->i915->drm.struct_mutex); } static struct i915_request *alloc_request(struct intel_overlay *overlay) @@ -350,8 +351,9 @@ static void intel_overlay_release_old_vma(struct intel_overlay *overlay) i915_vma_put(vma); } -static void intel_overlay_release_old_vid_tail(struct i915_gem_active *active, - struct i915_request *rq) +static void +intel_overlay_release_old_vid_tail(struct i915_active_request *active, + struct i915_request *rq) { struct intel_overlay *overlay = container_of(active, typeof(*overlay), last_flip); @@ -359,7 +361,7 @@ static void intel_overlay_release_old_vid_tail(struct i915_gem_active *active, intel_overlay_release_old_vma(overlay); } -static void intel_overlay_off_tail(struct i915_gem_active *active, +static void intel_overlay_off_tail(struct i915_active_request *active, struct i915_request *rq) { struct intel_overlay *overlay = @@ -422,8 +424,8 @@ static int intel_overlay_off(struct intel_overlay *overlay) * We have to be careful not to repeat work forever an make forward progess. */ static int intel_overlay_recover_from_interrupt(struct intel_overlay *overlay) { - return i915_gem_active_retire(&overlay->last_flip, - &overlay->i915->drm.struct_mutex); + return i915_active_request_retire(&overlay->last_flip, + &overlay->i915->drm.struct_mutex); } /* Wait for pending overlay flip and release old frame. @@ -479,8 +481,6 @@ void intel_overlay_reset(struct drm_i915_private *dev_priv) if (!overlay) return; - intel_overlay_release_old_vid(overlay); - overlay->old_xscale = 0; overlay->old_yscale = 0; overlay->crtc = NULL; @@ -541,7 +541,7 @@ static u32 calc_swidthsw(struct drm_i915_private *dev_priv, u32 offset, u32 widt { u32 sw; - if (IS_GEN2(dev_priv)) + if (IS_GEN(dev_priv, 2)) sw = ALIGN((offset & 31) + width, 32); else sw = ALIGN((offset & 63) + width, 64); @@ -778,7 +778,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay, u32 oconfig; oconfig = OCONF_CC_OUT_8BIT; - if (IS_GEN4(dev_priv)) + if (IS_GEN(dev_priv, 4)) oconfig |= OCONF_CSC_MODE_BT709; oconfig |= pipe == 0 ? OCONF_PIPE_A : OCONF_PIPE_B; @@ -1012,7 +1012,7 @@ static int check_overlay_src(struct drm_i915_private *dev_priv, if (rec->stride_Y & stride_mask || rec->stride_UV & stride_mask) return -EINVAL; - if (IS_GEN4(dev_priv) && rec->stride_Y < 512) + if (IS_GEN(dev_priv, 4) && rec->stride_Y < 512) return -EINVAL; tmp = (rec->flags & I915_OVERLAY_TYPE_MASK) == I915_OVERLAY_YUV_PLANAR ? @@ -1246,7 +1246,7 @@ int intel_overlay_attrs_ioctl(struct drm_device *dev, void *data, attrs->contrast = overlay->contrast; attrs->saturation = overlay->saturation; - if (!IS_GEN2(dev_priv)) { + if (!IS_GEN(dev_priv, 2)) { attrs->gamma0 = I915_READ(OGAMC0); attrs->gamma1 = I915_READ(OGAMC1); attrs->gamma2 = I915_READ(OGAMC2); @@ -1270,7 +1270,7 @@ int intel_overlay_attrs_ioctl(struct drm_device *dev, void *data, update_reg_attrs(overlay, overlay->regs); if (attrs->flags & I915_OVERLAY_UPDATE_GAMMA) { - if (IS_GEN2(dev_priv)) + if (IS_GEN(dev_priv, 2)) goto out_unlock; if (overlay->active) { @@ -1338,7 +1338,7 @@ err_put_bo: return err; } -void intel_setup_overlay(struct drm_i915_private *dev_priv) +void intel_overlay_setup(struct drm_i915_private *dev_priv) { struct intel_overlay *overlay; int ret; @@ -1358,7 +1358,7 @@ void intel_setup_overlay(struct drm_i915_private *dev_priv) overlay->contrast = 75; overlay->saturation = 146; - init_request_active(&overlay->last_flip, NULL); + INIT_ACTIVE_REQUEST(&overlay->last_flip); mutex_lock(&dev_priv->drm.struct_mutex); @@ -1387,7 +1387,7 @@ out_free: kfree(overlay); } -void intel_cleanup_overlay(struct drm_i915_private *dev_priv) +void intel_overlay_cleanup(struct drm_i915_private *dev_priv) { struct intel_overlay *overlay; diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c index 4a9f139e7b73..beca98d2b035 100644 --- a/drivers/gpu/drm/i915/intel_panel.c +++ b/drivers/gpu/drm/i915/intel_panel.c @@ -111,7 +111,7 @@ intel_pch_panel_fitting(struct intel_crtc *intel_crtc, /* Native modes don't need fitting */ if (adjusted_mode->crtc_hdisplay == pipe_config->pipe_src_w && adjusted_mode->crtc_vdisplay == pipe_config->pipe_src_h && - !pipe_config->ycbcr420) + pipe_config->output_format != INTEL_OUTPUT_FORMAT_YCBCR420) goto done; switch (fitting_mode) { @@ -505,7 +505,7 @@ static u32 _vlv_get_backlight(struct drm_i915_private *dev_priv, enum pipe pipe) static u32 vlv_get_backlight(struct intel_connector *connector) { struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - enum pipe pipe = intel_get_pipe_from_connector(connector); + enum pipe pipe = intel_connector_get_pipe(connector); return _vlv_get_backlight(dev_priv, pipe); } @@ -563,7 +563,7 @@ static void i9xx_set_backlight(const struct drm_connector_state *conn_state, u32 pci_write_config_byte(dev_priv->drm.pdev, LBPC, lbpc); } - if (IS_GEN4(dev_priv)) { + if (IS_GEN(dev_priv, 4)) { mask = BACKLIGHT_DUTY_CYCLE_MASK; } else { level <<= 1; @@ -763,7 +763,7 @@ static void pwm_disable_backlight(const struct drm_connector_state *old_conn_sta struct intel_panel *panel = &connector->panel; /* Disable the backlight */ - pwm_config(panel->backlight.pwm, 0, CRC_PMIC_PWM_PERIOD_NS); + intel_panel_actually_set_backlight(old_conn_state, 0); usleep_range(2000, 3000); pwm_disable(panel->backlight.pwm); } @@ -929,7 +929,7 @@ static void i9xx_enable_backlight(const struct intel_crtc_state *crtc_state, * 855gm only, but checking for gen2 is safe, as 855gm is the only gen2 * that has backlight. */ - if (IS_GEN2(dev_priv)) + if (IS_GEN(dev_priv, 2)) I915_WRITE(BLC_HIST_CTL, BLM_HISTOGRAM_ENABLE); } @@ -1087,20 +1087,11 @@ static void pwm_enable_backlight(const struct intel_crtc_state *crtc_state, intel_panel_actually_set_backlight(conn_state, panel->backlight.level); } -void intel_panel_enable_backlight(const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state) +static void __intel_panel_enable_backlight(const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) { struct intel_connector *connector = to_intel_connector(conn_state->connector); - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); struct intel_panel *panel = &connector->panel; - enum pipe pipe = to_intel_crtc(crtc_state->base.crtc)->pipe; - - if (!panel->backlight.present) - return; - - DRM_DEBUG_KMS("pipe %c\n", pipe_name(pipe)); - - mutex_lock(&dev_priv->backlight_lock); WARN_ON(panel->backlight.max == 0); @@ -1117,6 +1108,24 @@ void intel_panel_enable_backlight(const struct intel_crtc_state *crtc_state, panel->backlight.enabled = true; if (panel->backlight.device) panel->backlight.device->props.power = FB_BLANK_UNBLANK; +} + +void intel_panel_enable_backlight(const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + enum pipe pipe = to_intel_crtc(crtc_state->base.crtc)->pipe; + + if (!panel->backlight.present) + return; + + DRM_DEBUG_KMS("pipe %c\n", pipe_name(pipe)); + + mutex_lock(&dev_priv->backlight_lock); + + __intel_panel_enable_backlight(crtc_state, conn_state); mutex_unlock(&dev_priv->backlight_lock); } @@ -1203,17 +1212,20 @@ static int intel_backlight_device_get_brightness(struct backlight_device *bd) struct intel_connector *connector = bl_get_data(bd); struct drm_device *dev = connector->base.dev; struct drm_i915_private *dev_priv = to_i915(dev); - u32 hw_level; - int ret; + intel_wakeref_t wakeref; + int ret = 0; - intel_runtime_pm_get(dev_priv); - drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); + with_intel_runtime_pm(dev_priv, wakeref) { + u32 hw_level; - hw_level = intel_panel_get_backlight(connector); - ret = scale_hw_to_user(connector, hw_level, bd->props.max_brightness); + drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); - drm_modeset_unlock(&dev->mode_config.connection_mutex); - intel_runtime_pm_put(dev_priv); + hw_level = intel_panel_get_backlight(connector); + ret = scale_hw_to_user(connector, + hw_level, bd->props.max_brightness); + + drm_modeset_unlock(&dev->mode_config.connection_mutex); + } return ret; } @@ -1484,8 +1496,8 @@ static int lpt_setup_backlight(struct intel_connector *connector, enum pipe unus { struct drm_i915_private *dev_priv = to_i915(connector->base.dev); struct intel_panel *panel = &connector->panel; - u32 pch_ctl1, pch_ctl2, val; - bool alt; + u32 cpu_ctl2, pch_ctl1, pch_ctl2, val; + bool alt, cpu_mode; if (HAS_PCH_LPT(dev_priv)) alt = I915_READ(SOUTH_CHICKEN2) & LPT_PWM_GRANULARITY; @@ -1499,6 +1511,8 @@ static int lpt_setup_backlight(struct intel_connector *connector, enum pipe unus pch_ctl2 = I915_READ(BLC_PWM_PCH_CTL2); panel->backlight.max = pch_ctl2 >> 16; + cpu_ctl2 = I915_READ(BLC_PWM_CPU_CTL2); + if (!panel->backlight.max) panel->backlight.max = get_backlight_max_vbt(connector); @@ -1507,12 +1521,28 @@ static int lpt_setup_backlight(struct intel_connector *connector, enum pipe unus panel->backlight.min = get_backlight_min_vbt(connector); - val = lpt_get_backlight(connector); + panel->backlight.enabled = pch_ctl1 & BLM_PCH_PWM_ENABLE; + + cpu_mode = panel->backlight.enabled && HAS_PCH_LPT(dev_priv) && + !(pch_ctl1 & BLM_PCH_OVERRIDE_ENABLE) && + (cpu_ctl2 & BLM_PWM_ENABLE); + if (cpu_mode) + val = pch_get_backlight(connector); + else + val = lpt_get_backlight(connector); val = intel_panel_compute_brightness(connector, val); panel->backlight.level = clamp(val, panel->backlight.min, panel->backlight.max); - panel->backlight.enabled = pch_ctl1 & BLM_PCH_PWM_ENABLE; + if (cpu_mode) { + DRM_DEBUG_KMS("CPU backlight register was enabled, switching to PCH override\n"); + + /* Write converted CPU PWM value to PCH override register */ + lpt_set_backlight(connector->base.state, panel->backlight.level); + I915_WRITE(BLC_PWM_PCH_CTL1, pch_ctl1 | BLM_PCH_OVERRIDE_ENABLE); + + I915_WRITE(BLC_PWM_CPU_CTL2, cpu_ctl2 & ~BLM_PWM_ENABLE); + } return 0; } @@ -1557,7 +1587,7 @@ static int i9xx_setup_backlight(struct intel_connector *connector, enum pipe unu ctl = I915_READ(BLC_PWM_CTL); - if (IS_GEN2(dev_priv) || IS_I915GM(dev_priv) || IS_I945GM(dev_priv)) + if (IS_GEN(dev_priv, 2) || IS_I915GM(dev_priv) || IS_I945GM(dev_priv)) panel->backlight.combination_mode = ctl & BLM_LEGACY_MODE; if (IS_PINEVIEW(dev_priv)) @@ -1773,6 +1803,24 @@ static int pwm_setup_backlight(struct intel_connector *connector, return 0; } +void intel_panel_update_backlight(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + struct intel_panel *panel = &connector->panel; + + if (!panel->backlight.present) + return; + + mutex_lock(&dev_priv->backlight_lock); + if (!panel->backlight.enabled) + __intel_panel_enable_backlight(crtc_state, conn_state); + + mutex_unlock(&dev_priv->backlight_lock); +} + int intel_panel_setup_backlight(struct drm_connector *connector, enum pipe pipe) { struct drm_i915_private *dev_priv = to_i915(connector->dev); @@ -1814,11 +1862,8 @@ int intel_panel_setup_backlight(struct drm_connector *connector, enum pipe pipe) return 0; } -void intel_panel_destroy_backlight(struct drm_connector *connector) +static void intel_panel_destroy_backlight(struct intel_panel *panel) { - struct intel_connector *intel_connector = to_intel_connector(connector); - struct intel_panel *panel = &intel_connector->panel; - /* dispose of the pwm */ if (panel->backlight.pwm) pwm_put(panel->backlight.pwm); @@ -1889,7 +1934,7 @@ intel_panel_init_backlight_funcs(struct intel_panel *panel) panel->backlight.get = vlv_get_backlight; panel->backlight.hz_to_pwm = vlv_hz_to_pwm; } - } else if (IS_GEN4(dev_priv)) { + } else if (IS_GEN(dev_priv, 4)) { panel->backlight.setup = i965_setup_backlight; panel->backlight.enable = i965_enable_backlight; panel->backlight.disable = i965_disable_backlight; @@ -1923,6 +1968,8 @@ void intel_panel_fini(struct intel_panel *panel) struct intel_connector *intel_connector = container_of(panel, struct intel_connector, panel); + intel_panel_destroy_backlight(panel); + if (panel->fixed_mode) drm_mode_destroy(intel_connector->base.dev, panel->fixed_mode); diff --git a/drivers/gpu/drm/i915/intel_pipe_crc.c b/drivers/gpu/drm/i915/intel_pipe_crc.c index f3c9010e332a..a8554dc4f196 100644 --- a/drivers/gpu/drm/i915/intel_pipe_crc.c +++ b/drivers/gpu/drm/i915/intel_pipe_crc.c @@ -44,7 +44,7 @@ static const char * const pipe_crc_sources[] = { }; static int i8xx_pipe_crc_ctl_reg(enum intel_pipe_crc_source *source, - uint32_t *val) + u32 *val) { if (*source == INTEL_PIPE_CRC_SOURCE_AUTO) *source = INTEL_PIPE_CRC_SOURCE_PIPE; @@ -120,7 +120,7 @@ static int i9xx_pipe_crc_auto_source(struct drm_i915_private *dev_priv, static int vlv_pipe_crc_ctl_reg(struct drm_i915_private *dev_priv, enum pipe pipe, enum intel_pipe_crc_source *source, - uint32_t *val) + u32 *val) { bool need_stable_symbols = false; @@ -165,7 +165,7 @@ static int vlv_pipe_crc_ctl_reg(struct drm_i915_private *dev_priv, * - DisplayPort scrambling: used for EMI reduction */ if (need_stable_symbols) { - uint32_t tmp = I915_READ(PORT_DFT2_G4X); + u32 tmp = I915_READ(PORT_DFT2_G4X); tmp |= DC_BALANCE_RESET_VLV; switch (pipe) { @@ -190,7 +190,7 @@ static int vlv_pipe_crc_ctl_reg(struct drm_i915_private *dev_priv, static int i9xx_pipe_crc_ctl_reg(struct drm_i915_private *dev_priv, enum pipe pipe, enum intel_pipe_crc_source *source, - uint32_t *val) + u32 *val) { bool need_stable_symbols = false; @@ -244,7 +244,7 @@ static int i9xx_pipe_crc_ctl_reg(struct drm_i915_private *dev_priv, * - DisplayPort scrambling: used for EMI reduction */ if (need_stable_symbols) { - uint32_t tmp = I915_READ(PORT_DFT2_G4X); + u32 tmp = I915_READ(PORT_DFT2_G4X); WARN_ON(!IS_G4X(dev_priv)); @@ -265,7 +265,7 @@ static int i9xx_pipe_crc_ctl_reg(struct drm_i915_private *dev_priv, static void vlv_undo_pipe_scramble_reset(struct drm_i915_private *dev_priv, enum pipe pipe) { - uint32_t tmp = I915_READ(PORT_DFT2_G4X); + u32 tmp = I915_READ(PORT_DFT2_G4X); switch (pipe) { case PIPE_A: @@ -289,7 +289,7 @@ static void vlv_undo_pipe_scramble_reset(struct drm_i915_private *dev_priv, static void g4x_undo_pipe_scramble_reset(struct drm_i915_private *dev_priv, enum pipe pipe) { - uint32_t tmp = I915_READ(PORT_DFT2_G4X); + u32 tmp = I915_READ(PORT_DFT2_G4X); if (pipe == PIPE_A) tmp &= ~PIPE_A_SCRAMBLE_RESET; @@ -304,7 +304,7 @@ static void g4x_undo_pipe_scramble_reset(struct drm_i915_private *dev_priv, } static int ilk_pipe_crc_ctl_reg(enum intel_pipe_crc_source *source, - uint32_t *val) + u32 *val) { if (*source == INTEL_PIPE_CRC_SOURCE_AUTO) *source = INTEL_PIPE_CRC_SOURCE_PIPE; @@ -392,7 +392,7 @@ unlock: static int ivb_pipe_crc_ctl_reg(struct drm_i915_private *dev_priv, enum pipe pipe, enum intel_pipe_crc_source *source, - uint32_t *val, + u32 *val, bool set_wa) { if (*source == INTEL_PIPE_CRC_SOURCE_AUTO) @@ -427,13 +427,13 @@ static int get_new_crc_ctl_reg(struct drm_i915_private *dev_priv, enum intel_pipe_crc_source *source, u32 *val, bool set_wa) { - if (IS_GEN2(dev_priv)) + if (IS_GEN(dev_priv, 2)) return i8xx_pipe_crc_ctl_reg(source, val); else if (INTEL_GEN(dev_priv) < 5) return i9xx_pipe_crc_ctl_reg(dev_priv, pipe, source, val); else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) return vlv_pipe_crc_ctl_reg(dev_priv, pipe, source, val); - else if (IS_GEN5(dev_priv) || IS_GEN6(dev_priv)) + else if (IS_GEN_RANGE(dev_priv, 5, 6)) return ilk_pipe_crc_ctl_reg(source, val); else return ivb_pipe_crc_ctl_reg(dev_priv, pipe, source, val, set_wa); @@ -544,13 +544,13 @@ static int intel_is_valid_crc_source(struct drm_i915_private *dev_priv, const enum intel_pipe_crc_source source) { - if (IS_GEN2(dev_priv)) + if (IS_GEN(dev_priv, 2)) return i8xx_crc_source_valid(dev_priv, source); else if (INTEL_GEN(dev_priv) < 5) return i9xx_crc_source_valid(dev_priv, source); else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) return vlv_crc_source_valid(dev_priv, source); - else if (IS_GEN5(dev_priv) || IS_GEN6(dev_priv)) + else if (IS_GEN_RANGE(dev_priv, 5, 6)) return ilk_crc_source_valid(dev_priv, source); else return ivb_crc_source_valid(dev_priv, source); @@ -589,6 +589,7 @@ int intel_crtc_set_crc_source(struct drm_crtc *crtc, const char *source_name) struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[crtc->index]; enum intel_display_power_domain power_domain; enum intel_pipe_crc_source source; + intel_wakeref_t wakeref; u32 val = 0; /* shut up gcc */ int ret = 0; @@ -598,7 +599,8 @@ int intel_crtc_set_crc_source(struct drm_crtc *crtc, const char *source_name) } power_domain = POWER_DOMAIN_PIPE(crtc->index); - if (!intel_display_power_get_if_enabled(dev_priv, power_domain)) { + wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); + if (!wakeref) { DRM_DEBUG_KMS("Trying to capture CRC while pipe is off\n"); return -EIO; } @@ -624,7 +626,7 @@ int intel_crtc_set_crc_source(struct drm_crtc *crtc, const char *source_name) pipe_crc->skipped = 0; out: - intel_display_power_put(dev_priv, power_domain); + intel_display_power_put(dev_priv, power_domain, wakeref); return ret; } diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index 3fe358db1276..54307f1df6cf 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -26,13 +26,16 @@ */ #include <linux/cpufreq.h> +#include <linux/module.h> #include <linux/pm_runtime.h> + +#include <drm/drm_atomic_helper.h> +#include <drm/drm_fourcc.h> #include <drm/drm_plane_helper.h> + #include "i915_drv.h" #include "intel_drv.h" #include "../../../platform/x86/intel_ips.h" -#include <linux/module.h> -#include <drm/drm_atomic_helper.h> /** * DOC: RC6 @@ -480,7 +483,7 @@ static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state) int sprite0_start, sprite1_start; switch (pipe) { - uint32_t dsparb, dsparb2, dsparb3; + u32 dsparb, dsparb2, dsparb3; case PIPE_A: dsparb = I915_READ(DSPARB); dsparb2 = I915_READ(DSPARB2); @@ -513,7 +516,7 @@ static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state) static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv, enum i9xx_plane_id i9xx_plane) { - uint32_t dsparb = I915_READ(DSPARB); + u32 dsparb = I915_READ(DSPARB); int size; size = dsparb & 0x7f; @@ -529,7 +532,7 @@ static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv, static int i830_get_fifo_size(struct drm_i915_private *dev_priv, enum i9xx_plane_id i9xx_plane) { - uint32_t dsparb = I915_READ(DSPARB); + u32 dsparb = I915_READ(DSPARB); int size; size = dsparb & 0x1ff; @@ -546,7 +549,7 @@ static int i830_get_fifo_size(struct drm_i915_private *dev_priv, static int i845_get_fifo_size(struct drm_i915_private *dev_priv, enum i9xx_plane_id i9xx_plane) { - uint32_t dsparb = I915_READ(DSPARB); + u32 dsparb = I915_READ(DSPARB); int size; size = dsparb & 0x7f; @@ -667,9 +670,9 @@ static unsigned int intel_wm_method1(unsigned int pixel_rate, unsigned int cpp, unsigned int latency) { - uint64_t ret; + u64 ret; - ret = (uint64_t) pixel_rate * cpp * latency; + ret = (u64)pixel_rate * cpp * latency; ret = DIV_ROUND_UP_ULL(ret, 10000); return ret; @@ -1089,9 +1092,9 @@ static int g4x_fbc_fifo_size(int level) } } -static uint16_t g4x_compute_wm(const struct intel_crtc_state *crtc_state, - const struct intel_plane_state *plane_state, - int level) +static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state, + int level) { struct intel_plane *plane = to_intel_plane(plane_state->base.plane); struct drm_i915_private *dev_priv = to_i915(plane->base.dev); @@ -1188,9 +1191,9 @@ static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state, return dirty; } -static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate, - const struct intel_plane_state *pstate, - uint32_t pri_val); +static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *cstate, + const struct intel_plane_state *pstate, + u32 pri_val); static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state) @@ -1399,10 +1402,9 @@ static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state) return 0; } -static int g4x_compute_intermediate_wm(struct drm_device *dev, - struct intel_crtc *crtc, - struct intel_crtc_state *new_crtc_state) +static int g4x_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state) { + struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc); struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate; const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal; struct intel_atomic_state *intel_state = @@ -1599,9 +1601,9 @@ static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv) } } -static uint16_t vlv_compute_wm_level(const struct intel_crtc_state *crtc_state, - const struct intel_plane_state *plane_state, - int level) +static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state, + int level) { struct intel_plane *plane = to_intel_plane(plane_state->base.plane); struct drm_i915_private *dev_priv = to_i915(plane->base.dev); @@ -1969,7 +1971,7 @@ static void vlv_atomic_update_fifo(struct intel_atomic_state *state, spin_lock(&dev_priv->uncore.lock); switch (crtc->pipe) { - uint32_t dsparb, dsparb2, dsparb3; + u32 dsparb, dsparb2, dsparb3; case PIPE_A: dsparb = I915_READ_FW(DSPARB); dsparb2 = I915_READ_FW(DSPARB2); @@ -2032,10 +2034,9 @@ static void vlv_atomic_update_fifo(struct intel_atomic_state *state, #undef VLV_FIFO -static int vlv_compute_intermediate_wm(struct drm_device *dev, - struct intel_crtc *crtc, - struct intel_crtc_state *new_crtc_state) +static int vlv_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state) { + struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc); struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate; const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal; struct intel_atomic_state *intel_state = @@ -2264,8 +2265,8 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc) { struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev); const struct intel_watermark_params *wm_info; - uint32_t fwater_lo; - uint32_t fwater_hi; + u32 fwater_lo; + u32 fwater_hi; int cwm, srwm = 1; int fifo_size; int planea_wm, planeb_wm; @@ -2273,7 +2274,7 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc) if (IS_I945GM(dev_priv)) wm_info = &i945_wm_info; - else if (!IS_GEN2(dev_priv)) + else if (!IS_GEN(dev_priv, 2)) wm_info = &i915_wm_info; else wm_info = &i830_a_wm_info; @@ -2287,7 +2288,7 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc) crtc->base.primary->state->fb; int cpp; - if (IS_GEN2(dev_priv)) + if (IS_GEN(dev_priv, 2)) cpp = 4; else cpp = fb->format->cpp[0]; @@ -2302,7 +2303,7 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc) planea_wm = wm_info->max_wm; } - if (IS_GEN2(dev_priv)) + if (IS_GEN(dev_priv, 2)) wm_info = &i830_bc_wm_info; fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B); @@ -2314,7 +2315,7 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc) crtc->base.primary->state->fb; int cpp; - if (IS_GEN2(dev_priv)) + if (IS_GEN(dev_priv, 2)) cpp = 4; else cpp = fb->format->cpp[0]; @@ -2408,7 +2409,7 @@ static void i845_update_wm(struct intel_crtc *unused_crtc) struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev); struct intel_crtc *crtc; const struct drm_display_mode *adjusted_mode; - uint32_t fwater_lo; + u32 fwater_lo; int planea_wm; crtc = single_enabled_crtc(dev_priv); @@ -2457,8 +2458,7 @@ static unsigned int ilk_wm_method2(unsigned int pixel_rate, return ret; } -static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels, - uint8_t cpp) +static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp) { /* * Neither of these should be possible since this function shouldn't be @@ -2475,22 +2475,21 @@ static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels, } struct ilk_wm_maximums { - uint16_t pri; - uint16_t spr; - uint16_t cur; - uint16_t fbc; + u16 pri; + u16 spr; + u16 cur; + u16 fbc; }; /* * For both WM_PIPE and WM_LP. * mem_value must be in 0.1us units. */ -static uint32_t ilk_compute_pri_wm(const struct intel_crtc_state *cstate, - const struct intel_plane_state *pstate, - uint32_t mem_value, - bool is_lp) +static u32 ilk_compute_pri_wm(const struct intel_crtc_state *cstate, + const struct intel_plane_state *pstate, + u32 mem_value, bool is_lp) { - uint32_t method1, method2; + u32 method1, method2; int cpp; if (mem_value == 0) @@ -2518,11 +2517,11 @@ static uint32_t ilk_compute_pri_wm(const struct intel_crtc_state *cstate, * For both WM_PIPE and WM_LP. * mem_value must be in 0.1us units. */ -static uint32_t ilk_compute_spr_wm(const struct intel_crtc_state *cstate, - const struct intel_plane_state *pstate, - uint32_t mem_value) +static u32 ilk_compute_spr_wm(const struct intel_crtc_state *cstate, + const struct intel_plane_state *pstate, + u32 mem_value) { - uint32_t method1, method2; + u32 method1, method2; int cpp; if (mem_value == 0) @@ -2545,9 +2544,9 @@ static uint32_t ilk_compute_spr_wm(const struct intel_crtc_state *cstate, * For both WM_PIPE and WM_LP. * mem_value must be in 0.1us units. */ -static uint32_t ilk_compute_cur_wm(const struct intel_crtc_state *cstate, - const struct intel_plane_state *pstate, - uint32_t mem_value) +static u32 ilk_compute_cur_wm(const struct intel_crtc_state *cstate, + const struct intel_plane_state *pstate, + u32 mem_value) { int cpp; @@ -2565,9 +2564,9 @@ static uint32_t ilk_compute_cur_wm(const struct intel_crtc_state *cstate, } /* Only for WM_LP. */ -static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate, - const struct intel_plane_state *pstate, - uint32_t pri_val) +static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *cstate, + const struct intel_plane_state *pstate, + u32 pri_val) { int cpp; @@ -2626,13 +2625,12 @@ static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv) } /* Calculate the maximum primary/sprite plane watermark */ -static unsigned int ilk_plane_wm_max(const struct drm_device *dev, +static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv, int level, const struct intel_wm_config *config, enum intel_ddb_partitioning ddb_partitioning, bool is_sprite) { - struct drm_i915_private *dev_priv = to_i915(dev); unsigned int fifo_size = ilk_display_fifo_size(dev_priv); /* if sprites aren't enabled, sprites get nothing */ @@ -2668,7 +2666,7 @@ static unsigned int ilk_plane_wm_max(const struct drm_device *dev, } /* Calculate the maximum cursor plane watermark */ -static unsigned int ilk_cursor_wm_max(const struct drm_device *dev, +static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv, int level, const struct intel_wm_config *config) { @@ -2677,19 +2675,19 @@ static unsigned int ilk_cursor_wm_max(const struct drm_device *dev, return 64; /* otherwise just report max that registers can hold */ - return ilk_cursor_wm_reg_max(to_i915(dev), level); + return ilk_cursor_wm_reg_max(dev_priv, level); } -static void ilk_compute_wm_maximums(const struct drm_device *dev, +static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv, int level, const struct intel_wm_config *config, enum intel_ddb_partitioning ddb_partitioning, struct ilk_wm_maximums *max) { - max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false); - max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true); - max->cur = ilk_cursor_wm_max(dev, level, config); - max->fbc = ilk_fbc_wm_reg_max(to_i915(dev)); + max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false); + max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true); + max->cur = ilk_cursor_wm_max(dev_priv, level, config); + max->fbc = ilk_fbc_wm_reg_max(dev_priv); } static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv, @@ -2734,9 +2732,9 @@ static bool ilk_validate_wm_level(int level, DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n", level, result->cur_val, max->cur); - result->pri_val = min_t(uint32_t, result->pri_val, max->pri); - result->spr_val = min_t(uint32_t, result->spr_val, max->spr); - result->cur_val = min_t(uint32_t, result->cur_val, max->cur); + result->pri_val = min_t(u32, result->pri_val, max->pri); + result->spr_val = min_t(u32, result->spr_val, max->spr); + result->cur_val = min_t(u32, result->cur_val, max->cur); result->enable = true; } @@ -2752,9 +2750,9 @@ static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv, const struct intel_plane_state *curstate, struct intel_wm_level *result) { - uint16_t pri_latency = dev_priv->wm.pri_latency[level]; - uint16_t spr_latency = dev_priv->wm.spr_latency[level]; - uint16_t cur_latency = dev_priv->wm.cur_latency[level]; + u16 pri_latency = dev_priv->wm.pri_latency[level]; + u16 spr_latency = dev_priv->wm.spr_latency[level]; + u16 cur_latency = dev_priv->wm.cur_latency[level]; /* WM1+ latency values stored in 0.5us units */ if (level > 0) { @@ -2778,7 +2776,7 @@ static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv, result->enable = true; } -static uint32_t +static u32 hsw_compute_linetime_wm(const struct intel_crtc_state *cstate) { const struct intel_atomic_state *intel_state = @@ -2807,10 +2805,10 @@ hsw_compute_linetime_wm(const struct intel_crtc_state *cstate) } static void intel_read_wm_latency(struct drm_i915_private *dev_priv, - uint16_t wm[8]) + u16 wm[8]) { if (INTEL_GEN(dev_priv) >= 9) { - uint32_t val; + u32 val; int ret, i; int level, max_level = ilk_wm_max_level(dev_priv); @@ -2894,7 +2892,7 @@ static void intel_read_wm_latency(struct drm_i915_private *dev_priv, wm[0] += 1; } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { - uint64_t sskpd = I915_READ64(MCH_SSKPD); + u64 sskpd = I915_READ64(MCH_SSKPD); wm[0] = (sskpd >> 56) & 0xFF; if (wm[0] == 0) @@ -2904,14 +2902,14 @@ static void intel_read_wm_latency(struct drm_i915_private *dev_priv, wm[3] = (sskpd >> 20) & 0x1FF; wm[4] = (sskpd >> 32) & 0x1FF; } else if (INTEL_GEN(dev_priv) >= 6) { - uint32_t sskpd = I915_READ(MCH_SSKPD); + u32 sskpd = I915_READ(MCH_SSKPD); wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK; wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK; wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK; wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK; } else if (INTEL_GEN(dev_priv) >= 5) { - uint32_t mltr = I915_READ(MLTR_ILK); + u32 mltr = I915_READ(MLTR_ILK); /* ILK primary LP0 latency is 700 ns */ wm[0] = 7; @@ -2923,18 +2921,18 @@ static void intel_read_wm_latency(struct drm_i915_private *dev_priv, } static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv, - uint16_t wm[5]) + u16 wm[5]) { /* ILK sprite LP0 latency is 1300 ns */ - if (IS_GEN5(dev_priv)) + if (IS_GEN(dev_priv, 5)) wm[0] = 13; } static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv, - uint16_t wm[5]) + u16 wm[5]) { /* ILK cursor LP0 latency is 1300 ns */ - if (IS_GEN5(dev_priv)) + if (IS_GEN(dev_priv, 5)) wm[0] = 13; } @@ -2953,7 +2951,7 @@ int ilk_wm_max_level(const struct drm_i915_private *dev_priv) static void intel_print_wm_latency(struct drm_i915_private *dev_priv, const char *name, - const uint16_t wm[8]) + const u16 wm[8]) { int level, max_level = ilk_wm_max_level(dev_priv); @@ -2982,7 +2980,7 @@ static void intel_print_wm_latency(struct drm_i915_private *dev_priv, } static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv, - uint16_t wm[5], uint16_t min) + u16 wm[5], u16 min) { int level, max_level = ilk_wm_max_level(dev_priv); @@ -2991,7 +2989,7 @@ static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv, wm[0] = max(wm[0], min); for (level = 1; level <= max_level; level++) - wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5)); + wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5)); return true; } @@ -3061,7 +3059,7 @@ static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv) intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency); intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency); - if (IS_GEN6(dev_priv)) { + if (IS_GEN(dev_priv, 6)) { snb_wm_latency_quirk(dev_priv); snb_wm_lp3_irq_quirk(dev_priv); } @@ -3073,7 +3071,7 @@ static void skl_setup_wm_latency(struct drm_i915_private *dev_priv) intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency); } -static bool ilk_validate_pipe_wm(struct drm_device *dev, +static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv, struct intel_pipe_wm *pipe_wm) { /* LP0 watermark maximums depend on this pipe alone */ @@ -3085,7 +3083,7 @@ static bool ilk_validate_pipe_wm(struct drm_device *dev, struct ilk_wm_maximums max; /* LP0 watermarks always use 1/2 DDB partitioning */ - ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max); + ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max); /* At least LP0 must be valid */ if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) { @@ -3150,7 +3148,7 @@ static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate) if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) pipe_wm->linetime = hsw_compute_linetime_wm(cstate); - if (!ilk_validate_pipe_wm(dev, pipe_wm)) + if (!ilk_validate_pipe_wm(dev_priv, pipe_wm)) return -EINVAL; ilk_compute_wm_reg_maximums(dev_priv, 1, &max); @@ -3180,17 +3178,17 @@ static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate) * state and the new state. These can be programmed to the hardware * immediately. */ -static int ilk_compute_intermediate_wm(struct drm_device *dev, - struct intel_crtc *intel_crtc, - struct intel_crtc_state *newstate) +static int ilk_compute_intermediate_wm(struct intel_crtc_state *newstate) { + struct intel_crtc *intel_crtc = to_intel_crtc(newstate->base.crtc); + struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev); struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate; struct intel_atomic_state *intel_state = to_intel_atomic_state(newstate->base.state); const struct intel_crtc_state *oldstate = intel_atomic_get_old_crtc_state(intel_state, intel_crtc); const struct intel_pipe_wm *b = &oldstate->wm.ilk.optimal; - int level, max_level = ilk_wm_max_level(to_i915(dev)); + int level, max_level = ilk_wm_max_level(dev_priv); /* * Start with the final, target watermarks, then combine with the @@ -3198,7 +3196,8 @@ static int ilk_compute_intermediate_wm(struct drm_device *dev, * and after the vblank. */ *a = newstate->wm.ilk.optimal; - if (!newstate->base.active || drm_atomic_crtc_needs_modeset(&newstate->base)) + if (!newstate->base.active || drm_atomic_crtc_needs_modeset(&newstate->base) || + intel_state->skip_intermediate_wm) return 0; a->pipe_enabled |= b->pipe_enabled; @@ -3222,7 +3221,7 @@ static int ilk_compute_intermediate_wm(struct drm_device *dev, * there's no safe way to transition from the old state to * the new state, so we need to fail the atomic transaction. */ - if (!ilk_validate_pipe_wm(dev, a)) + if (!ilk_validate_pipe_wm(dev_priv, a)) return -EINVAL; /* @@ -3238,7 +3237,7 @@ static int ilk_compute_intermediate_wm(struct drm_device *dev, /* * Merge the watermarks from all active pipes for a specific level. */ -static void ilk_merge_wm_level(struct drm_device *dev, +static void ilk_merge_wm_level(struct drm_i915_private *dev_priv, int level, struct intel_wm_level *ret_wm) { @@ -3246,7 +3245,7 @@ static void ilk_merge_wm_level(struct drm_device *dev, ret_wm->enable = true; - for_each_intel_crtc(dev, intel_crtc) { + for_each_intel_crtc(&dev_priv->drm, intel_crtc) { const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk; const struct intel_wm_level *wm = &active->wm[level]; @@ -3271,12 +3270,11 @@ static void ilk_merge_wm_level(struct drm_device *dev, /* * Merge all low power watermarks for all active pipes. */ -static void ilk_wm_merge(struct drm_device *dev, +static void ilk_wm_merge(struct drm_i915_private *dev_priv, const struct intel_wm_config *config, const struct ilk_wm_maximums *max, struct intel_pipe_wm *merged) { - struct drm_i915_private *dev_priv = to_i915(dev); int level, max_level = ilk_wm_max_level(dev_priv); int last_enabled_level = max_level; @@ -3292,7 +3290,7 @@ static void ilk_wm_merge(struct drm_device *dev, for (level = 1; level <= max_level; level++) { struct intel_wm_level *wm = &merged->wm[level]; - ilk_merge_wm_level(dev, level, wm); + ilk_merge_wm_level(dev_priv, level, wm); if (level > last_enabled_level) wm->enable = false; @@ -3317,7 +3315,7 @@ static void ilk_wm_merge(struct drm_device *dev, * What we should check here is whether FBC can be * enabled sometime later. */ - if (IS_GEN5(dev_priv) && !merged->fbc_wm_enabled && + if (IS_GEN(dev_priv, 5) && !merged->fbc_wm_enabled && intel_fbc_is_active(dev_priv)) { for (level = 2; level <= max_level; level++) { struct intel_wm_level *wm = &merged->wm[level]; @@ -3334,22 +3332,20 @@ static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm) } /* The value we need to program into the WM_LPx latency field */ -static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level) +static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv, + int level) { - struct drm_i915_private *dev_priv = to_i915(dev); - if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) return 2 * level; else return dev_priv->wm.pri_latency[level]; } -static void ilk_compute_wm_results(struct drm_device *dev, +static void ilk_compute_wm_results(struct drm_i915_private *dev_priv, const struct intel_pipe_wm *merged, enum intel_ddb_partitioning partitioning, struct ilk_wm_values *results) { - struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc; int level, wm_lp; @@ -3369,7 +3365,7 @@ static void ilk_compute_wm_results(struct drm_device *dev, * disabled. Doing otherwise could cause underruns. */ results->wm_lp[wm_lp - 1] = - (ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) | + (ilk_wm_lp_latency(dev_priv, level) << WM1_LP_LATENCY_SHIFT) | (r->pri_val << WM1_LP_SR_SHIFT) | r->cur_val; @@ -3395,7 +3391,7 @@ static void ilk_compute_wm_results(struct drm_device *dev, } /* LP0 register values */ - for_each_intel_crtc(dev, intel_crtc) { + for_each_intel_crtc(&dev_priv->drm, intel_crtc) { enum pipe pipe = intel_crtc->pipe; const struct intel_wm_level *r = &intel_crtc->wm.active.ilk.wm[0]; @@ -3414,11 +3410,12 @@ static void ilk_compute_wm_results(struct drm_device *dev, /* Find the result with the highest level enabled. Check for enable_fbc_wm in * case both are at the same level. Prefer r1 in case they're the same. */ -static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev, - struct intel_pipe_wm *r1, - struct intel_pipe_wm *r2) +static struct intel_pipe_wm * +ilk_find_best_result(struct drm_i915_private *dev_priv, + struct intel_pipe_wm *r1, + struct intel_pipe_wm *r2) { - int level, max_level = ilk_wm_max_level(to_i915(dev)); + int level, max_level = ilk_wm_max_level(dev_priv); int level1 = 0, level2 = 0; for (level = 1; level <= max_level; level++) { @@ -3539,7 +3536,7 @@ static void ilk_write_wm_values(struct drm_i915_private *dev_priv, { struct ilk_wm_values *previous = &dev_priv->wm.hw; unsigned int dirty; - uint32_t val; + u32 val; dirty = ilk_compute_wm_dirty(dev_priv, previous, results); if (!dirty) @@ -3637,28 +3634,16 @@ static u8 intel_enabled_dbuf_slices_num(struct drm_i915_private *dev_priv) * FIXME: We still don't have the proper code detect if we need to apply the WA, * so assume we'll always need it in order to avoid underruns. */ -static bool skl_needs_memory_bw_wa(struct intel_atomic_state *state) +static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - - if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv)) - return true; - - return false; + return IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv); } static bool intel_has_sagv(struct drm_i915_private *dev_priv) { - if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv) || - IS_CANNONLAKE(dev_priv)) - return true; - - if (IS_SKYLAKE(dev_priv) && - dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED) - return true; - - return false; + return (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) && + dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED; } /* @@ -3683,25 +3668,25 @@ intel_enable_sagv(struct drm_i915_private *dev_priv) if (dev_priv->sagv_status == I915_SAGV_ENABLED) return 0; - DRM_DEBUG_KMS("Enabling the SAGV\n"); + DRM_DEBUG_KMS("Enabling SAGV\n"); mutex_lock(&dev_priv->pcu_lock); ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL, GEN9_SAGV_ENABLE); - /* We don't need to wait for the SAGV when enabling */ + /* We don't need to wait for SAGV when enabling */ mutex_unlock(&dev_priv->pcu_lock); /* * Some skl systems, pre-release machines in particular, - * don't actually have an SAGV. + * don't actually have SAGV. */ if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) { DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n"); dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED; return 0; } else if (ret < 0) { - DRM_ERROR("Failed to enable the SAGV\n"); + DRM_ERROR("Failed to enable SAGV\n"); return ret; } @@ -3720,7 +3705,7 @@ intel_disable_sagv(struct drm_i915_private *dev_priv) if (dev_priv->sagv_status == I915_SAGV_DISABLED) return 0; - DRM_DEBUG_KMS("Disabling the SAGV\n"); + DRM_DEBUG_KMS("Disabling SAGV\n"); mutex_lock(&dev_priv->pcu_lock); /* bspec says to keep retrying for at least 1 ms */ @@ -3732,14 +3717,14 @@ intel_disable_sagv(struct drm_i915_private *dev_priv) /* * Some skl systems, pre-release machines in particular, - * don't actually have an SAGV. + * don't actually have SAGV. */ if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) { DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n"); dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED; return 0; } else if (ret < 0) { - DRM_ERROR("Failed to disable the SAGV (%d)\n", ret); + DRM_ERROR("Failed to disable SAGV (%d)\n", ret); return ret; } @@ -3762,15 +3747,15 @@ bool intel_can_enable_sagv(struct drm_atomic_state *state) if (!intel_has_sagv(dev_priv)) return false; - if (IS_GEN9(dev_priv)) + if (IS_GEN(dev_priv, 9)) sagv_block_time_us = 30; - else if (IS_GEN10(dev_priv)) + else if (IS_GEN(dev_priv, 10)) sagv_block_time_us = 20; else sagv_block_time_us = 10; /* - * SKL+ workaround: bspec recommends we disable the SAGV when we have + * SKL+ workaround: bspec recommends we disable SAGV when we have * more then one pipe enabled * * If there are no active CRTCs, no additional checks need be performed @@ -3803,7 +3788,7 @@ bool intel_can_enable_sagv(struct drm_atomic_state *state) latency = dev_priv->wm.skl_latency[level]; - if (skl_needs_memory_bw_wa(intel_state) && + if (skl_needs_memory_bw_wa(dev_priv) && plane->base.state->fb->modifier == I915_FORMAT_MOD_X_TILED) latency += 15; @@ -3811,7 +3796,7 @@ bool intel_can_enable_sagv(struct drm_atomic_state *state) /* * If any of the planes on this pipe don't enable wm levels that * incur memory latencies higher than sagv_block_time_us we - * can't enable the SAGV. + * can't enable SAGV. */ if (latency < sagv_block_time_us) return false; @@ -3822,7 +3807,7 @@ bool intel_can_enable_sagv(struct drm_atomic_state *state) static u16 intel_get_ddb_size(struct drm_i915_private *dev_priv, const struct intel_crtc_state *cstate, - const unsigned int total_data_rate, + const u64 total_data_rate, const int num_active, struct skl_ddb_allocation *ddb) { @@ -3836,12 +3821,17 @@ static u16 intel_get_ddb_size(struct drm_i915_private *dev_priv, return ddb_size - 4; /* 4 blocks for bypass path allocation */ adjusted_mode = &cstate->base.adjusted_mode; - total_data_bw = (u64)total_data_rate * drm_mode_vrefresh(adjusted_mode); + total_data_bw = total_data_rate * drm_mode_vrefresh(adjusted_mode); /* * 12GB/s is maximum BW supported by single DBuf slice. + * + * FIXME dbuf slice code is broken: + * - must wait for planes to stop using the slice before powering it off + * - plane straddling both slices is illegal in multi-pipe scenarios + * - should validate we stay within the hw bandwidth limits */ - if (total_data_bw >= GBps(12) || num_active > 1) { + if (0 && (num_active > 1 || total_data_bw >= GBps(12))) { ddb->enabled_slices = 2; } else { ddb->enabled_slices = 1; @@ -3852,16 +3842,15 @@ static u16 intel_get_ddb_size(struct drm_i915_private *dev_priv, } static void -skl_ddb_get_pipe_allocation_limits(struct drm_device *dev, +skl_ddb_get_pipe_allocation_limits(struct drm_i915_private *dev_priv, const struct intel_crtc_state *cstate, - const unsigned int total_data_rate, + const u64 total_data_rate, struct skl_ddb_allocation *ddb, struct skl_ddb_entry *alloc, /* out */ int *num_active /* out */) { struct drm_atomic_state *state = cstate->base.state; struct intel_atomic_state *intel_state = to_intel_atomic_state(state); - struct drm_i915_private *dev_priv = to_i915(dev); struct drm_crtc *for_crtc = cstate->base.crtc; const struct drm_crtc_state *crtc_state; const struct drm_crtc *crtc; @@ -3941,14 +3930,9 @@ static unsigned int skl_cursor_allocation(int num_active) static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv, struct skl_ddb_entry *entry, u32 reg) { - u16 mask; - if (INTEL_GEN(dev_priv) >= 11) - mask = ICL_DDB_ENTRY_MASK; - else - mask = SKL_DDB_ENTRY_MASK; - entry->start = reg & mask; - entry->end = (reg >> DDB_ENTRY_END_SHIFT) & mask; + entry->start = reg & DDB_ENTRY_MASK; + entry->end = (reg >> DDB_ENTRY_END_SHIFT) & DDB_ENTRY_MASK; if (entry->end) entry->end += 1; @@ -3958,73 +3942,70 @@ static void skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv, const enum pipe pipe, const enum plane_id plane_id, - struct skl_ddb_allocation *ddb /* out */) + struct skl_ddb_entry *ddb_y, + struct skl_ddb_entry *ddb_uv) { - u32 val, val2 = 0; - int fourcc, pixel_format; + u32 val, val2; + u32 fourcc = 0; /* Cursor doesn't support NV12/planar, so no extra calculation needed */ if (plane_id == PLANE_CURSOR) { val = I915_READ(CUR_BUF_CFG(pipe)); - skl_ddb_entry_init_from_hw(dev_priv, - &ddb->plane[pipe][plane_id], val); + skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val); return; } val = I915_READ(PLANE_CTL(pipe, plane_id)); /* No DDB allocated for disabled planes */ - if (!(val & PLANE_CTL_ENABLE)) - return; + if (val & PLANE_CTL_ENABLE) + fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK, + val & PLANE_CTL_ORDER_RGBX, + val & PLANE_CTL_ALPHA_MASK); - pixel_format = val & PLANE_CTL_FORMAT_MASK; - fourcc = skl_format_to_fourcc(pixel_format, - val & PLANE_CTL_ORDER_RGBX, - val & PLANE_CTL_ALPHA_MASK); - - val = I915_READ(PLANE_BUF_CFG(pipe, plane_id)); - /* - * FIXME: add proper NV12 support for ICL. Avoid reading unclaimed - * registers for now. - */ - if (INTEL_GEN(dev_priv) < 11) + if (INTEL_GEN(dev_priv) >= 11) { + val = I915_READ(PLANE_BUF_CFG(pipe, plane_id)); + skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val); + } else { + val = I915_READ(PLANE_BUF_CFG(pipe, plane_id)); val2 = I915_READ(PLANE_NV12_BUF_CFG(pipe, plane_id)); - if (fourcc == DRM_FORMAT_NV12) { - skl_ddb_entry_init_from_hw(dev_priv, - &ddb->plane[pipe][plane_id], val2); - skl_ddb_entry_init_from_hw(dev_priv, - &ddb->uv_plane[pipe][plane_id], val); - } else { - skl_ddb_entry_init_from_hw(dev_priv, - &ddb->plane[pipe][plane_id], val); + if (fourcc == DRM_FORMAT_NV12) + swap(val, val2); + + skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val); + skl_ddb_entry_init_from_hw(dev_priv, ddb_uv, val2); } } -void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv, - struct skl_ddb_allocation *ddb /* out */) +void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc, + struct skl_ddb_entry *ddb_y, + struct skl_ddb_entry *ddb_uv) { - struct intel_crtc *crtc; - - memset(ddb, 0, sizeof(*ddb)); - - ddb->enabled_slices = intel_enabled_dbuf_slices_num(dev_priv); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum intel_display_power_domain power_domain; + enum pipe pipe = crtc->pipe; + intel_wakeref_t wakeref; + enum plane_id plane_id; - for_each_intel_crtc(&dev_priv->drm, crtc) { - enum intel_display_power_domain power_domain; - enum plane_id plane_id; - enum pipe pipe = crtc->pipe; + power_domain = POWER_DOMAIN_PIPE(pipe); + wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); + if (!wakeref) + return; - power_domain = POWER_DOMAIN_PIPE(pipe); - if (!intel_display_power_get_if_enabled(dev_priv, power_domain)) - continue; + for_each_plane_id_on_crtc(crtc, plane_id) + skl_ddb_get_hw_plane_state(dev_priv, pipe, + plane_id, + &ddb_y[plane_id], + &ddb_uv[plane_id]); - for_each_plane_id_on_crtc(crtc, plane_id) - skl_ddb_get_hw_plane_state(dev_priv, pipe, - plane_id, ddb); + intel_display_power_put(dev_priv, power_domain, wakeref); +} - intel_display_power_put(dev_priv, power_domain); - } +void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv, + struct skl_ddb_allocation *ddb /* out */) +{ + ddb->enabled_slices = intel_enabled_dbuf_slices_num(dev_priv); } /* @@ -4048,7 +4029,7 @@ skl_plane_downscale_amount(const struct intel_crtc_state *cstate, const struct intel_plane_state *pstate) { struct intel_plane *plane = to_intel_plane(pstate->base.plane); - uint32_t src_w, src_h, dst_w, dst_h; + u32 src_w, src_h, dst_w, dst_h; uint_fixed_16_16_t fp_w_ratio, fp_h_ratio; uint_fixed_16_16_t downscale_h, downscale_w; @@ -4094,8 +4075,8 @@ skl_pipe_downscale_amount(const struct intel_crtc_state *crtc_state) return pipe_downscale; if (crtc_state->pch_pfit.enabled) { - uint32_t src_w, src_h, dst_w, dst_h; - uint32_t pfit_size = crtc_state->pch_pfit.size; + u32 src_w, src_h, dst_w, dst_h; + u32 pfit_size = crtc_state->pch_pfit.size; uint_fixed_16_16_t fp_w_ratio, fp_h_ratio; uint_fixed_16_16_t downscale_h, downscale_w; @@ -4128,7 +4109,7 @@ int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc, const struct drm_plane_state *pstate; struct intel_plane_state *intel_pstate; int crtc_clock, dotclk; - uint32_t pipe_max_pixel_rate; + u32 pipe_max_pixel_rate; uint_fixed_16_16_t pipe_downscale; uint_fixed_16_16_t max_downscale = u32_to_fixed16(1); @@ -4177,23 +4158,24 @@ int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc, return 0; } -static unsigned int +static u64 skl_plane_relative_data_rate(const struct intel_crtc_state *cstate, - const struct drm_plane_state *pstate, + const struct intel_plane_state *intel_pstate, const int plane) { - struct intel_plane *intel_plane = to_intel_plane(pstate->plane); - struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate); - uint32_t data_rate; - uint32_t width = 0, height = 0; + struct intel_plane *intel_plane = + to_intel_plane(intel_pstate->base.plane); + u32 data_rate; + u32 width = 0, height = 0; struct drm_framebuffer *fb; u32 format; uint_fixed_16_16_t down_scale_amount; + u64 rate; if (!intel_pstate->base.visible) return 0; - fb = pstate->fb; + fb = intel_pstate->base.fb; format = fb->format->format; if (intel_plane->id == PLANE_CURSOR) @@ -4215,28 +4197,26 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *cstate, height /= 2; } - data_rate = width * height * fb->format->cpp[plane]; + data_rate = width * height; down_scale_amount = skl_plane_downscale_amount(cstate, intel_pstate); - return mul_round_up_u32_fixed16(data_rate, down_scale_amount); + rate = mul_round_up_u32_fixed16(data_rate, down_scale_amount); + + rate *= fb->format->cpp[plane]; + return rate; } -/* - * We don't overflow 32 bits. Worst case is 3 planes enabled, each fetching - * a 8192x4096@32bpp framebuffer: - * 3 * 4096 * 8192 * 4 < 2^32 - */ -static unsigned int +static u64 skl_get_total_relative_data_rate(struct intel_crtc_state *intel_cstate, - unsigned int *plane_data_rate, - unsigned int *uv_plane_data_rate) + u64 *plane_data_rate, + u64 *uv_plane_data_rate) { struct drm_crtc_state *cstate = &intel_cstate->base; struct drm_atomic_state *state = cstate->state; struct drm_plane *plane; const struct drm_plane_state *pstate; - unsigned int total_data_rate = 0; + u64 total_data_rate = 0; if (WARN_ON(!state)) return 0; @@ -4244,110 +4224,79 @@ skl_get_total_relative_data_rate(struct intel_crtc_state *intel_cstate, /* Calculate and cache data rate for each plane */ drm_atomic_crtc_state_for_each_plane_state(plane, pstate, cstate) { enum plane_id plane_id = to_intel_plane(plane)->id; - unsigned int rate; + u64 rate; + const struct intel_plane_state *intel_pstate = + to_intel_plane_state(pstate); /* packed/y */ rate = skl_plane_relative_data_rate(intel_cstate, - pstate, 0); + intel_pstate, 0); plane_data_rate[plane_id] = rate; - total_data_rate += rate; /* uv-plane */ rate = skl_plane_relative_data_rate(intel_cstate, - pstate, 1); + intel_pstate, 1); uv_plane_data_rate[plane_id] = rate; - total_data_rate += rate; } return total_data_rate; } -static uint16_t -skl_ddb_min_alloc(const struct drm_plane_state *pstate, const int plane) +static u64 +icl_get_total_relative_data_rate(struct intel_crtc_state *intel_cstate, + u64 *plane_data_rate) { - struct drm_framebuffer *fb = pstate->fb; - struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate); - uint32_t src_w, src_h; - uint32_t min_scanlines = 8; - uint8_t plane_bpp; - - if (WARN_ON(!fb)) - return 0; + struct drm_crtc_state *cstate = &intel_cstate->base; + struct drm_atomic_state *state = cstate->state; + struct drm_plane *plane; + const struct drm_plane_state *pstate; + u64 total_data_rate = 0; - /* For packed formats, and uv-plane, return 0 */ - if (plane == 1 && fb->format->format != DRM_FORMAT_NV12) + if (WARN_ON(!state)) return 0; - /* For Non Y-tile return 8-blocks */ - if (fb->modifier != I915_FORMAT_MOD_Y_TILED && - fb->modifier != I915_FORMAT_MOD_Yf_TILED && - fb->modifier != I915_FORMAT_MOD_Y_TILED_CCS && - fb->modifier != I915_FORMAT_MOD_Yf_TILED_CCS) - return 8; - - /* - * Src coordinates are already rotated by 270 degrees for - * the 90/270 degree plane rotation cases (to match the - * GTT mapping), hence no need to account for rotation here. - */ - src_w = drm_rect_width(&intel_pstate->base.src) >> 16; - src_h = drm_rect_height(&intel_pstate->base.src) >> 16; - - /* Halve UV plane width and height for NV12 */ - if (plane == 1) { - src_w /= 2; - src_h /= 2; - } - - plane_bpp = fb->format->cpp[plane]; - - if (drm_rotation_90_or_270(pstate->rotation)) { - switch (plane_bpp) { - case 1: - min_scanlines = 32; - break; - case 2: - min_scanlines = 16; - break; - case 4: - min_scanlines = 8; - break; - case 8: - min_scanlines = 4; - break; - default: - WARN(1, "Unsupported pixel depth %u for rotation", - plane_bpp); - min_scanlines = 32; - } - } - - return DIV_ROUND_UP((4 * src_w * plane_bpp), 512) * min_scanlines/4 + 3; -} - -static void -skl_ddb_calc_min(const struct intel_crtc_state *cstate, int num_active, - uint16_t *minimum, uint16_t *uv_minimum) -{ - const struct drm_plane_state *pstate; - struct drm_plane *plane; - - drm_atomic_crtc_state_for_each_plane_state(plane, pstate, &cstate->base) { + /* Calculate and cache data rate for each plane */ + drm_atomic_crtc_state_for_each_plane_state(plane, pstate, cstate) { + const struct intel_plane_state *intel_pstate = + to_intel_plane_state(pstate); enum plane_id plane_id = to_intel_plane(plane)->id; + u64 rate; - if (plane_id == PLANE_CURSOR) - continue; + if (!intel_pstate->linked_plane) { + rate = skl_plane_relative_data_rate(intel_cstate, + intel_pstate, 0); + plane_data_rate[plane_id] = rate; + total_data_rate += rate; + } else { + enum plane_id y_plane_id; - if (!pstate->visible) - continue; + /* + * The slave plane might not iterate in + * drm_atomic_crtc_state_for_each_plane_state(), + * and needs the master plane state which may be + * NULL if we try get_new_plane_state(), so we + * always calculate from the master. + */ + if (intel_pstate->slave) + continue; - minimum[plane_id] = skl_ddb_min_alloc(pstate, 0); - uv_minimum[plane_id] = skl_ddb_min_alloc(pstate, 1); + /* Y plane rate is calculated on the slave */ + rate = skl_plane_relative_data_rate(intel_cstate, + intel_pstate, 0); + y_plane_id = intel_pstate->linked_plane->id; + plane_data_rate[y_plane_id] = rate; + total_data_rate += rate; + + rate = skl_plane_relative_data_rate(intel_cstate, + intel_pstate, 1); + plane_data_rate[plane_id] = rate; + total_data_rate += rate; + } } - minimum[PLANE_CURSOR] = skl_cursor_allocation(num_active); + return total_data_rate; } static int @@ -4356,23 +4305,24 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate, { struct drm_atomic_state *state = cstate->base.state; struct drm_crtc *crtc = cstate->base.crtc; - struct drm_device *dev = crtc->dev; + struct drm_i915_private *dev_priv = to_i915(crtc->dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - enum pipe pipe = intel_crtc->pipe; struct skl_ddb_entry *alloc = &cstate->wm.skl.ddb; - uint16_t alloc_size, start; - uint16_t minimum[I915_MAX_PLANES] = {}; - uint16_t uv_minimum[I915_MAX_PLANES] = {}; - unsigned int total_data_rate; + struct skl_plane_wm *wm; + u16 alloc_size, start = 0; + u16 total[I915_MAX_PLANES] = {}; + u16 uv_total[I915_MAX_PLANES] = {}; + u64 total_data_rate; enum plane_id plane_id; int num_active; - unsigned int plane_data_rate[I915_MAX_PLANES] = {}; - unsigned int uv_plane_data_rate[I915_MAX_PLANES] = {}; - uint16_t total_min_blocks = 0; + u64 plane_data_rate[I915_MAX_PLANES] = {}; + u64 uv_plane_data_rate[I915_MAX_PLANES] = {}; + u32 blocks; + int level; /* Clear the partitioning for disabled planes. */ - memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe])); - memset(ddb->uv_plane[pipe], 0, sizeof(ddb->uv_plane[pipe])); + memset(cstate->wm.skl.plane_ddb_y, 0, sizeof(cstate->wm.skl.plane_ddb_y)); + memset(cstate->wm.skl.plane_ddb_uv, 0, sizeof(cstate->wm.skl.plane_ddb_uv)); if (WARN_ON(!state)) return 0; @@ -4382,89 +4332,151 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate, return 0; } - total_data_rate = skl_get_total_relative_data_rate(cstate, - plane_data_rate, - uv_plane_data_rate); - skl_ddb_get_pipe_allocation_limits(dev, cstate, total_data_rate, ddb, - alloc, &num_active); + if (INTEL_GEN(dev_priv) < 11) + total_data_rate = + skl_get_total_relative_data_rate(cstate, + plane_data_rate, + uv_plane_data_rate); + else + total_data_rate = + icl_get_total_relative_data_rate(cstate, + plane_data_rate); + + skl_ddb_get_pipe_allocation_limits(dev_priv, cstate, total_data_rate, + ddb, alloc, &num_active); alloc_size = skl_ddb_entry_size(alloc); if (alloc_size == 0) return 0; - skl_ddb_calc_min(cstate, num_active, minimum, uv_minimum); + /* Allocate fixed number of blocks for cursor. */ + total[PLANE_CURSOR] = skl_cursor_allocation(num_active); + alloc_size -= total[PLANE_CURSOR]; + cstate->wm.skl.plane_ddb_y[PLANE_CURSOR].start = + alloc->end - total[PLANE_CURSOR]; + cstate->wm.skl.plane_ddb_y[PLANE_CURSOR].end = alloc->end; + + if (total_data_rate == 0) + return 0; /* - * 1. Allocate the mininum required blocks for each active plane - * and allocate the cursor, it doesn't require extra allocation - * proportional to the data rate. + * Find the highest watermark level for which we can satisfy the block + * requirement of active planes. */ + for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) { + blocks = 0; + for_each_plane_id_on_crtc(intel_crtc, plane_id) { + if (plane_id == PLANE_CURSOR) + continue; - for_each_plane_id_on_crtc(intel_crtc, plane_id) { - total_min_blocks += minimum[plane_id]; - total_min_blocks += uv_minimum[plane_id]; + wm = &cstate->wm.skl.optimal.planes[plane_id]; + blocks += wm->wm[level].min_ddb_alloc; + blocks += wm->uv_wm[level].min_ddb_alloc; + } + + if (blocks < alloc_size) { + alloc_size -= blocks; + break; + } } - if (total_min_blocks > alloc_size) { + if (level < 0) { DRM_DEBUG_KMS("Requested display configuration exceeds system DDB limitations"); - DRM_DEBUG_KMS("minimum required %d/%d\n", total_min_blocks, - alloc_size); + DRM_DEBUG_KMS("minimum required %d/%d\n", blocks, + alloc_size); return -EINVAL; } - alloc_size -= total_min_blocks; - ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - minimum[PLANE_CURSOR]; - ddb->plane[pipe][PLANE_CURSOR].end = alloc->end; - /* - * 2. Distribute the remaining space in proportion to the amount of - * data each plane needs to fetch from memory. - * - * FIXME: we may not allocate every single block here. + * Grant each plane the blocks it requires at the highest achievable + * watermark level, plus an extra share of the leftover blocks + * proportional to its relative data rate. */ - if (total_data_rate == 0) - return 0; + for_each_plane_id_on_crtc(intel_crtc, plane_id) { + u64 rate; + u16 extra; + + if (plane_id == PLANE_CURSOR) + continue; + + /* + * We've accounted for all active planes; remaining planes are + * all disabled. + */ + if (total_data_rate == 0) + break; + wm = &cstate->wm.skl.optimal.planes[plane_id]; + + rate = plane_data_rate[plane_id]; + extra = min_t(u16, alloc_size, + DIV64_U64_ROUND_UP(alloc_size * rate, + total_data_rate)); + total[plane_id] = wm->wm[level].min_ddb_alloc + extra; + alloc_size -= extra; + total_data_rate -= rate; + + if (total_data_rate == 0) + break; + + rate = uv_plane_data_rate[plane_id]; + extra = min_t(u16, alloc_size, + DIV64_U64_ROUND_UP(alloc_size * rate, + total_data_rate)); + uv_total[plane_id] = wm->uv_wm[level].min_ddb_alloc + extra; + alloc_size -= extra; + total_data_rate -= rate; + } + WARN_ON(alloc_size != 0 || total_data_rate != 0); + + /* Set the actual DDB start/end points for each plane */ start = alloc->start; for_each_plane_id_on_crtc(intel_crtc, plane_id) { - unsigned int data_rate, uv_data_rate; - uint16_t plane_blocks, uv_plane_blocks; + struct skl_ddb_entry *plane_alloc, *uv_plane_alloc; if (plane_id == PLANE_CURSOR) continue; - data_rate = plane_data_rate[plane_id]; + plane_alloc = &cstate->wm.skl.plane_ddb_y[plane_id]; + uv_plane_alloc = &cstate->wm.skl.plane_ddb_uv[plane_id]; - /* - * allocation for (packed formats) or (uv-plane part of planar format): - * promote the expression to 64 bits to avoid overflowing, the - * result is < available as data_rate / total_data_rate < 1 - */ - plane_blocks = minimum[plane_id]; - plane_blocks += div_u64((uint64_t)alloc_size * data_rate, - total_data_rate); + /* Gen11+ uses a separate plane for UV watermarks */ + WARN_ON(INTEL_GEN(dev_priv) >= 11 && uv_total[plane_id]); /* Leave disabled planes at (0,0) */ - if (data_rate) { - ddb->plane[pipe][plane_id].start = start; - ddb->plane[pipe][plane_id].end = start + plane_blocks; + if (total[plane_id]) { + plane_alloc->start = start; + start += total[plane_id]; + plane_alloc->end = start; } - start += plane_blocks; - - /* Allocate DDB for UV plane for planar format/NV12 */ - uv_data_rate = uv_plane_data_rate[plane_id]; - - uv_plane_blocks = uv_minimum[plane_id]; - uv_plane_blocks += div_u64((uint64_t)alloc_size * uv_data_rate, - total_data_rate); + if (uv_total[plane_id]) { + uv_plane_alloc->start = start; + start += uv_total[plane_id]; + uv_plane_alloc->end = start; + } + } - if (uv_data_rate) { - ddb->uv_plane[pipe][plane_id].start = start; - ddb->uv_plane[pipe][plane_id].end = - start + uv_plane_blocks; + /* + * When we calculated watermark values we didn't know how high + * of a level we'd actually be able to hit, so we just marked + * all levels as "enabled." Go back now and disable the ones + * that aren't actually possible. + */ + for (level++; level <= ilk_wm_max_level(dev_priv); level++) { + for_each_plane_id_on_crtc(intel_crtc, plane_id) { + wm = &cstate->wm.skl.optimal.planes[plane_id]; + memset(&wm->wm[level], 0, sizeof(wm->wm[level])); } + } - start += uv_plane_blocks; + /* + * Go back and disable the transition watermark if it turns out we + * don't have enough DDB blocks for it. + */ + for_each_plane_id_on_crtc(intel_crtc, plane_id) { + wm = &cstate->wm.skl.optimal.planes[plane_id]; + if (wm->trans_wm.plane_res_b >= total[plane_id]) + memset(&wm->trans_wm, 0, sizeof(wm->trans_wm)); } return 0; @@ -4477,10 +4489,10 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate, * 2xcdclk is 1350 MHz and the pixel rate should never exceed that. */ static uint_fixed_16_16_t -skl_wm_method1(const struct drm_i915_private *dev_priv, uint32_t pixel_rate, - uint8_t cpp, uint32_t latency, uint32_t dbuf_block_size) +skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate, + u8 cpp, u32 latency, u32 dbuf_block_size) { - uint32_t wm_intermediate_val; + u32 wm_intermediate_val; uint_fixed_16_16_t ret; if (latency == 0) @@ -4495,12 +4507,11 @@ skl_wm_method1(const struct drm_i915_private *dev_priv, uint32_t pixel_rate, return ret; } -static uint_fixed_16_16_t skl_wm_method2(uint32_t pixel_rate, - uint32_t pipe_htotal, - uint32_t latency, - uint_fixed_16_16_t plane_blocks_per_line) +static uint_fixed_16_16_t +skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency, + uint_fixed_16_16_t plane_blocks_per_line) { - uint32_t wm_intermediate_val; + u32 wm_intermediate_val; uint_fixed_16_16_t ret; if (latency == 0) @@ -4514,10 +4525,10 @@ static uint_fixed_16_16_t skl_wm_method2(uint32_t pixel_rate, } static uint_fixed_16_16_t -intel_get_linetime_us(struct intel_crtc_state *cstate) +intel_get_linetime_us(const struct intel_crtc_state *cstate) { - uint32_t pixel_rate; - uint32_t crtc_htotal; + u32 pixel_rate; + u32 crtc_htotal; uint_fixed_16_16_t linetime_us; if (!cstate->base.active) @@ -4534,11 +4545,11 @@ intel_get_linetime_us(struct intel_crtc_state *cstate) return linetime_us; } -static uint32_t +static u32 skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *cstate, const struct intel_plane_state *pstate) { - uint64_t adjusted_pixel_rate; + u64 adjusted_pixel_rate; uint_fixed_16_16_t downscale_amount; /* Shouldn't reach here on disabled planes... */ @@ -4557,24 +4568,18 @@ skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *cstate, } static int -skl_compute_plane_wm_params(const struct drm_i915_private *dev_priv, - struct intel_crtc_state *cstate, +skl_compute_plane_wm_params(const struct intel_crtc_state *cstate, const struct intel_plane_state *intel_pstate, - struct skl_wm_params *wp, int plane_id) + struct skl_wm_params *wp, int color_plane) { struct intel_plane *plane = to_intel_plane(intel_pstate->base.plane); + struct drm_i915_private *dev_priv = to_i915(plane->base.dev); const struct drm_plane_state *pstate = &intel_pstate->base; const struct drm_framebuffer *fb = pstate->fb; - uint32_t interm_pbpl; - struct intel_atomic_state *state = - to_intel_atomic_state(cstate->base.state); - bool apply_memory_bw_wa = skl_needs_memory_bw_wa(state); - - if (!intel_wm_plane_visible(cstate, intel_pstate)) - return 0; + u32 interm_pbpl; /* only NV12 format has two planes */ - if (plane_id == 1 && fb->format->format != DRM_FORMAT_NV12) { + if (color_plane == 1 && fb->format->format != DRM_FORMAT_NV12) { DRM_DEBUG_KMS("Non NV12 format have single plane\n"); return -EINVAL; } @@ -4599,15 +4604,15 @@ skl_compute_plane_wm_params(const struct drm_i915_private *dev_priv, wp->width = drm_rect_width(&intel_pstate->base.src) >> 16; } - if (plane_id == 1 && wp->is_planar) + if (color_plane == 1 && wp->is_planar) wp->width /= 2; - wp->cpp = fb->format->cpp[plane_id]; + wp->cpp = fb->format->cpp[color_plane]; wp->plane_pixel_rate = skl_adjusted_plane_pixel_rate(cstate, intel_pstate); if (INTEL_GEN(dev_priv) >= 11 && - fb->modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 8) + fb->modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1) wp->dbuf_block_size = 256; else wp->dbuf_block_size = 512; @@ -4632,7 +4637,7 @@ skl_compute_plane_wm_params(const struct drm_i915_private *dev_priv, wp->y_min_scanlines = 4; } - if (apply_memory_bw_wa) + if (skl_needs_memory_bw_wa(dev_priv)) wp->y_min_scanlines *= 2; wp->plane_bytes_per_line = wp->width * wp->cpp; @@ -4646,7 +4651,7 @@ skl_compute_plane_wm_params(const struct drm_i915_private *dev_priv, wp->plane_blocks_per_line = div_fixed16(interm_pbpl, wp->y_min_scanlines); - } else if (wp->x_tiled && IS_GEN9(dev_priv)) { + } else if (wp->x_tiled && IS_GEN(dev_priv, 9)) { interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line, wp->dbuf_block_size); wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl); @@ -4664,29 +4669,33 @@ skl_compute_plane_wm_params(const struct drm_i915_private *dev_priv, return 0; } -static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv, - struct intel_crtc_state *cstate, - const struct intel_plane_state *intel_pstate, - uint16_t ddb_allocation, - int level, - const struct skl_wm_params *wp, - const struct skl_wm_level *result_prev, - struct skl_wm_level *result /* out */) +static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level) { - const struct drm_plane_state *pstate = &intel_pstate->base; - uint32_t latency = dev_priv->wm.skl_latency[level]; + if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) + return true; + + /* The number of lines are ignored for the level 0 watermark. */ + return level > 0; +} + +static void skl_compute_plane_wm(const struct intel_crtc_state *cstate, + const struct intel_plane_state *intel_pstate, + int level, + const struct skl_wm_params *wp, + const struct skl_wm_level *result_prev, + struct skl_wm_level *result /* out */) +{ + struct drm_i915_private *dev_priv = + to_i915(intel_pstate->base.plane->dev); + u32 latency = dev_priv->wm.skl_latency[level]; uint_fixed_16_16_t method1, method2; uint_fixed_16_16_t selected_result; - uint32_t res_blocks, res_lines; - struct intel_atomic_state *state = - to_intel_atomic_state(cstate->base.state); - bool apply_memory_bw_wa = skl_needs_memory_bw_wa(state); - uint32_t min_disp_buf_needed; + u32 res_blocks, res_lines, min_ddb_alloc = 0; - if (latency == 0 || - !intel_wm_plane_visible(cstate, intel_pstate)) { - result->plane_en = false; - return 0; + if (latency == 0) { + /* reject it */ + result->min_ddb_alloc = U16_MAX; + return; } /* Display WA #1141: kbl,cfl */ @@ -4695,7 +4704,7 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv, dev_priv->ipc_enabled) latency += 4; - if (apply_memory_bw_wa && wp->x_tiled) + if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled) latency += 15; method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate, @@ -4710,222 +4719,172 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv, } else { if ((wp->cpp * cstate->base.adjusted_mode.crtc_htotal / wp->dbuf_block_size < 1) && - (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) + (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) { selected_result = method2; - else if (ddb_allocation >= - fixed16_to_u32_round_up(wp->plane_blocks_per_line)) - selected_result = min_fixed16(method1, method2); - else if (latency >= wp->linetime_us) - selected_result = min_fixed16(method1, method2); - else + } else if (latency >= wp->linetime_us) { + if (IS_GEN(dev_priv, 9) && + !IS_GEMINILAKE(dev_priv)) + selected_result = min_fixed16(method1, method2); + else + selected_result = method2; + } else { selected_result = method1; + } } res_blocks = fixed16_to_u32_round_up(selected_result) + 1; res_lines = div_round_up_fixed16(selected_result, wp->plane_blocks_per_line); - /* Display WA #1125: skl,bxt,kbl,glk */ - if (level == 0 && wp->rc_surface) - res_blocks += fixed16_to_u32_round_up(wp->y_tile_minimum); + if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv)) { + /* Display WA #1125: skl,bxt,kbl */ + if (level == 0 && wp->rc_surface) + res_blocks += + fixed16_to_u32_round_up(wp->y_tile_minimum); + + /* Display WA #1126: skl,bxt,kbl */ + if (level >= 1 && level <= 7) { + if (wp->y_tiled) { + res_blocks += + fixed16_to_u32_round_up(wp->y_tile_minimum); + res_lines += wp->y_min_scanlines; + } else { + res_blocks++; + } - /* Display WA #1126: skl,bxt,kbl,glk */ - if (level >= 1 && level <= 7) { - if (wp->y_tiled) { - res_blocks += fixed16_to_u32_round_up( - wp->y_tile_minimum); - res_lines += wp->y_min_scanlines; - } else { - res_blocks++; + /* + * Make sure result blocks for higher latency levels are + * atleast as high as level below the current level. + * Assumption in DDB algorithm optimization for special + * cases. Also covers Display WA #1125 for RC. + */ + if (result_prev->plane_res_b > res_blocks) + res_blocks = result_prev->plane_res_b; } - - /* - * Make sure result blocks for higher latency levels are atleast - * as high as level below the current level. - * Assumption in DDB algorithm optimization for special cases. - * Also covers Display WA #1125 for RC. - */ - if (result_prev->plane_res_b > res_blocks) - res_blocks = result_prev->plane_res_b; } if (INTEL_GEN(dev_priv) >= 11) { if (wp->y_tiled) { - uint32_t extra_lines; - uint_fixed_16_16_t fp_min_disp_buf_needed; + int extra_lines; if (res_lines % wp->y_min_scanlines == 0) extra_lines = wp->y_min_scanlines; else extra_lines = wp->y_min_scanlines * 2 - - res_lines % wp->y_min_scanlines; + res_lines % wp->y_min_scanlines; - fp_min_disp_buf_needed = mul_u32_fixed16(res_lines + - extra_lines, - wp->plane_blocks_per_line); - min_disp_buf_needed = fixed16_to_u32_round_up( - fp_min_disp_buf_needed); + min_ddb_alloc = mul_round_up_u32_fixed16(res_lines + extra_lines, + wp->plane_blocks_per_line); } else { - min_disp_buf_needed = DIV_ROUND_UP(res_blocks * 11, 10); + min_ddb_alloc = res_blocks + + DIV_ROUND_UP(res_blocks, 10); } - } else { - min_disp_buf_needed = res_blocks; } - if ((level > 0 && res_lines > 31) || - res_blocks >= ddb_allocation || - min_disp_buf_needed >= ddb_allocation) { - result->plane_en = false; - - /* - * If there are no valid level 0 watermarks, then we can't - * support this display configuration. - */ - if (level) { - return 0; - } else { - struct drm_plane *plane = pstate->plane; + if (!skl_wm_has_lines(dev_priv, level)) + res_lines = 0; - DRM_DEBUG_KMS("Requested display configuration exceeds system watermark limitations\n"); - DRM_DEBUG_KMS("[PLANE:%d:%s] blocks required = %u/%u, lines required = %u/31\n", - plane->base.id, plane->name, - res_blocks, ddb_allocation, res_lines); - return -EINVAL; - } + if (res_lines > 31) { + /* reject it */ + result->min_ddb_alloc = U16_MAX; + return; } /* - * Display WA #826 (SKL:ALL, BXT:ALL) & #1059 (CNL:A) - * disable wm level 1-7 on NV12 planes + * If res_lines is valid, assume we can use this watermark level + * for now. We'll come back and disable it after we calculate the + * DDB allocation if it turns out we don't actually have enough + * blocks to satisfy it. */ - if (wp->is_planar && level >= 1 && - (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv) || - IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_A0))) { - result->plane_en = false; - return 0; - } - - /* The number of lines are ignored for the level 0 watermark. */ result->plane_res_b = res_blocks; result->plane_res_l = res_lines; + /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */ + result->min_ddb_alloc = max(min_ddb_alloc, res_blocks) + 1; result->plane_en = true; - - return 0; } -static int -skl_compute_wm_levels(const struct drm_i915_private *dev_priv, - struct skl_ddb_allocation *ddb, - struct intel_crtc_state *cstate, +static void +skl_compute_wm_levels(const struct intel_crtc_state *cstate, const struct intel_plane_state *intel_pstate, const struct skl_wm_params *wm_params, - struct skl_plane_wm *wm, - int plane_id) + struct skl_wm_level *levels) { - struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc); - struct drm_plane *plane = intel_pstate->base.plane; - struct intel_plane *intel_plane = to_intel_plane(plane); - uint16_t ddb_blocks; - enum pipe pipe = intel_crtc->pipe; + struct drm_i915_private *dev_priv = + to_i915(intel_pstate->base.plane->dev); int level, max_level = ilk_wm_max_level(dev_priv); - enum plane_id intel_plane_id = intel_plane->id; - int ret; - - if (WARN_ON(!intel_pstate->base.fb)) - return -EINVAL; - - ddb_blocks = plane_id ? - skl_ddb_entry_size(&ddb->uv_plane[pipe][intel_plane_id]) : - skl_ddb_entry_size(&ddb->plane[pipe][intel_plane_id]); + struct skl_wm_level *result_prev = &levels[0]; for (level = 0; level <= max_level; level++) { - struct skl_wm_level *result = plane_id ? &wm->uv_wm[level] : - &wm->wm[level]; - struct skl_wm_level *result_prev; - - if (level) - result_prev = plane_id ? &wm->uv_wm[level - 1] : - &wm->wm[level - 1]; - else - result_prev = plane_id ? &wm->uv_wm[0] : &wm->wm[0]; - - ret = skl_compute_plane_wm(dev_priv, - cstate, - intel_pstate, - ddb_blocks, - level, - wm_params, - result_prev, - result); - if (ret) - return ret; - } + struct skl_wm_level *result = &levels[level]; - if (intel_pstate->base.fb->format->format == DRM_FORMAT_NV12) - wm->is_planar = true; + skl_compute_plane_wm(cstate, intel_pstate, level, wm_params, + result_prev, result); - return 0; + result_prev = result; + } } -static uint32_t -skl_compute_linetime_wm(struct intel_crtc_state *cstate) +static u32 +skl_compute_linetime_wm(const struct intel_crtc_state *cstate) { struct drm_atomic_state *state = cstate->base.state; struct drm_i915_private *dev_priv = to_i915(state->dev); uint_fixed_16_16_t linetime_us; - uint32_t linetime_wm; + u32 linetime_wm; linetime_us = intel_get_linetime_us(cstate); - - if (is_fixed16_zero(linetime_us)) - return 0; - linetime_wm = fixed16_to_u32_round_up(mul_u32_fixed16(8, linetime_us)); - /* Display WA #1135: bxt:ALL GLK:ALL */ - if ((IS_BROXTON(dev_priv) || IS_GEMINILAKE(dev_priv)) && - dev_priv->ipc_enabled) + /* Display WA #1135: BXT:ALL GLK:ALL */ + if (IS_GEN9_LP(dev_priv) && dev_priv->ipc_enabled) linetime_wm /= 2; return linetime_wm; } -static void skl_compute_transition_wm(struct intel_crtc_state *cstate, - struct skl_wm_params *wp, - struct skl_wm_level *wm_l0, - uint16_t ddb_allocation, - struct skl_wm_level *trans_wm /* out */) +static void skl_compute_transition_wm(const struct intel_crtc_state *cstate, + const struct skl_wm_params *wp, + struct skl_plane_wm *wm) { struct drm_device *dev = cstate->base.crtc->dev; const struct drm_i915_private *dev_priv = to_i915(dev); - uint16_t trans_min, trans_y_tile_min; - const uint16_t trans_amount = 10; /* This is configurable amount */ - uint16_t trans_offset_b, res_blocks; - - if (!cstate->base.active) - goto exit; + u16 trans_min, trans_y_tile_min; + const u16 trans_amount = 10; /* This is configurable amount */ + u16 wm0_sel_res_b, trans_offset_b, res_blocks; /* Transition WM are not recommended by HW team for GEN9 */ if (INTEL_GEN(dev_priv) <= 9) - goto exit; + return; /* Transition WM don't make any sense if ipc is disabled */ if (!dev_priv->ipc_enabled) - goto exit; + return; - trans_min = 0; - if (INTEL_GEN(dev_priv) >= 10) + trans_min = 14; + if (INTEL_GEN(dev_priv) >= 11) trans_min = 4; trans_offset_b = trans_min + trans_amount; + /* + * The spec asks for Selected Result Blocks for wm0 (the real value), + * not Result Blocks (the integer value). Pay attention to the capital + * letters. The value wm_l0->plane_res_b is actually Result Blocks, but + * since Result Blocks is the ceiling of Selected Result Blocks plus 1, + * and since we later will have to get the ceiling of the sum in the + * transition watermarks calculation, we can just pretend Selected + * Result Blocks is Result Blocks minus 1 and it should work for the + * current platforms. + */ + wm0_sel_res_b = wm->wm[0].plane_res_b - 1; + if (wp->y_tiled) { - trans_y_tile_min = (uint16_t) mul_round_up_u32_fixed16(2, - wp->y_tile_minimum); - res_blocks = max(wm_l0->plane_res_b, trans_y_tile_min) + + trans_y_tile_min = + (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum); + res_blocks = max(wm0_sel_res_b, trans_y_tile_min) + trans_offset_b; } else { - res_blocks = wm_l0->plane_res_b + trans_offset_b; + res_blocks = wm0_sel_res_b + trans_offset_b; /* WA BUG:1938466 add one block for non y-tile planes */ if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_A0)) @@ -4933,28 +4892,127 @@ static void skl_compute_transition_wm(struct intel_crtc_state *cstate, } - res_blocks += 1; + /* + * Just assume we can enable the transition watermark. After + * computing the DDB we'll come back and disable it if that + * assumption turns out to be false. + */ + wm->trans_wm.plane_res_b = res_blocks + 1; + wm->trans_wm.plane_en = true; +} - if (res_blocks < ddb_allocation) { - trans_wm->plane_res_b = res_blocks; - trans_wm->plane_en = true; - return; +static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state, + enum plane_id plane_id, int color_plane) +{ + struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id]; + struct skl_wm_params wm_params; + int ret; + + ret = skl_compute_plane_wm_params(crtc_state, plane_state, + &wm_params, color_plane); + if (ret) + return ret; + + skl_compute_wm_levels(crtc_state, plane_state, &wm_params, wm->wm); + skl_compute_transition_wm(crtc_state, &wm_params, wm); + + return 0; +} + +static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state, + enum plane_id plane_id) +{ + struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id]; + struct skl_wm_params wm_params; + int ret; + + wm->is_planar = true; + + /* uv plane watermarks must also be validated for NV12/Planar */ + ret = skl_compute_plane_wm_params(crtc_state, plane_state, + &wm_params, 1); + if (ret) + return ret; + + skl_compute_wm_levels(crtc_state, plane_state, &wm_params, wm->uv_wm); + + return 0; +} + +static int skl_build_plane_wm(struct skl_pipe_wm *pipe_wm, + struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state) +{ + struct intel_plane *plane = to_intel_plane(plane_state->base.plane); + const struct drm_framebuffer *fb = plane_state->base.fb; + enum plane_id plane_id = plane->id; + int ret; + + if (!intel_wm_plane_visible(crtc_state, plane_state)) + return 0; + + ret = skl_build_plane_wm_single(crtc_state, plane_state, + plane_id, 0); + if (ret) + return ret; + + if (fb->format->is_yuv && fb->format->num_planes > 1) { + ret = skl_build_plane_wm_uv(crtc_state, plane_state, + plane_id); + if (ret) + return ret; } -exit: - trans_wm->plane_en = false; + return 0; +} + +static int icl_build_plane_wm(struct skl_pipe_wm *pipe_wm, + struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state) +{ + enum plane_id plane_id = to_intel_plane(plane_state->base.plane)->id; + int ret; + + /* Watermarks calculated in master */ + if (plane_state->slave) + return 0; + + if (plane_state->linked_plane) { + const struct drm_framebuffer *fb = plane_state->base.fb; + enum plane_id y_plane_id = plane_state->linked_plane->id; + + WARN_ON(!intel_wm_plane_visible(crtc_state, plane_state)); + WARN_ON(!fb->format->is_yuv || + fb->format->num_planes == 1); + + ret = skl_build_plane_wm_single(crtc_state, plane_state, + y_plane_id, 0); + if (ret) + return ret; + + ret = skl_build_plane_wm_single(crtc_state, plane_state, + plane_id, 1); + if (ret) + return ret; + } else if (intel_wm_plane_visible(crtc_state, plane_state)) { + ret = skl_build_plane_wm_single(crtc_state, plane_state, + plane_id, 0); + if (ret) + return ret; + } + + return 0; } static int skl_build_pipe_wm(struct intel_crtc_state *cstate, - struct skl_ddb_allocation *ddb, struct skl_pipe_wm *pipe_wm) { - struct drm_device *dev = cstate->base.crtc->dev; + struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev); struct drm_crtc_state *crtc_state = &cstate->base; - const struct drm_i915_private *dev_priv = to_i915(dev); struct drm_plane *plane; const struct drm_plane_state *pstate; - struct skl_plane_wm *wm; int ret; /* @@ -4966,44 +5024,15 @@ static int skl_build_pipe_wm(struct intel_crtc_state *cstate, drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) { const struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate); - enum plane_id plane_id = to_intel_plane(plane)->id; - struct skl_wm_params wm_params; - enum pipe pipe = to_intel_crtc(cstate->base.crtc)->pipe; - uint16_t ddb_blocks; - wm = &pipe_wm->planes[plane_id]; - ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][plane_id]); - - ret = skl_compute_plane_wm_params(dev_priv, cstate, - intel_pstate, &wm_params, 0); - if (ret) - return ret; - - ret = skl_compute_wm_levels(dev_priv, ddb, cstate, - intel_pstate, &wm_params, wm, 0); + if (INTEL_GEN(dev_priv) >= 11) + ret = icl_build_plane_wm(pipe_wm, + cstate, intel_pstate); + else + ret = skl_build_plane_wm(pipe_wm, + cstate, intel_pstate); if (ret) return ret; - - skl_compute_transition_wm(cstate, &wm_params, &wm->wm[0], - ddb_blocks, &wm->trans_wm); - - /* uv plane watermarks must also be validated for NV12/Planar */ - if (wm_params.is_planar) { - memset(&wm_params, 0, sizeof(struct skl_wm_params)); - wm->is_planar = true; - - ret = skl_compute_plane_wm_params(dev_priv, cstate, - intel_pstate, - &wm_params, 1); - if (ret) - return ret; - - ret = skl_compute_wm_levels(dev_priv, ddb, cstate, - intel_pstate, &wm_params, - wm, 1); - if (ret) - return ret; - } } pipe_wm->linetime = skl_compute_linetime_wm(cstate); @@ -5016,16 +5045,16 @@ static void skl_ddb_entry_write(struct drm_i915_private *dev_priv, const struct skl_ddb_entry *entry) { if (entry->end) - I915_WRITE(reg, (entry->end - 1) << 16 | entry->start); + I915_WRITE_FW(reg, (entry->end - 1) << 16 | entry->start); else - I915_WRITE(reg, 0); + I915_WRITE_FW(reg, 0); } static void skl_write_wm_level(struct drm_i915_private *dev_priv, i915_reg_t reg, const struct skl_wm_level *level) { - uint32_t val = 0; + u32 val = 0; if (level->plane_en) { val |= PLANE_WM_EN; @@ -5033,19 +5062,22 @@ static void skl_write_wm_level(struct drm_i915_private *dev_priv, val |= level->plane_res_l << PLANE_WM_LINES_SHIFT; } - I915_WRITE(reg, val); + I915_WRITE_FW(reg, val); } -static void skl_write_plane_wm(struct intel_crtc *intel_crtc, - const struct skl_plane_wm *wm, - const struct skl_ddb_allocation *ddb, - enum plane_id plane_id) +void skl_write_plane_wm(struct intel_plane *plane, + const struct intel_crtc_state *crtc_state) { - struct drm_crtc *crtc = &intel_crtc->base; - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(plane->base.dev); int level, max_level = ilk_wm_max_level(dev_priv); - enum pipe pipe = intel_crtc->pipe; + enum plane_id plane_id = plane->id; + enum pipe pipe = plane->pipe; + const struct skl_plane_wm *wm = + &crtc_state->wm.skl.optimal.planes[plane_id]; + const struct skl_ddb_entry *ddb_y = + &crtc_state->wm.skl.plane_ddb_y[plane_id]; + const struct skl_ddb_entry *ddb_uv = + &crtc_state->wm.skl.plane_ddb_uv[plane_id]; for (level = 0; level <= max_level; level++) { skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level), @@ -5054,35 +5086,32 @@ static void skl_write_plane_wm(struct intel_crtc *intel_crtc, skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id), &wm->trans_wm); - skl_ddb_entry_write(dev_priv, PLANE_BUF_CFG(pipe, plane_id), - &ddb->plane[pipe][plane_id]); - /* FIXME: add proper NV12 support for ICL. */ - if (INTEL_GEN(dev_priv) >= 11) - return skl_ddb_entry_write(dev_priv, - PLANE_BUF_CFG(pipe, plane_id), - &ddb->plane[pipe][plane_id]); - if (wm->is_planar) { - skl_ddb_entry_write(dev_priv, PLANE_BUF_CFG(pipe, plane_id), - &ddb->uv_plane[pipe][plane_id]); + if (INTEL_GEN(dev_priv) >= 11) { skl_ddb_entry_write(dev_priv, - PLANE_NV12_BUF_CFG(pipe, plane_id), - &ddb->plane[pipe][plane_id]); - } else { - skl_ddb_entry_write(dev_priv, PLANE_BUF_CFG(pipe, plane_id), - &ddb->plane[pipe][plane_id]); - I915_WRITE(PLANE_NV12_BUF_CFG(pipe, plane_id), 0x0); + PLANE_BUF_CFG(pipe, plane_id), ddb_y); + return; } + + if (wm->is_planar) + swap(ddb_y, ddb_uv); + + skl_ddb_entry_write(dev_priv, + PLANE_BUF_CFG(pipe, plane_id), ddb_y); + skl_ddb_entry_write(dev_priv, + PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_uv); } -static void skl_write_cursor_wm(struct intel_crtc *intel_crtc, - const struct skl_plane_wm *wm, - const struct skl_ddb_allocation *ddb) +void skl_write_cursor_wm(struct intel_plane *plane, + const struct intel_crtc_state *crtc_state) { - struct drm_crtc *crtc = &intel_crtc->base; - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(plane->base.dev); int level, max_level = ilk_wm_max_level(dev_priv); - enum pipe pipe = intel_crtc->pipe; + enum plane_id plane_id = plane->id; + enum pipe pipe = plane->pipe; + const struct skl_plane_wm *wm = + &crtc_state->wm.skl.optimal.planes[plane_id]; + const struct skl_ddb_entry *ddb = + &crtc_state->wm.skl.plane_ddb_y[plane_id]; for (level = 0; level <= max_level; level++) { skl_write_wm_level(dev_priv, CUR_WM(pipe, level), @@ -5090,22 +5119,47 @@ static void skl_write_cursor_wm(struct intel_crtc *intel_crtc, } skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), &wm->trans_wm); - skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), - &ddb->plane[pipe][PLANE_CURSOR]); + skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb); } bool skl_wm_level_equals(const struct skl_wm_level *l1, const struct skl_wm_level *l2) { - if (l1->plane_en != l2->plane_en) - return false; + return l1->plane_en == l2->plane_en && + l1->plane_res_l == l2->plane_res_l && + l1->plane_res_b == l2->plane_res_b; +} - /* If both planes aren't enabled, the rest shouldn't matter */ - if (!l1->plane_en) - return true; +static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv, + const struct skl_plane_wm *wm1, + const struct skl_plane_wm *wm2) +{ + int level, max_level = ilk_wm_max_level(dev_priv); - return (l1->plane_res_l == l2->plane_res_l && - l1->plane_res_b == l2->plane_res_b); + for (level = 0; level <= max_level; level++) { + if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]) || + !skl_wm_level_equals(&wm1->uv_wm[level], &wm2->uv_wm[level])) + return false; + } + + return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm); +} + +static bool skl_pipe_wm_equals(struct intel_crtc *crtc, + const struct skl_pipe_wm *wm1, + const struct skl_pipe_wm *wm2) +{ + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum plane_id plane_id; + + for_each_plane_id_on_crtc(crtc, plane_id) { + if (!skl_plane_wm_equals(dev_priv, + &wm1->planes[plane_id], + &wm2->planes[plane_id])) + return false; + } + + return wm1->linetime == wm2->linetime; } static inline bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a, @@ -5114,106 +5168,100 @@ static inline bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a, return a->start < b->end && b->start < a->end; } -bool skl_ddb_allocation_overlaps(struct drm_i915_private *dev_priv, - const struct skl_ddb_entry **entries, - const struct skl_ddb_entry *ddb, - int ignore) +bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb, + const struct skl_ddb_entry entries[], + int num_entries, int ignore_idx) { - enum pipe pipe; + int i; - for_each_pipe(dev_priv, pipe) { - if (pipe != ignore && entries[pipe] && - skl_ddb_entries_overlap(ddb, entries[pipe])) + for (i = 0; i < num_entries; i++) { + if (i != ignore_idx && + skl_ddb_entries_overlap(ddb, &entries[i])) return true; } return false; } -static int skl_update_pipe_wm(struct drm_crtc_state *cstate, +static int skl_update_pipe_wm(struct intel_crtc_state *cstate, const struct skl_pipe_wm *old_pipe_wm, struct skl_pipe_wm *pipe_wm, /* out */ - struct skl_ddb_allocation *ddb, /* out */ bool *changed /* out */) { - struct intel_crtc_state *intel_cstate = to_intel_crtc_state(cstate); + struct intel_crtc *crtc = to_intel_crtc(cstate->base.crtc); int ret; - ret = skl_build_pipe_wm(intel_cstate, ddb, pipe_wm); + ret = skl_build_pipe_wm(cstate, pipe_wm); if (ret) return ret; - if (!memcmp(old_pipe_wm, pipe_wm, sizeof(*pipe_wm))) - *changed = false; - else - *changed = true; + *changed = !skl_pipe_wm_equals(crtc, old_pipe_wm, pipe_wm); return 0; } -static uint32_t -pipes_modified(struct drm_atomic_state *state) +static u32 +pipes_modified(struct intel_atomic_state *state) { - struct drm_crtc *crtc; - struct drm_crtc_state *cstate; - uint32_t i, ret = 0; + struct intel_crtc *crtc; + struct intel_crtc_state *cstate; + u32 i, ret = 0; - for_each_new_crtc_in_state(state, crtc, cstate, i) - ret |= drm_crtc_mask(crtc); + for_each_new_intel_crtc_in_state(state, crtc, cstate, i) + ret |= drm_crtc_mask(&crtc->base); return ret; } static int -skl_ddb_add_affected_planes(struct intel_crtc_state *cstate) +skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state, + struct intel_crtc_state *new_crtc_state) { - struct drm_atomic_state *state = cstate->base.state; - struct drm_device *dev = state->dev; - struct drm_crtc *crtc = cstate->base.crtc; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_atomic_state *intel_state = to_intel_atomic_state(state); - struct skl_ddb_allocation *new_ddb = &intel_state->wm_results.ddb; - struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb; - struct drm_plane_state *plane_state; - struct drm_plane *plane; - enum pipe pipe = intel_crtc->pipe; + struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->base.state); + struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct intel_plane *plane; - drm_for_each_plane_mask(plane, dev, cstate->base.plane_mask) { - enum plane_id plane_id = to_intel_plane(plane)->id; + for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) { + struct intel_plane_state *plane_state; + enum plane_id plane_id = plane->id; - if (skl_ddb_entry_equal(&cur_ddb->plane[pipe][plane_id], - &new_ddb->plane[pipe][plane_id]) && - skl_ddb_entry_equal(&cur_ddb->uv_plane[pipe][plane_id], - &new_ddb->uv_plane[pipe][plane_id])) + if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id], + &new_crtc_state->wm.skl.plane_ddb_y[plane_id]) && + skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_uv[plane_id], + &new_crtc_state->wm.skl.plane_ddb_uv[plane_id])) continue; - plane_state = drm_atomic_get_plane_state(state, plane); + plane_state = intel_atomic_get_plane_state(state, plane); if (IS_ERR(plane_state)) return PTR_ERR(plane_state); + + new_crtc_state->update_planes |= BIT(plane_id); } return 0; } static int -skl_compute_ddb(struct drm_atomic_state *state) +skl_compute_ddb(struct intel_atomic_state *state) { - const struct drm_i915_private *dev_priv = to_i915(state->dev); - struct intel_atomic_state *intel_state = to_intel_atomic_state(state); - struct skl_ddb_allocation *ddb = &intel_state->wm_results.ddb; + const struct drm_i915_private *dev_priv = to_i915(state->base.dev); + struct skl_ddb_allocation *ddb = &state->wm_results.ddb; + struct intel_crtc_state *old_crtc_state; + struct intel_crtc_state *new_crtc_state; struct intel_crtc *crtc; - struct intel_crtc_state *cstate; int ret, i; memcpy(ddb, &dev_priv->wm.skl_hw.ddb, sizeof(*ddb)); - for_each_new_intel_crtc_in_state(intel_state, crtc, cstate, i) { - ret = skl_allocate_pipe_ddb(cstate, ddb); + for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, + new_crtc_state, i) { + ret = skl_allocate_pipe_ddb(new_crtc_state, ddb); if (ret) return ret; - ret = skl_ddb_add_affected_planes(cstate); + ret = skl_ddb_add_affected_planes(old_crtc_state, + new_crtc_state); if (ret) return ret; } @@ -5222,52 +5270,43 @@ skl_compute_ddb(struct drm_atomic_state *state) } static void -skl_print_wm_changes(const struct drm_atomic_state *state) +skl_print_wm_changes(struct intel_atomic_state *state) { - const struct drm_device *dev = state->dev; - const struct drm_i915_private *dev_priv = to_i915(dev); - const struct intel_atomic_state *intel_state = - to_intel_atomic_state(state); - const struct drm_crtc *crtc; - const struct drm_crtc_state *cstate; - const struct intel_plane *intel_plane; - const struct skl_ddb_allocation *old_ddb = &dev_priv->wm.skl_hw.ddb; - const struct skl_ddb_allocation *new_ddb = &intel_state->wm_results.ddb; + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + const struct intel_crtc_state *old_crtc_state; + const struct intel_crtc_state *new_crtc_state; + struct intel_plane *plane; + struct intel_crtc *crtc; int i; - for_each_new_crtc_in_state(state, crtc, cstate, i) { - const struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - enum pipe pipe = intel_crtc->pipe; - - for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { - enum plane_id plane_id = intel_plane->id; + for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, + new_crtc_state, i) { + for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) { + enum plane_id plane_id = plane->id; const struct skl_ddb_entry *old, *new; - old = &old_ddb->plane[pipe][plane_id]; - new = &new_ddb->plane[pipe][plane_id]; + old = &old_crtc_state->wm.skl.plane_ddb_y[plane_id]; + new = &new_crtc_state->wm.skl.plane_ddb_y[plane_id]; if (skl_ddb_entry_equal(old, new)) continue; - DRM_DEBUG_ATOMIC("[PLANE:%d:%s] ddb (%d - %d) -> (%d - %d)\n", - intel_plane->base.base.id, - intel_plane->base.name, - old->start, old->end, - new->start, new->end); + DRM_DEBUG_KMS("[PLANE:%d:%s] ddb (%d - %d) -> (%d - %d)\n", + plane->base.base.id, plane->base.name, + old->start, old->end, + new->start, new->end); } } } static int -skl_ddb_add_affected_pipes(struct drm_atomic_state *state, bool *changed) +skl_ddb_add_affected_pipes(struct intel_atomic_state *state, bool *changed) { - struct drm_device *dev = state->dev; + struct drm_device *dev = state->base.dev; const struct drm_i915_private *dev_priv = to_i915(dev); - const struct drm_crtc *crtc; - const struct drm_crtc_state *cstate; - struct intel_crtc *intel_crtc; - struct intel_atomic_state *intel_state = to_intel_atomic_state(state); - uint32_t realloc_pipes = pipes_modified(state); + struct intel_crtc *crtc; + struct intel_crtc_state *crtc_state; + u32 realloc_pipes = pipes_modified(state); int ret, i; /* @@ -5285,7 +5324,7 @@ skl_ddb_add_affected_pipes(struct drm_atomic_state *state, bool *changed) * since any racing commits that want to update them would need to * hold _all_ CRTC state mutexes. */ - for_each_new_crtc_in_state(state, crtc, cstate, i) + for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) (*changed) = true; if (!*changed) @@ -5299,20 +5338,20 @@ skl_ddb_add_affected_pipes(struct drm_atomic_state *state, bool *changed) */ if (dev_priv->wm.distrust_bios_wm) { ret = drm_modeset_lock(&dev->mode_config.connection_mutex, - state->acquire_ctx); + state->base.acquire_ctx); if (ret) return ret; - intel_state->active_pipe_changes = ~0; + state->active_pipe_changes = ~0; /* - * We usually only initialize intel_state->active_crtcs if we + * We usually only initialize state->active_crtcs if we * we're doing a modeset; make sure this field is always * initialized during the sanitization process that happens * on the first commit too. */ - if (!intel_state->modeset) - intel_state->active_crtcs = dev_priv->active_crtcs; + if (!state->modeset) + state->active_crtcs = dev_priv->active_crtcs; } /* @@ -5328,33 +5367,91 @@ skl_ddb_add_affected_pipes(struct drm_atomic_state *state, bool *changed) * any other display updates race with this transaction, so we need * to grab the lock on *all* CRTC's. */ - if (intel_state->active_pipe_changes || intel_state->modeset) { + if (state->active_pipe_changes || state->modeset) { realloc_pipes = ~0; - intel_state->wm_results.dirty_pipes = ~0; + state->wm_results.dirty_pipes = ~0; } /* * We're not recomputing for the pipes not included in the commit, so * make sure we start with the current state. */ - for_each_intel_crtc_mask(dev, intel_crtc, realloc_pipes) { - struct intel_crtc_state *cstate; + for_each_intel_crtc_mask(dev, crtc, realloc_pipes) { + crtc_state = intel_atomic_get_crtc_state(&state->base, crtc); + if (IS_ERR(crtc_state)) + return PTR_ERR(crtc_state); + } - cstate = intel_atomic_get_crtc_state(state, intel_crtc); - if (IS_ERR(cstate)) - return PTR_ERR(cstate); + return 0; +} + +/* + * To make sure the cursor watermark registers are always consistent + * with our computed state the following scenario needs special + * treatment: + * + * 1. enable cursor + * 2. move cursor entirely offscreen + * 3. disable cursor + * + * Step 2. does call .disable_plane() but does not zero the watermarks + * (since we consider an offscreen cursor still active for the purposes + * of watermarks). Step 3. would not normally call .disable_plane() + * because the actual plane visibility isn't changing, and we don't + * deallocate the cursor ddb until the pipe gets disabled. So we must + * force step 3. to call .disable_plane() to update the watermark + * registers properly. + * + * Other planes do not suffer from this issues as their watermarks are + * calculated based on the actual plane visibility. The only time this + * can trigger for the other planes is during the initial readout as the + * default value of the watermarks registers is not zero. + */ +static int skl_wm_add_affected_planes(struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + const struct intel_crtc_state *old_crtc_state = + intel_atomic_get_old_crtc_state(state, crtc); + struct intel_crtc_state *new_crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + struct intel_plane *plane; + + for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) { + struct intel_plane_state *plane_state; + enum plane_id plane_id = plane->id; + + /* + * Force a full wm update for every plane on modeset. + * Required because the reset value of the wm registers + * is non-zero, whereas we want all disabled planes to + * have zero watermarks. So if we turn off the relevant + * power well the hardware state will go out of sync + * with the software state. + */ + if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->base) && + skl_plane_wm_equals(dev_priv, + &old_crtc_state->wm.skl.optimal.planes[plane_id], + &new_crtc_state->wm.skl.optimal.planes[plane_id])) + continue; + + plane_state = intel_atomic_get_plane_state(state, plane); + if (IS_ERR(plane_state)) + return PTR_ERR(plane_state); + + new_crtc_state->update_planes |= BIT(plane_id); } return 0; } static int -skl_compute_wm(struct drm_atomic_state *state) +skl_compute_wm(struct intel_atomic_state *state) { - struct drm_crtc *crtc; - struct drm_crtc_state *cstate; - struct intel_atomic_state *intel_state = to_intel_atomic_state(state); - struct skl_ddb_values *results = &intel_state->wm_results; + struct intel_crtc *crtc; + struct intel_crtc_state *cstate; + struct intel_crtc_state *old_crtc_state; + struct skl_ddb_values *results = &state->wm_results; struct skl_pipe_wm *pipe_wm; bool changed = false; int ret, i; @@ -5366,42 +5463,34 @@ skl_compute_wm(struct drm_atomic_state *state) if (ret || !changed) return ret; - ret = skl_compute_ddb(state); - if (ret) - return ret; - /* * Calculate WM's for all pipes that are part of this transaction. - * Note that the DDB allocation above may have added more CRTC's that + * Note that skl_ddb_add_affected_pipes may have added more CRTC's that * weren't otherwise being modified (and set bits in dirty_pipes) if * pipe allocations had to change. - * - * FIXME: Now that we're doing this in the atomic check phase, we - * should allow skl_update_pipe_wm() to return failure in cases where - * no suitable watermark values can be found. */ - for_each_new_crtc_in_state(state, crtc, cstate, i) { - struct intel_crtc_state *intel_cstate = - to_intel_crtc_state(cstate); + for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, + cstate, i) { const struct skl_pipe_wm *old_pipe_wm = - &to_intel_crtc_state(crtc->state)->wm.skl.optimal; + &old_crtc_state->wm.skl.optimal; - pipe_wm = &intel_cstate->wm.skl.optimal; - ret = skl_update_pipe_wm(cstate, old_pipe_wm, pipe_wm, - &results->ddb, &changed); + pipe_wm = &cstate->wm.skl.optimal; + ret = skl_update_pipe_wm(cstate, old_pipe_wm, pipe_wm, &changed); if (ret) return ret; - if (changed) - results->dirty_pipes |= drm_crtc_mask(crtc); - - if ((results->dirty_pipes & drm_crtc_mask(crtc)) == 0) - /* This pipe's WM's did not change */ - continue; + ret = skl_wm_add_affected_planes(state, crtc); + if (ret) + return ret; - intel_cstate->update_wm_pre = true; + if (changed) + results->dirty_pipes |= drm_crtc_mask(&crtc->base); } + ret = skl_compute_ddb(state); + if (ret) + return ret; + skl_print_wm_changes(state); return 0; @@ -5413,23 +5502,12 @@ static void skl_atomic_update_crtc_wm(struct intel_atomic_state *state, struct intel_crtc *crtc = to_intel_crtc(cstate->base.crtc); struct drm_i915_private *dev_priv = to_i915(state->base.dev); struct skl_pipe_wm *pipe_wm = &cstate->wm.skl.optimal; - const struct skl_ddb_allocation *ddb = &state->wm_results.ddb; enum pipe pipe = crtc->pipe; - enum plane_id plane_id; if (!(state->wm_results.dirty_pipes & drm_crtc_mask(&crtc->base))) return; I915_WRITE(PIPE_WM_LINETIME(pipe), pipe_wm->linetime); - - for_each_plane_id_on_crtc(crtc, plane_id) { - if (plane_id != PLANE_CURSOR) - skl_write_plane_wm(crtc, &pipe_wm->planes[plane_id], - ddb, plane_id); - else - skl_write_cursor_wm(crtc, &pipe_wm->planes[plane_id], - ddb); - } } static void skl_initial_wm(struct intel_atomic_state *state, @@ -5439,8 +5517,6 @@ static void skl_initial_wm(struct intel_atomic_state *state, struct drm_device *dev = intel_crtc->base.dev; struct drm_i915_private *dev_priv = to_i915(dev); struct skl_ddb_values *results = &state->wm_results; - struct skl_ddb_values *hw_vals = &dev_priv->wm.skl_hw; - enum pipe pipe = intel_crtc->pipe; if ((results->dirty_pipes & drm_crtc_mask(&intel_crtc->base)) == 0) return; @@ -5450,21 +5526,16 @@ static void skl_initial_wm(struct intel_atomic_state *state, if (cstate->base.active_changed) skl_atomic_update_crtc_wm(state, cstate); - memcpy(hw_vals->ddb.uv_plane[pipe], results->ddb.uv_plane[pipe], - sizeof(hw_vals->ddb.uv_plane[pipe])); - memcpy(hw_vals->ddb.plane[pipe], results->ddb.plane[pipe], - sizeof(hw_vals->ddb.plane[pipe])); - mutex_unlock(&dev_priv->wm.wm_mutex); } -static void ilk_compute_wm_config(struct drm_device *dev, +static void ilk_compute_wm_config(struct drm_i915_private *dev_priv, struct intel_wm_config *config) { struct intel_crtc *crtc; /* Compute the currently _active_ config */ - for_each_intel_crtc(dev, crtc) { + for_each_intel_crtc(&dev_priv->drm, crtc) { const struct intel_pipe_wm *wm = &crtc->wm.active.ilk; if (!wm->pipe_enabled) @@ -5478,25 +5549,24 @@ static void ilk_compute_wm_config(struct drm_device *dev, static void ilk_program_watermarks(struct drm_i915_private *dev_priv) { - struct drm_device *dev = &dev_priv->drm; struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm; struct ilk_wm_maximums max; struct intel_wm_config config = {}; struct ilk_wm_values results = {}; enum intel_ddb_partitioning partitioning; - ilk_compute_wm_config(dev, &config); + ilk_compute_wm_config(dev_priv, &config); - ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max); - ilk_wm_merge(dev, &config, &max, &lp_wm_1_2); + ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max); + ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2); /* 5/6 split only in single pipe config on IVB+ */ if (INTEL_GEN(dev_priv) >= 7 && config.num_pipes_active == 1 && config.sprites_enabled) { - ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max); - ilk_wm_merge(dev, &config, &max, &lp_wm_5_6); + ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max); + ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6); - best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6); + best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6); } else { best_lp_wm = &lp_wm_1_2; } @@ -5504,7 +5574,7 @@ static void ilk_program_watermarks(struct drm_i915_private *dev_priv) partitioning = (best_lp_wm == &lp_wm_1_2) ? INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6; - ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results); + ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results); ilk_write_wm_values(dev_priv, &results); } @@ -5535,7 +5605,7 @@ static void ilk_optimize_watermarks(struct intel_atomic_state *state, mutex_unlock(&dev_priv->wm.wm_mutex); } -static inline void skl_wm_level_from_reg_val(uint32_t val, +static inline void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level) { level->plane_en = val & PLANE_WM_EN; @@ -5544,19 +5614,18 @@ static inline void skl_wm_level_from_reg_val(uint32_t val, PLANE_WM_LINES_MASK; } -void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc, +void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc, struct skl_pipe_wm *out) { - struct drm_i915_private *dev_priv = to_i915(crtc->dev); - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - enum pipe pipe = intel_crtc->pipe; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; int level, max_level; enum plane_id plane_id; - uint32_t val; + u32 val; max_level = ilk_wm_max_level(dev_priv); - for_each_plane_id_on_crtc(intel_crtc, plane_id) { + for_each_plane_id_on_crtc(crtc, plane_id) { struct skl_plane_wm *wm = &out->planes[plane_id]; for (level = 0; level <= max_level; level++) { @@ -5576,54 +5645,43 @@ void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc, skl_wm_level_from_reg_val(val, &wm->trans_wm); } - if (!intel_crtc->active) + if (!crtc->active) return; out->linetime = I915_READ(PIPE_WM_LINETIME(pipe)); } -void skl_wm_get_hw_state(struct drm_device *dev) +void skl_wm_get_hw_state(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); struct skl_ddb_values *hw = &dev_priv->wm.skl_hw; struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb; - struct drm_crtc *crtc; - struct intel_crtc *intel_crtc; + struct intel_crtc *crtc; struct intel_crtc_state *cstate; skl_ddb_get_hw_state(dev_priv, ddb); - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { - intel_crtc = to_intel_crtc(crtc); - cstate = to_intel_crtc_state(crtc->state); + for_each_intel_crtc(&dev_priv->drm, crtc) { + cstate = to_intel_crtc_state(crtc->base.state); skl_pipe_wm_get_hw_state(crtc, &cstate->wm.skl.optimal); - if (intel_crtc->active) - hw->dirty_pipes |= drm_crtc_mask(crtc); + if (crtc->active) + hw->dirty_pipes |= drm_crtc_mask(&crtc->base); } if (dev_priv->active_crtcs) { /* Fully recompute DDB on first atomic commit */ dev_priv->wm.distrust_bios_wm = true; - } else { - /* - * Easy/common case; just sanitize DDB now if everything off - * Keep dbuf slice info intact - */ - memset(ddb->plane, 0, sizeof(ddb->plane)); - memset(ddb->uv_plane, 0, sizeof(ddb->uv_plane)); } } -static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc) +static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc) { - struct drm_device *dev = crtc->dev; + struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = to_i915(dev); struct ilk_wm_values *hw = &dev_priv->wm.hw; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state); + struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->base.state); struct intel_pipe_wm *active = &cstate->wm.ilk.optimal; - enum pipe pipe = intel_crtc->pipe; + enum pipe pipe = crtc->pipe; static const i915_reg_t wm0_pipe_reg[] = { [PIPE_A] = WM0_PIPEA_ILK, [PIPE_B] = WM0_PIPEB_ILK, @@ -5636,7 +5694,7 @@ static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc) memset(active, 0, sizeof(*active)); - active->pipe_enabled = intel_crtc->active; + active->pipe_enabled = crtc->active; if (active->pipe_enabled) { u32 tmp = hw->wm_pipe[pipe]; @@ -5664,7 +5722,7 @@ static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc) active->wm[level].enable = true; } - intel_crtc->wm.active.ilk = *active; + crtc->wm.active.ilk = *active; } #define _FW_WM(value, plane) \ @@ -5675,7 +5733,7 @@ static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc) static void g4x_read_wm_values(struct drm_i915_private *dev_priv, struct g4x_wm_values *wm) { - uint32_t tmp; + u32 tmp; tmp = I915_READ(DSPFW1); wm->sr.plane = _FW_WM(tmp, SR); @@ -5702,7 +5760,7 @@ static void vlv_read_wm_values(struct drm_i915_private *dev_priv, struct vlv_wm_values *wm) { enum pipe pipe; - uint32_t tmp; + u32 tmp; for_each_pipe(dev_priv, pipe) { tmp = I915_READ(VLV_DDL(pipe)); @@ -5774,9 +5832,8 @@ static void vlv_read_wm_values(struct drm_i915_private *dev_priv, #undef _FW_WM #undef _FW_WM_VLV -void g4x_wm_get_hw_state(struct drm_device *dev) +void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); struct g4x_wm_values *wm = &dev_priv->wm.g4x; struct intel_crtc *crtc; @@ -5784,7 +5841,7 @@ void g4x_wm_get_hw_state(struct drm_device *dev) wm->cxsr = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN; - for_each_intel_crtc(dev, crtc) { + for_each_intel_crtc(&dev_priv->drm, crtc) { struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state); struct g4x_wm_state *active = &crtc->wm.active.g4x; @@ -5915,9 +5972,8 @@ void g4x_wm_sanitize(struct drm_i915_private *dev_priv) mutex_unlock(&dev_priv->wm.wm_mutex); } -void vlv_wm_get_hw_state(struct drm_device *dev) +void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); struct vlv_wm_values *wm = &dev_priv->wm.vlv; struct intel_crtc *crtc; u32 val; @@ -5961,7 +6017,7 @@ void vlv_wm_get_hw_state(struct drm_device *dev) mutex_unlock(&dev_priv->pcu_lock); } - for_each_intel_crtc(dev, crtc) { + for_each_intel_crtc(&dev_priv->drm, crtc) { struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state); struct vlv_wm_state *active = &crtc->wm.active.vlv; @@ -6078,15 +6134,14 @@ static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv) */ } -void ilk_wm_get_hw_state(struct drm_device *dev) +void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); struct ilk_wm_values *hw = &dev_priv->wm.hw; - struct drm_crtc *crtc; + struct intel_crtc *crtc; ilk_init_lp_watermarks(dev_priv); - for_each_crtc(dev, crtc) + for_each_intel_crtc(&dev_priv->drm, crtc) ilk_pipe_wm_get_hw_state(crtc); hw->wm_lp[0] = I915_READ(WM1_LP_ILK); @@ -6155,14 +6210,8 @@ void intel_enable_ipc(struct drm_i915_private *dev_priv) { u32 val; - /* Display WA #0477 WaDisableIPC: skl */ - if (IS_SKYLAKE(dev_priv)) - dev_priv->ipc_enabled = false; - - /* Display WA #1141: SKL:all KBL:all CFL */ - if ((IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) && - !dev_priv->dram_info.symmetric_memory) - dev_priv->ipc_enabled = false; + if (!HAS_IPC(dev_priv)) + return; val = I915_READ(DISP_ARB_CTL2); @@ -6176,11 +6225,15 @@ void intel_enable_ipc(struct drm_i915_private *dev_priv) void intel_init_ipc(struct drm_i915_private *dev_priv) { - dev_priv->ipc_enabled = false; if (!HAS_IPC(dev_priv)) return; - dev_priv->ipc_enabled = true; + /* Display WA #1141: SKL:all KBL:all CFL */ + if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) + dev_priv->ipc_enabled = dev_priv->dram_info.symmetric_memory; + else + dev_priv->ipc_enabled = true; + intel_enable_ipc(dev_priv); } @@ -6189,10 +6242,6 @@ void intel_init_ipc(struct drm_i915_private *dev_priv) */ DEFINE_SPINLOCK(mchdev_lock); -/* Global for IPS driver to get at the current i915 device. Protected by - * mchdev_lock. */ -static struct drm_i915_private *i915_mch_dev; - bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val) { u16 rgvswctl; @@ -6655,7 +6704,7 @@ void gen6_rps_boost(struct i915_request *rq, if (!rps->enabled) return; - if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags)) + if (i915_request_signaled(rq)) return; /* Serializes with i915_request_retire() */ @@ -6899,7 +6948,7 @@ static void gen9_enable_rps(struct drm_i915_private *dev_priv) intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); /* Program defaults and thresholds for RPS */ - if (IS_GEN9(dev_priv)) + if (IS_GEN(dev_priv, 9)) I915_WRITE(GEN6_RC_VIDEO_FREQ, GEN9_FREQUENCY(dev_priv->gt_pm.rps.rp1_freq)); @@ -7135,9 +7184,9 @@ static void gen6_enable_rc6(struct drm_i915_private *dev_priv) rc6vids = 0; ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids); - if (IS_GEN6(dev_priv) && ret) { + if (IS_GEN(dev_priv, 6) && ret) { DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n"); - } else if (IS_GEN6(dev_priv) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) { + } else if (IS_GEN(dev_priv, 6) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) { DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n", GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450); rc6vids &= 0xffff00; @@ -7262,7 +7311,7 @@ static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv) val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE); - switch (INTEL_INFO(dev_priv)->sseu.eu_total) { + switch (RUNTIME_INFO(dev_priv)->sseu.eu_total) { case 8: /* (2 * 4) config */ rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT); @@ -7835,16 +7884,17 @@ static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv) unsigned long i915_chipset_val(struct drm_i915_private *dev_priv) { - unsigned long val; + intel_wakeref_t wakeref; + unsigned long val = 0; - if (!IS_GEN5(dev_priv)) + if (!IS_GEN(dev_priv, 5)) return 0; - spin_lock_irq(&mchdev_lock); - - val = __i915_chipset_val(dev_priv); - - spin_unlock_irq(&mchdev_lock); + with_intel_runtime_pm(dev_priv, wakeref) { + spin_lock_irq(&mchdev_lock); + val = __i915_chipset_val(dev_priv); + spin_unlock_irq(&mchdev_lock); + } return val; } @@ -7921,14 +7971,16 @@ static void __i915_update_gfx_val(struct drm_i915_private *dev_priv) void i915_update_gfx_val(struct drm_i915_private *dev_priv) { - if (!IS_GEN5(dev_priv)) - return; - - spin_lock_irq(&mchdev_lock); + intel_wakeref_t wakeref; - __i915_update_gfx_val(dev_priv); + if (!IS_GEN(dev_priv, 5)) + return; - spin_unlock_irq(&mchdev_lock); + with_intel_runtime_pm(dev_priv, wakeref) { + spin_lock_irq(&mchdev_lock); + __i915_update_gfx_val(dev_priv); + spin_unlock_irq(&mchdev_lock); + } } static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv) @@ -7970,18 +8022,34 @@ static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv) unsigned long i915_gfx_val(struct drm_i915_private *dev_priv) { - unsigned long val; + intel_wakeref_t wakeref; + unsigned long val = 0; - if (!IS_GEN5(dev_priv)) + if (!IS_GEN(dev_priv, 5)) return 0; - spin_lock_irq(&mchdev_lock); + with_intel_runtime_pm(dev_priv, wakeref) { + spin_lock_irq(&mchdev_lock); + val = __i915_gfx_val(dev_priv); + spin_unlock_irq(&mchdev_lock); + } + + return val; +} - val = __i915_gfx_val(dev_priv); +static struct drm_i915_private *i915_mch_dev; - spin_unlock_irq(&mchdev_lock); +static struct drm_i915_private *mchdev_get(void) +{ + struct drm_i915_private *i915; - return val; + rcu_read_lock(); + i915 = i915_mch_dev; + if (!kref_get_unless_zero(&i915->drm.ref)) + i915 = NULL; + rcu_read_unlock(); + + return i915; } /** @@ -7992,23 +8060,24 @@ unsigned long i915_gfx_val(struct drm_i915_private *dev_priv) */ unsigned long i915_read_mch_val(void) { - struct drm_i915_private *dev_priv; - unsigned long chipset_val, graphics_val, ret = 0; - - spin_lock_irq(&mchdev_lock); - if (!i915_mch_dev) - goto out_unlock; - dev_priv = i915_mch_dev; - - chipset_val = __i915_chipset_val(dev_priv); - graphics_val = __i915_gfx_val(dev_priv); + struct drm_i915_private *i915; + unsigned long chipset_val = 0; + unsigned long graphics_val = 0; + intel_wakeref_t wakeref; - ret = chipset_val + graphics_val; + i915 = mchdev_get(); + if (!i915) + return 0; -out_unlock: - spin_unlock_irq(&mchdev_lock); + with_intel_runtime_pm(i915, wakeref) { + spin_lock_irq(&mchdev_lock); + chipset_val = __i915_chipset_val(i915); + graphics_val = __i915_gfx_val(i915); + spin_unlock_irq(&mchdev_lock); + } - return ret; + drm_dev_put(&i915->drm); + return chipset_val + graphics_val; } EXPORT_SYMBOL_GPL(i915_read_mch_val); @@ -8019,23 +8088,19 @@ EXPORT_SYMBOL_GPL(i915_read_mch_val); */ bool i915_gpu_raise(void) { - struct drm_i915_private *dev_priv; - bool ret = true; - - spin_lock_irq(&mchdev_lock); - if (!i915_mch_dev) { - ret = false; - goto out_unlock; - } - dev_priv = i915_mch_dev; + struct drm_i915_private *i915; - if (dev_priv->ips.max_delay > dev_priv->ips.fmax) - dev_priv->ips.max_delay--; + i915 = mchdev_get(); + if (!i915) + return false; -out_unlock: + spin_lock_irq(&mchdev_lock); + if (i915->ips.max_delay > i915->ips.fmax) + i915->ips.max_delay--; spin_unlock_irq(&mchdev_lock); - return ret; + drm_dev_put(&i915->drm); + return true; } EXPORT_SYMBOL_GPL(i915_gpu_raise); @@ -8047,23 +8112,19 @@ EXPORT_SYMBOL_GPL(i915_gpu_raise); */ bool i915_gpu_lower(void) { - struct drm_i915_private *dev_priv; - bool ret = true; - - spin_lock_irq(&mchdev_lock); - if (!i915_mch_dev) { - ret = false; - goto out_unlock; - } - dev_priv = i915_mch_dev; + struct drm_i915_private *i915; - if (dev_priv->ips.max_delay < dev_priv->ips.min_delay) - dev_priv->ips.max_delay++; + i915 = mchdev_get(); + if (!i915) + return false; -out_unlock: + spin_lock_irq(&mchdev_lock); + if (i915->ips.max_delay < i915->ips.min_delay) + i915->ips.max_delay++; spin_unlock_irq(&mchdev_lock); - return ret; + drm_dev_put(&i915->drm); + return true; } EXPORT_SYMBOL_GPL(i915_gpu_lower); @@ -8074,13 +8135,16 @@ EXPORT_SYMBOL_GPL(i915_gpu_lower); */ bool i915_gpu_busy(void) { - bool ret = false; + struct drm_i915_private *i915; + bool ret; - spin_lock_irq(&mchdev_lock); - if (i915_mch_dev) - ret = i915_mch_dev->gt.awake; - spin_unlock_irq(&mchdev_lock); + i915 = mchdev_get(); + if (!i915) + return false; + ret = i915->gt.awake; + + drm_dev_put(&i915->drm); return ret; } EXPORT_SYMBOL_GPL(i915_gpu_busy); @@ -8093,24 +8157,19 @@ EXPORT_SYMBOL_GPL(i915_gpu_busy); */ bool i915_gpu_turbo_disable(void) { - struct drm_i915_private *dev_priv; - bool ret = true; - - spin_lock_irq(&mchdev_lock); - if (!i915_mch_dev) { - ret = false; - goto out_unlock; - } - dev_priv = i915_mch_dev; - - dev_priv->ips.max_delay = dev_priv->ips.fstart; + struct drm_i915_private *i915; + bool ret; - if (!ironlake_set_drps(dev_priv, dev_priv->ips.fstart)) - ret = false; + i915 = mchdev_get(); + if (!i915) + return false; -out_unlock: + spin_lock_irq(&mchdev_lock); + i915->ips.max_delay = i915->ips.fstart; + ret = ironlake_set_drps(i915, i915->ips.fstart); spin_unlock_irq(&mchdev_lock); + drm_dev_put(&i915->drm); return ret; } EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable); @@ -8139,18 +8198,14 @@ void intel_gpu_ips_init(struct drm_i915_private *dev_priv) { /* We only register the i915 ips part with intel-ips once everything is * set up, to avoid intel-ips sneaking in and reading bogus values. */ - spin_lock_irq(&mchdev_lock); - i915_mch_dev = dev_priv; - spin_unlock_irq(&mchdev_lock); + rcu_assign_pointer(i915_mch_dev, dev_priv); ips_ping_for_i915_load(); } void intel_gpu_ips_teardown(void) { - spin_lock_irq(&mchdev_lock); - i915_mch_dev = NULL; - spin_unlock_irq(&mchdev_lock); + rcu_assign_pointer(i915_mch_dev, NULL); } static void intel_init_emon(struct drm_i915_private *dev_priv) @@ -8260,7 +8315,7 @@ void intel_init_gt_powersave(struct drm_i915_private *dev_priv) intel_freq_opcode(dev_priv, 450)); /* After setting max-softlimit, find the overclock max freq */ - if (IS_GEN6(dev_priv) || + if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) { u32 params = 0; @@ -8489,7 +8544,7 @@ static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv) static void ilk_init_clock_gating(struct drm_i915_private *dev_priv) { - uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; + u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; /* * Required for FBC @@ -8561,7 +8616,7 @@ static void ilk_init_clock_gating(struct drm_i915_private *dev_priv) static void cpt_init_clock_gating(struct drm_i915_private *dev_priv) { int pipe; - uint32_t val; + u32 val; /* * On Ibex Peak and Cougar Point, we need to disable clock @@ -8596,7 +8651,7 @@ static void cpt_init_clock_gating(struct drm_i915_private *dev_priv) static void gen6_check_mch_setup(struct drm_i915_private *dev_priv) { - uint32_t tmp; + u32 tmp; tmp = I915_READ(MCH_SSKPD); if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL) @@ -8606,7 +8661,7 @@ static void gen6_check_mch_setup(struct drm_i915_private *dev_priv) static void gen6_init_clock_gating(struct drm_i915_private *dev_priv) { - uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; + u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate); @@ -8700,7 +8755,7 @@ static void gen6_init_clock_gating(struct drm_i915_private *dev_priv) static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv) { - uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE); + u32 reg = I915_READ(GEN7_FF_THREAD_MODE); /* * WaVSThreadDispatchOverride:ivb,vlv @@ -8736,7 +8791,7 @@ static void lpt_init_clock_gating(struct drm_i915_private *dev_priv) static void lpt_suspend_hw(struct drm_i915_private *dev_priv) { if (HAS_PCH_LPT_LP(dev_priv)) { - uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D); + u32 val = I915_READ(SOUTH_DSPCLK_GATE_D); val &= ~PCH_LP_PARTITION_LEVEL_DISABLE; I915_WRITE(SOUTH_DSPCLK_GATE_D, val); @@ -8774,6 +8829,10 @@ static void icl_init_clock_gating(struct drm_i915_private *dev_priv) /* This is not an Wa. Enable to reduce Sampler power */ I915_WRITE(GEN10_DFR_RATIO_EN_AND_CHICKEN, I915_READ(GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE); + + /* WaEnable32PlaneMode:icl */ + I915_WRITE(GEN9_CSFE_CHICKEN1_RCS, + _MASKED_BIT_ENABLE(GEN11_ENABLE_32_PLANE_MODE)); } static void cnp_init_clock_gating(struct drm_i915_private *dev_priv) @@ -8970,7 +9029,7 @@ static void hsw_init_clock_gating(struct drm_i915_private *dev_priv) static void ivb_init_clock_gating(struct drm_i915_private *dev_priv) { - uint32_t snpcr; + u32 snpcr; I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE); @@ -9179,7 +9238,7 @@ static void chv_init_clock_gating(struct drm_i915_private *dev_priv) static void g4x_init_clock_gating(struct drm_i915_private *dev_priv) { - uint32_t dspclk_gate; + u32 dspclk_gate; I915_WRITE(RENCLK_GATE_D1, 0); I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE | @@ -9326,9 +9385,9 @@ void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv) dev_priv->display.init_clock_gating = ivb_init_clock_gating; else if (IS_VALLEYVIEW(dev_priv)) dev_priv->display.init_clock_gating = vlv_init_clock_gating; - else if (IS_GEN6(dev_priv)) + else if (IS_GEN(dev_priv, 6)) dev_priv->display.init_clock_gating = gen6_init_clock_gating; - else if (IS_GEN5(dev_priv)) + else if (IS_GEN(dev_priv, 5)) dev_priv->display.init_clock_gating = ilk_init_clock_gating; else if (IS_G4X(dev_priv)) dev_priv->display.init_clock_gating = g4x_init_clock_gating; @@ -9336,11 +9395,11 @@ void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv) dev_priv->display.init_clock_gating = i965gm_init_clock_gating; else if (IS_I965G(dev_priv)) dev_priv->display.init_clock_gating = i965g_init_clock_gating; - else if (IS_GEN3(dev_priv)) + else if (IS_GEN(dev_priv, 3)) dev_priv->display.init_clock_gating = gen3_init_clock_gating; else if (IS_I85X(dev_priv) || IS_I865G(dev_priv)) dev_priv->display.init_clock_gating = i85x_init_clock_gating; - else if (IS_GEN2(dev_priv)) + else if (IS_GEN(dev_priv, 2)) dev_priv->display.init_clock_gating = i830_init_clock_gating; else { MISSING_CASE(INTEL_DEVID(dev_priv)); @@ -9351,12 +9410,10 @@ void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv) /* Set up chip specific power management-related functions */ void intel_init_pm(struct drm_i915_private *dev_priv) { - intel_fbc_init(dev_priv); - /* For cxsr */ if (IS_PINEVIEW(dev_priv)) i915_pineview_get_mem_freq(dev_priv); - else if (IS_GEN5(dev_priv)) + else if (IS_GEN(dev_priv, 5)) i915_ironlake_get_mem_freq(dev_priv); /* For FIFO watermark updates */ @@ -9368,9 +9425,9 @@ void intel_init_pm(struct drm_i915_private *dev_priv) } else if (HAS_PCH_SPLIT(dev_priv)) { ilk_setup_wm_latency(dev_priv); - if ((IS_GEN5(dev_priv) && dev_priv->wm.pri_latency[1] && + if ((IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[1] && dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) || - (!IS_GEN5(dev_priv) && dev_priv->wm.pri_latency[0] && + (!IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[0] && dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) { dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm; dev_priv->display.compute_intermediate_wm = @@ -9411,12 +9468,12 @@ void intel_init_pm(struct drm_i915_private *dev_priv) dev_priv->display.update_wm = NULL; } else dev_priv->display.update_wm = pineview_update_wm; - } else if (IS_GEN4(dev_priv)) { + } else if (IS_GEN(dev_priv, 4)) { dev_priv->display.update_wm = i965_update_wm; - } else if (IS_GEN3(dev_priv)) { + } else if (IS_GEN(dev_priv, 3)) { dev_priv->display.update_wm = i9xx_update_wm; dev_priv->display.get_fifo_size = i9xx_get_fifo_size; - } else if (IS_GEN2(dev_priv)) { + } else if (IS_GEN(dev_priv, 2)) { if (INTEL_INFO(dev_priv)->num_pipes == 1) { dev_priv->display.update_wm = i845_update_wm; dev_priv->display.get_fifo_size = i845_get_fifo_size; @@ -9431,7 +9488,7 @@ void intel_init_pm(struct drm_i915_private *dev_priv) static inline int gen6_check_mailbox_status(struct drm_i915_private *dev_priv) { - uint32_t flags = + u32 flags = I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_ERROR_MASK; switch (flags) { @@ -9454,7 +9511,7 @@ static inline int gen6_check_mailbox_status(struct drm_i915_private *dev_priv) static inline int gen7_check_mailbox_status(struct drm_i915_private *dev_priv) { - uint32_t flags = + u32 flags = I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_ERROR_MASK; switch (flags) { diff --git a/drivers/gpu/drm/i915/intel_psr.c b/drivers/gpu/drm/i915/intel_psr.c index b6838b525502..84a0fb981561 100644 --- a/drivers/gpu/drm/i915/intel_psr.c +++ b/drivers/gpu/drm/i915/intel_psr.c @@ -51,7 +51,6 @@ * must be correctly synchronized/cancelled when shutting down the pipe." */ -#include <drm/drmP.h> #include "intel_drv.h" #include "i915_drv.h" @@ -71,33 +70,58 @@ static bool psr_global_enabled(u32 debug) static bool intel_psr2_enabled(struct drm_i915_private *dev_priv, const struct intel_crtc_state *crtc_state) { + /* Cannot enable DSC and PSR2 simultaneously */ + WARN_ON(crtc_state->dsc_params.compression_enable && + crtc_state->has_psr2); + switch (dev_priv->psr.debug & I915_PSR_DEBUG_MODE_MASK) { + case I915_PSR_DEBUG_DISABLE: case I915_PSR_DEBUG_FORCE_PSR1: return false; + case I915_PSR_DEBUG_DEFAULT: + if (i915_modparams.enable_psr <= 0) + return false; default: return crtc_state->has_psr2; } } +static int edp_psr_shift(enum transcoder cpu_transcoder) +{ + switch (cpu_transcoder) { + case TRANSCODER_A: + return EDP_PSR_TRANSCODER_A_SHIFT; + case TRANSCODER_B: + return EDP_PSR_TRANSCODER_B_SHIFT; + case TRANSCODER_C: + return EDP_PSR_TRANSCODER_C_SHIFT; + default: + MISSING_CASE(cpu_transcoder); + /* fallthrough */ + case TRANSCODER_EDP: + return EDP_PSR_TRANSCODER_EDP_SHIFT; + } +} + void intel_psr_irq_control(struct drm_i915_private *dev_priv, u32 debug) { u32 debug_mask, mask; + enum transcoder cpu_transcoder; + u32 transcoders = BIT(TRANSCODER_EDP); - mask = EDP_PSR_ERROR(TRANSCODER_EDP); - debug_mask = EDP_PSR_POST_EXIT(TRANSCODER_EDP) | - EDP_PSR_PRE_ENTRY(TRANSCODER_EDP); - - if (INTEL_GEN(dev_priv) >= 8) { - mask |= EDP_PSR_ERROR(TRANSCODER_A) | - EDP_PSR_ERROR(TRANSCODER_B) | - EDP_PSR_ERROR(TRANSCODER_C); - - debug_mask |= EDP_PSR_POST_EXIT(TRANSCODER_A) | - EDP_PSR_PRE_ENTRY(TRANSCODER_A) | - EDP_PSR_POST_EXIT(TRANSCODER_B) | - EDP_PSR_PRE_ENTRY(TRANSCODER_B) | - EDP_PSR_POST_EXIT(TRANSCODER_C) | - EDP_PSR_PRE_ENTRY(TRANSCODER_C); + if (INTEL_GEN(dev_priv) >= 8) + transcoders |= BIT(TRANSCODER_A) | + BIT(TRANSCODER_B) | + BIT(TRANSCODER_C); + + debug_mask = 0; + mask = 0; + for_each_cpu_transcoder_masked(dev_priv, cpu_transcoder, transcoders) { + int shift = edp_psr_shift(cpu_transcoder); + + mask |= EDP_PSR_ERROR(shift); + debug_mask |= EDP_PSR_POST_EXIT(shift) | + EDP_PSR_PRE_ENTRY(shift); } if (debug & I915_PSR_DEBUG_IRQ) @@ -148,6 +172,7 @@ void intel_psr_irq_handler(struct drm_i915_private *dev_priv, u32 psr_iir) u32 transcoders = BIT(TRANSCODER_EDP); enum transcoder cpu_transcoder; ktime_t time_ns = ktime_get(); + u32 mask = 0; if (INTEL_GEN(dev_priv) >= 8) transcoders |= BIT(TRANSCODER_A) | @@ -155,18 +180,32 @@ void intel_psr_irq_handler(struct drm_i915_private *dev_priv, u32 psr_iir) BIT(TRANSCODER_C); for_each_cpu_transcoder_masked(dev_priv, cpu_transcoder, transcoders) { - /* FIXME: Exit PSR and link train manually when this happens. */ - if (psr_iir & EDP_PSR_ERROR(cpu_transcoder)) - DRM_DEBUG_KMS("[transcoder %s] PSR aux error\n", - transcoder_name(cpu_transcoder)); + int shift = edp_psr_shift(cpu_transcoder); + + if (psr_iir & EDP_PSR_ERROR(shift)) { + DRM_WARN("[transcoder %s] PSR aux error\n", + transcoder_name(cpu_transcoder)); + + dev_priv->psr.irq_aux_error = true; - if (psr_iir & EDP_PSR_PRE_ENTRY(cpu_transcoder)) { + /* + * If this interruption is not masked it will keep + * interrupting so fast that it prevents the scheduled + * work to run. + * Also after a PSR error, we don't want to arm PSR + * again so we don't care about unmask the interruption + * or unset irq_aux_error. + */ + mask |= EDP_PSR_ERROR(shift); + } + + if (psr_iir & EDP_PSR_PRE_ENTRY(shift)) { dev_priv->psr.last_entry_attempt = time_ns; DRM_DEBUG_KMS("[transcoder %s] PSR entry attempt in 2 vblanks\n", transcoder_name(cpu_transcoder)); } - if (psr_iir & EDP_PSR_POST_EXIT(cpu_transcoder)) { + if (psr_iir & EDP_PSR_POST_EXIT(shift)) { dev_priv->psr.last_exit = time_ns; DRM_DEBUG_KMS("[transcoder %s] PSR exit completed\n", transcoder_name(cpu_transcoder)); @@ -180,11 +219,18 @@ void intel_psr_irq_handler(struct drm_i915_private *dev_priv, u32 psr_iir) } } } + + if (mask) { + mask |= I915_READ(EDP_PSR_IMR); + I915_WRITE(EDP_PSR_IMR, mask); + + schedule_work(&dev_priv->psr.work); + } } static bool intel_dp_get_colorimetry_status(struct intel_dp *intel_dp) { - uint8_t dprx = 0; + u8 dprx = 0; if (drm_dp_dpcd_readb(&intel_dp->aux, DP_DPRX_FEATURE_ENUMERATION_LIST, &dprx) != 1) @@ -194,7 +240,7 @@ static bool intel_dp_get_colorimetry_status(struct intel_dp *intel_dp) static bool intel_dp_get_alpm_status(struct intel_dp *intel_dp) { - uint8_t alpm_caps = 0; + u8 alpm_caps = 0; if (drm_dp_dpcd_readb(&intel_dp->aux, DP_RECEIVER_ALPM_CAP, &alpm_caps) != 1) @@ -214,6 +260,32 @@ static u8 intel_dp_get_sink_sync_latency(struct intel_dp *intel_dp) return val; } +static u16 intel_dp_get_su_x_granulartiy(struct intel_dp *intel_dp) +{ + u16 val; + ssize_t r; + + /* + * Returning the default X granularity if granularity not required or + * if DPCD read fails + */ + if (!(intel_dp->psr_dpcd[1] & DP_PSR2_SU_GRANULARITY_REQUIRED)) + return 4; + + r = drm_dp_dpcd_read(&intel_dp->aux, DP_PSR2_SU_X_GRANULARITY, &val, 2); + if (r != 2) + DRM_DEBUG_KMS("Unable to read DP_PSR2_SU_X_GRANULARITY\n"); + + /* + * Spec says that if the value read is 0 the default granularity should + * be used instead. + */ + if (r != 2 || val == 0) + val = 4; + + return val; +} + void intel_psr_init_dpcd(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = @@ -227,10 +299,16 @@ void intel_psr_init_dpcd(struct intel_dp *intel_dp) DRM_DEBUG_KMS("eDP panel supports PSR version %x\n", intel_dp->psr_dpcd[0]); + if (drm_dp_has_quirk(&intel_dp->desc, DP_DPCD_QUIRK_NO_PSR)) { + DRM_DEBUG_KMS("PSR support not currently available for this panel\n"); + return; + } + if (!(intel_dp->edp_dpcd[1] & DP_EDP_SET_POWER_CAP)) { DRM_DEBUG_KMS("Panel lacks power state control, PSR cannot be enabled\n"); return; } + dev_priv->psr.sink_support = true; dev_priv->psr.sink_sync_latency = intel_dp_get_sink_sync_latency(intel_dp); @@ -262,6 +340,8 @@ void intel_psr_init_dpcd(struct intel_dp *intel_dp) if (dev_priv->psr.sink_psr2_support) { dev_priv->psr.colorimetry_support = intel_dp_get_colorimetry_status(intel_dp); + dev_priv->psr.su_x_granularity = + intel_dp_get_su_x_granulartiy(intel_dp); } } } @@ -294,7 +374,8 @@ static void intel_psr_setup_vsc(struct intel_dp *intel_dp, psr_vsc.sdp_header.HB3 = 0x8; } - intel_dig_port->write_infoframe(&intel_dig_port->base.base, crtc_state, + intel_dig_port->write_infoframe(&intel_dig_port->base, + crtc_state, DP_SDP_VSC, &psr_vsc, sizeof(psr_vsc)); } @@ -303,7 +384,7 @@ static void hsw_psr_setup_aux(struct intel_dp *intel_dp) struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); u32 aux_clock_divider, aux_ctl; int i; - static const uint8_t aux_msg[] = { + static const u8 aux_msg[] = { [0] = DP_AUX_NATIVE_WRITE << 4, [1] = DP_SET_POWER >> 8, [2] = DP_SET_POWER & 0xff, @@ -340,13 +421,15 @@ static void intel_psr_enable_sink(struct intel_dp *intel_dp) if (dev_priv->psr.psr2_enabled) { drm_dp_dpcd_writeb(&intel_dp->aux, DP_RECEIVER_ALPM_CONFIG, DP_ALPM_ENABLE); - dpcd_val |= DP_PSR_ENABLE_PSR2; + dpcd_val |= DP_PSR_ENABLE_PSR2 | DP_PSR_IRQ_HPD_WITH_CRC_ERRORS; + } else { + if (dev_priv->psr.link_standby) + dpcd_val |= DP_PSR_MAIN_LINK_ACTIVE; + + if (INTEL_GEN(dev_priv) >= 8) + dpcd_val |= DP_PSR_CRC_VERIFICATION; } - if (dev_priv->psr.link_standby) - dpcd_val |= DP_PSR_MAIN_LINK_ACTIVE; - if (!dev_priv->psr.psr2_enabled && INTEL_GEN(dev_priv) >= 8) - dpcd_val |= DP_PSR_CRC_VERIFICATION; drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, dpcd_val); drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, DP_SET_POWER_D0); @@ -420,9 +503,6 @@ static void hsw_activate_psr2(struct intel_dp *intel_dp) idle_frames = max(idle_frames, dev_priv->psr.sink_sync_latency + 1); val = idle_frames << EDP_PSR2_IDLE_FRAME_SHIFT; - /* FIXME: selective update is probably totally broken because it doesn't - * mesh at all with our frontbuffer tracking. And the hw alone isn't - * good enough. */ val |= EDP_PSR2_ENABLE | EDP_SU_TRACK_ENABLE; if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) val |= EDP_Y_COORDINATE_ENABLE; @@ -458,10 +538,20 @@ static bool intel_psr2_config_valid(struct intel_dp *intel_dp, if (!dev_priv->psr.sink_psr2_support) return false; + /* + * DSC and PSR2 cannot be enabled simultaneously. If a requested + * resolution requires DSC to be enabled, priority is given to DSC + * over PSR2. + */ + if (crtc_state->dsc_params.compression_enable) { + DRM_DEBUG_KMS("PSR2 cannot be enabled since DSC is enabled\n"); + return false; + } + if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) { psr_max_h = 4096; psr_max_v = 2304; - } else if (IS_GEN9(dev_priv)) { + } else if (IS_GEN(dev_priv, 9)) { psr_max_h = 3640; psr_max_v = 2304; } @@ -473,6 +563,18 @@ static bool intel_psr2_config_valid(struct intel_dp *intel_dp, return false; } + /* + * HW sends SU blocks of size four scan lines, which means the starting + * X coordinate and Y granularity requirements will always be met. We + * only need to validate the SU block width is a multiple of + * x granularity. + */ + if (crtc_hdisplay % dev_priv->psr.su_x_granularity) { + DRM_DEBUG_KMS("PSR2 not enabled, hdisplay(%d) not multiple of %d\n", + crtc_hdisplay, dev_priv->psr.su_x_granularity); + return false; + } + return true; } @@ -503,10 +605,8 @@ void intel_psr_compute_config(struct intel_dp *intel_dp, return; } - if (IS_HASWELL(dev_priv) && - I915_READ(HSW_STEREO_3D_CTL(crtc_state->cpu_transcoder)) & - S3D_ENABLE) { - DRM_DEBUG_KMS("PSR condition failed: Stereo 3D is Enabled\n"); + if (dev_priv->psr.sink_not_reliable) { + DRM_DEBUG_KMS("PSR sink implementation is not reliable\n"); return; } @@ -553,11 +653,31 @@ static void intel_psr_activate(struct intel_dp *intel_dp) dev_priv->psr.active = true; } +static i915_reg_t gen9_chicken_trans_reg(struct drm_i915_private *dev_priv, + enum transcoder cpu_transcoder) +{ + static const i915_reg_t regs[] = { + [TRANSCODER_A] = CHICKEN_TRANS_A, + [TRANSCODER_B] = CHICKEN_TRANS_B, + [TRANSCODER_C] = CHICKEN_TRANS_C, + [TRANSCODER_EDP] = CHICKEN_TRANS_EDP, + }; + + WARN_ON(INTEL_GEN(dev_priv) < 9); + + if (WARN_ON(cpu_transcoder >= ARRAY_SIZE(regs) || + !regs[cpu_transcoder].reg)) + cpu_transcoder = TRANSCODER_A; + + return regs[cpu_transcoder]; +} + static void intel_psr_enable_source(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; + u32 mask; /* Only HSW and BDW have PSR AUX registers that need to be setup. SKL+ * use hardcoded values PSR AUX transactions @@ -565,38 +685,32 @@ static void intel_psr_enable_source(struct intel_dp *intel_dp, if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) hsw_psr_setup_aux(intel_dp); - if (dev_priv->psr.psr2_enabled) { - u32 chicken = I915_READ(CHICKEN_TRANS(cpu_transcoder)); - - if (INTEL_GEN(dev_priv) == 9 && !IS_GEMINILAKE(dev_priv)) - chicken |= (PSR2_VSC_ENABLE_PROG_HEADER - | PSR2_ADD_VERTICAL_LINE_COUNT); - - else - chicken &= ~VSC_DATA_SEL_SOFTWARE_CONTROL; - I915_WRITE(CHICKEN_TRANS(cpu_transcoder), chicken); - - I915_WRITE(EDP_PSR_DEBUG, - EDP_PSR_DEBUG_MASK_MEMUP | - EDP_PSR_DEBUG_MASK_HPD | - EDP_PSR_DEBUG_MASK_LPSP | - EDP_PSR_DEBUG_MASK_MAX_SLEEP | - EDP_PSR_DEBUG_MASK_DISP_REG_WRITE); - } else { - /* - * Per Spec: Avoid continuous PSR exit by masking MEMUP - * and HPD. also mask LPSP to avoid dependency on other - * drivers that might block runtime_pm besides - * preventing other hw tracking issues now we can rely - * on frontbuffer tracking. - */ - I915_WRITE(EDP_PSR_DEBUG, - EDP_PSR_DEBUG_MASK_MEMUP | - EDP_PSR_DEBUG_MASK_HPD | - EDP_PSR_DEBUG_MASK_LPSP | - EDP_PSR_DEBUG_MASK_DISP_REG_WRITE | - EDP_PSR_DEBUG_MASK_MAX_SLEEP); + if (dev_priv->psr.psr2_enabled && (IS_GEN(dev_priv, 9) && + !IS_GEMINILAKE(dev_priv))) { + i915_reg_t reg = gen9_chicken_trans_reg(dev_priv, + cpu_transcoder); + u32 chicken = I915_READ(reg); + + chicken |= PSR2_VSC_ENABLE_PROG_HEADER | + PSR2_ADD_VERTICAL_LINE_COUNT; + I915_WRITE(reg, chicken); } + + /* + * Per Spec: Avoid continuous PSR exit by masking MEMUP and HPD also + * mask LPSP to avoid dependency on other drivers that might block + * runtime_pm besides preventing other hw tracking issues now we + * can rely on frontbuffer tracking. + */ + mask = EDP_PSR_DEBUG_MASK_MEMUP | + EDP_PSR_DEBUG_MASK_HPD | + EDP_PSR_DEBUG_MASK_LPSP | + EDP_PSR_DEBUG_MASK_MAX_SLEEP; + + if (INTEL_GEN(dev_priv) < 11) + mask |= EDP_PSR_DEBUG_MASK_DISP_REG_WRITE; + + I915_WRITE(EDP_PSR_DEBUG, mask); } static void intel_psr_enable_locked(struct drm_i915_private *dev_priv, @@ -646,6 +760,7 @@ void intel_psr_enable(struct intel_dp *intel_dp, dev_priv->psr.psr2_enabled = intel_psr2_enabled(dev_priv, crtc_state); dev_priv->psr.busy_frontbuffer_bits = 0; dev_priv->psr.prepared = true; + dev_priv->psr.pipe = to_intel_crtc(crtc_state->base.crtc)->pipe; if (psr_global_enabled(dev_priv->psr.debug)) intel_psr_enable_locked(dev_priv, crtc_state); @@ -656,49 +771,34 @@ unlock: mutex_unlock(&dev_priv->psr.lock); } -static void -intel_psr_disable_source(struct intel_dp *intel_dp) +static void intel_psr_exit(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - - if (dev_priv->psr.active) { - i915_reg_t psr_status; - u32 psr_status_mask; - - if (dev_priv->psr.psr2_enabled) { - psr_status = EDP_PSR2_STATUS; - psr_status_mask = EDP_PSR2_STATUS_STATE_MASK; - - I915_WRITE(EDP_PSR2_CTL, - I915_READ(EDP_PSR2_CTL) & - ~(EDP_PSR2_ENABLE | EDP_SU_TRACK_ENABLE)); - - } else { - psr_status = EDP_PSR_STATUS; - psr_status_mask = EDP_PSR_STATUS_STATE_MASK; - - I915_WRITE(EDP_PSR_CTL, - I915_READ(EDP_PSR_CTL) & ~EDP_PSR_ENABLE); - } + u32 val; - /* Wait till PSR is idle */ - if (intel_wait_for_register(dev_priv, - psr_status, psr_status_mask, 0, - 2000)) - DRM_ERROR("Timed out waiting for PSR Idle State\n"); + if (!dev_priv->psr.active) { + if (INTEL_GEN(dev_priv) >= 9) + WARN_ON(I915_READ(EDP_PSR2_CTL) & EDP_PSR2_ENABLE); + WARN_ON(I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE); + return; + } - dev_priv->psr.active = false; + if (dev_priv->psr.psr2_enabled) { + val = I915_READ(EDP_PSR2_CTL); + WARN_ON(!(val & EDP_PSR2_ENABLE)); + I915_WRITE(EDP_PSR2_CTL, val & ~EDP_PSR2_ENABLE); } else { - if (dev_priv->psr.psr2_enabled) - WARN_ON(I915_READ(EDP_PSR2_CTL) & EDP_PSR2_ENABLE); - else - WARN_ON(I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE); + val = I915_READ(EDP_PSR_CTL); + WARN_ON(!(val & EDP_PSR_ENABLE)); + I915_WRITE(EDP_PSR_CTL, val & ~EDP_PSR_ENABLE); } + dev_priv->psr.active = false; } static void intel_psr_disable_locked(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + i915_reg_t psr_status; + u32 psr_status_mask; lockdep_assert_held(&dev_priv->psr.lock); @@ -707,7 +807,21 @@ static void intel_psr_disable_locked(struct intel_dp *intel_dp) DRM_DEBUG_KMS("Disabling PSR%s\n", dev_priv->psr.psr2_enabled ? "2" : "1"); - intel_psr_disable_source(intel_dp); + + intel_psr_exit(dev_priv); + + if (dev_priv->psr.psr2_enabled) { + psr_status = EDP_PSR2_STATUS; + psr_status_mask = EDP_PSR2_STATUS_STATE_MASK; + } else { + psr_status = EDP_PSR_STATUS; + psr_status_mask = EDP_PSR_STATUS_STATE_MASK; + } + + /* Wait till PSR is idle */ + if (intel_wait_for_register(dev_priv, psr_status, psr_status_mask, 0, + 2000)) + DRM_ERROR("Timed out waiting PSR idle state\n"); /* Disable PSR on Sink */ drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, 0); @@ -893,6 +1007,16 @@ int intel_psr_set_debugfs_mode(struct drm_i915_private *dev_priv, return ret; } +static void intel_psr_handle_irq(struct drm_i915_private *dev_priv) +{ + struct i915_psr *psr = &dev_priv->psr; + + intel_psr_disable_locked(psr->dp); + psr->sink_not_reliable = true; + /* let's make sure that sink is awaken */ + drm_dp_dpcd_writeb(&psr->dp->aux, DP_SET_POWER, DP_SET_POWER_D0); +} + static void intel_psr_work(struct work_struct *work) { struct drm_i915_private *dev_priv = @@ -903,6 +1027,9 @@ static void intel_psr_work(struct work_struct *work) if (!dev_priv->psr.enabled) goto unlock; + if (READ_ONCE(dev_priv->psr.irq_aux_error)) + intel_psr_handle_irq(dev_priv); + /* * We have to make sure PSR is ready for re-enable * otherwise it keeps disabled until next full enable/disable cycle. @@ -925,25 +1052,6 @@ unlock: mutex_unlock(&dev_priv->psr.lock); } -static void intel_psr_exit(struct drm_i915_private *dev_priv) -{ - u32 val; - - if (!dev_priv->psr.active) - return; - - if (dev_priv->psr.psr2_enabled) { - val = I915_READ(EDP_PSR2_CTL); - WARN_ON(!(val & EDP_PSR2_ENABLE)); - I915_WRITE(EDP_PSR2_CTL, val & ~EDP_PSR2_ENABLE); - } else { - val = I915_READ(EDP_PSR_CTL); - WARN_ON(!(val & EDP_PSR_ENABLE)); - I915_WRITE(EDP_PSR_CTL, val & ~EDP_PSR_ENABLE); - } - dev_priv->psr.active = false; -} - /** * intel_psr_invalidate - Invalidade PSR * @dev_priv: i915 device @@ -960,9 +1068,6 @@ static void intel_psr_exit(struct drm_i915_private *dev_priv) void intel_psr_invalidate(struct drm_i915_private *dev_priv, unsigned frontbuffer_bits, enum fb_op_origin origin) { - struct drm_crtc *crtc; - enum pipe pipe; - if (!CAN_PSR(dev_priv)) return; @@ -975,10 +1080,7 @@ void intel_psr_invalidate(struct drm_i915_private *dev_priv, return; } - crtc = dp_to_dig_port(dev_priv->psr.dp)->base.base.crtc; - pipe = to_intel_crtc(crtc)->pipe; - - frontbuffer_bits &= INTEL_FRONTBUFFER_ALL_MASK(pipe); + frontbuffer_bits &= INTEL_FRONTBUFFER_ALL_MASK(dev_priv->psr.pipe); dev_priv->psr.busy_frontbuffer_bits |= frontbuffer_bits; if (frontbuffer_bits) @@ -1003,9 +1105,6 @@ void intel_psr_invalidate(struct drm_i915_private *dev_priv, void intel_psr_flush(struct drm_i915_private *dev_priv, unsigned frontbuffer_bits, enum fb_op_origin origin) { - struct drm_crtc *crtc; - enum pipe pipe; - if (!CAN_PSR(dev_priv)) return; @@ -1018,28 +1117,21 @@ void intel_psr_flush(struct drm_i915_private *dev_priv, return; } - crtc = dp_to_dig_port(dev_priv->psr.dp)->base.base.crtc; - pipe = to_intel_crtc(crtc)->pipe; - - frontbuffer_bits &= INTEL_FRONTBUFFER_ALL_MASK(pipe); + frontbuffer_bits &= INTEL_FRONTBUFFER_ALL_MASK(dev_priv->psr.pipe); dev_priv->psr.busy_frontbuffer_bits &= ~frontbuffer_bits; /* By definition flush = invalidate + flush */ if (frontbuffer_bits) { - if (dev_priv->psr.psr2_enabled) { - intel_psr_exit(dev_priv); - } else { - /* - * Display WA #0884: all - * This documented WA for bxt can be safely applied - * broadly so we can force HW tracking to exit PSR - * instead of disabling and re-enabling. - * Workaround tells us to write 0 to CUR_SURFLIVE_A, - * but it makes more sense write to the current active - * pipe. - */ - I915_WRITE(CURSURFLIVE(pipe), 0); - } + /* + * Display WA #0884: all + * This documented WA for bxt can be safely applied + * broadly so we can force HW tracking to exit PSR + * instead of disabling and re-enabling. + * Workaround tells us to write 0 to CUR_SURFLIVE_A, + * but it makes more sense write to the current active + * pipe. + */ + I915_WRITE(CURSURFLIVE(dev_priv->psr.pipe), 0); } if (!dev_priv->psr.active && !dev_priv->psr.busy_frontbuffer_bits) @@ -1056,6 +1148,8 @@ void intel_psr_flush(struct drm_i915_private *dev_priv, */ void intel_psr_init(struct drm_i915_private *dev_priv) { + u32 val; + if (!HAS_PSR(dev_priv)) return; @@ -1065,11 +1159,24 @@ void intel_psr_init(struct drm_i915_private *dev_priv) if (!dev_priv->psr.sink_support) return; - if (i915_modparams.enable_psr == -1) { - i915_modparams.enable_psr = dev_priv->vbt.psr.enable; + if (i915_modparams.enable_psr == -1) + if (INTEL_GEN(dev_priv) < 9 || !dev_priv->vbt.psr.enable) + i915_modparams.enable_psr = 0; - /* Per platform default: all disabled. */ - i915_modparams.enable_psr = 0; + /* + * If a PSR error happened and the driver is reloaded, the EDP_PSR_IIR + * will still keep the error set even after the reset done in the + * irq_preinstall and irq_uninstall hooks. + * And enabling in this situation cause the screen to freeze in the + * first time that PSR HW tries to activate so lets keep PSR disabled + * to avoid any rendering problems. + */ + val = I915_READ(EDP_PSR_IIR); + val &= EDP_PSR_ERROR(edp_psr_shift(TRANSCODER_EDP)); + if (val) { + DRM_DEBUG_KMS("PSR interruption error set\n"); + dev_priv->psr.sink_not_reliable = true; + return; } /* Set link_standby x link_off defaults */ @@ -1109,6 +1216,7 @@ void intel_psr_short_pulse(struct intel_dp *intel_dp) if ((val & DP_PSR_SINK_STATE_MASK) == DP_PSR_SINK_INTERNAL_ERROR) { DRM_DEBUG_KMS("PSR sink internal error, disabling PSR\n"); intel_psr_disable_locked(intel_dp); + psr->sink_not_reliable = true; } if (drm_dp_dpcd_readb(&intel_dp->aux, DP_PSR_ERROR_STATUS, &val) != 1) { @@ -1126,12 +1234,27 @@ void intel_psr_short_pulse(struct intel_dp *intel_dp) if (val & ~errors) DRM_ERROR("PSR_ERROR_STATUS unhandled errors %x\n", val & ~errors); - if (val & errors) + if (val & errors) { intel_psr_disable_locked(intel_dp); + psr->sink_not_reliable = true; + } /* clear status register */ drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_ERROR_STATUS, val); - - /* TODO: handle PSR2 errors */ exit: mutex_unlock(&psr->lock); } + +bool intel_psr_enabled(struct intel_dp *intel_dp) +{ + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + bool ret; + + if (!CAN_PSR(dev_priv) || !intel_dp_is_edp(intel_dp)) + return false; + + mutex_lock(&dev_priv->psr.lock); + ret = (dev_priv->psr.dp == intel_dp && dev_priv->psr.enabled); + mutex_unlock(&dev_priv->psr.lock); + + return ret; +} diff --git a/drivers/gpu/drm/i915/intel_quirks.c b/drivers/gpu/drm/i915/intel_quirks.c new file mode 100644 index 000000000000..ec2b0fc92b8b --- /dev/null +++ b/drivers/gpu/drm/i915/intel_quirks.c @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2018 Intel Corporation + */ + +#include <linux/dmi.h> + +#include "intel_drv.h" + +/* + * Some machines (Lenovo U160) do not work with SSC on LVDS for some reason + */ +static void quirk_ssc_force_disable(struct drm_i915_private *i915) +{ + i915->quirks |= QUIRK_LVDS_SSC_DISABLE; + DRM_INFO("applying lvds SSC disable quirk\n"); +} + +/* + * A machine (e.g. Acer Aspire 5734Z) may need to invert the panel backlight + * brightness value + */ +static void quirk_invert_brightness(struct drm_i915_private *i915) +{ + i915->quirks |= QUIRK_INVERT_BRIGHTNESS; + DRM_INFO("applying inverted panel brightness quirk\n"); +} + +/* Some VBT's incorrectly indicate no backlight is present */ +static void quirk_backlight_present(struct drm_i915_private *i915) +{ + i915->quirks |= QUIRK_BACKLIGHT_PRESENT; + DRM_INFO("applying backlight present quirk\n"); +} + +/* Toshiba Satellite P50-C-18C requires T12 delay to be min 800ms + * which is 300 ms greater than eDP spec T12 min. + */ +static void quirk_increase_t12_delay(struct drm_i915_private *i915) +{ + i915->quirks |= QUIRK_INCREASE_T12_DELAY; + DRM_INFO("Applying T12 delay quirk\n"); +} + +/* + * GeminiLake NUC HDMI outputs require additional off time + * this allows the onboard retimer to correctly sync to signal + */ +static void quirk_increase_ddi_disabled_time(struct drm_i915_private *i915) +{ + i915->quirks |= QUIRK_INCREASE_DDI_DISABLED_TIME; + DRM_INFO("Applying Increase DDI Disabled quirk\n"); +} + +struct intel_quirk { + int device; + int subsystem_vendor; + int subsystem_device; + void (*hook)(struct drm_i915_private *i915); +}; + +/* For systems that don't have a meaningful PCI subdevice/subvendor ID */ +struct intel_dmi_quirk { + void (*hook)(struct drm_i915_private *i915); + const struct dmi_system_id (*dmi_id_list)[]; +}; + +static int intel_dmi_reverse_brightness(const struct dmi_system_id *id) +{ + DRM_INFO("Backlight polarity reversed on %s\n", id->ident); + return 1; +} + +static const struct intel_dmi_quirk intel_dmi_quirks[] = { + { + .dmi_id_list = &(const struct dmi_system_id[]) { + { + .callback = intel_dmi_reverse_brightness, + .ident = "NCR Corporation", + .matches = {DMI_MATCH(DMI_SYS_VENDOR, "NCR Corporation"), + DMI_MATCH(DMI_PRODUCT_NAME, ""), + }, + }, + { } /* terminating entry */ + }, + .hook = quirk_invert_brightness, + }, +}; + +static struct intel_quirk intel_quirks[] = { + /* Lenovo U160 cannot use SSC on LVDS */ + { 0x0046, 0x17aa, 0x3920, quirk_ssc_force_disable }, + + /* Sony Vaio Y cannot use SSC on LVDS */ + { 0x0046, 0x104d, 0x9076, quirk_ssc_force_disable }, + + /* Acer Aspire 5734Z must invert backlight brightness */ + { 0x2a42, 0x1025, 0x0459, quirk_invert_brightness }, + + /* Acer/eMachines G725 */ + { 0x2a42, 0x1025, 0x0210, quirk_invert_brightness }, + + /* Acer/eMachines e725 */ + { 0x2a42, 0x1025, 0x0212, quirk_invert_brightness }, + + /* Acer/Packard Bell NCL20 */ + { 0x2a42, 0x1025, 0x034b, quirk_invert_brightness }, + + /* Acer Aspire 4736Z */ + { 0x2a42, 0x1025, 0x0260, quirk_invert_brightness }, + + /* Acer Aspire 5336 */ + { 0x2a42, 0x1025, 0x048a, quirk_invert_brightness }, + + /* Acer C720 and C720P Chromebooks (Celeron 2955U) have backlights */ + { 0x0a06, 0x1025, 0x0a11, quirk_backlight_present }, + + /* Acer C720 Chromebook (Core i3 4005U) */ + { 0x0a16, 0x1025, 0x0a11, quirk_backlight_present }, + + /* Apple Macbook 2,1 (Core 2 T7400) */ + { 0x27a2, 0x8086, 0x7270, quirk_backlight_present }, + + /* Apple Macbook 4,1 */ + { 0x2a02, 0x106b, 0x00a1, quirk_backlight_present }, + + /* Toshiba CB35 Chromebook (Celeron 2955U) */ + { 0x0a06, 0x1179, 0x0a88, quirk_backlight_present }, + + /* HP Chromebook 14 (Celeron 2955U) */ + { 0x0a06, 0x103c, 0x21ed, quirk_backlight_present }, + + /* Dell Chromebook 11 */ + { 0x0a06, 0x1028, 0x0a35, quirk_backlight_present }, + + /* Dell Chromebook 11 (2015 version) */ + { 0x0a16, 0x1028, 0x0a35, quirk_backlight_present }, + + /* Toshiba Satellite P50-C-18C */ + { 0x191B, 0x1179, 0xF840, quirk_increase_t12_delay }, + + /* GeminiLake NUC */ + { 0x3185, 0x8086, 0x2072, quirk_increase_ddi_disabled_time }, + { 0x3184, 0x8086, 0x2072, quirk_increase_ddi_disabled_time }, + /* ASRock ITX*/ + { 0x3185, 0x1849, 0x2212, quirk_increase_ddi_disabled_time }, + { 0x3184, 0x1849, 0x2212, quirk_increase_ddi_disabled_time }, +}; + +void intel_init_quirks(struct drm_i915_private *i915) +{ + struct pci_dev *d = i915->drm.pdev; + int i; + + for (i = 0; i < ARRAY_SIZE(intel_quirks); i++) { + struct intel_quirk *q = &intel_quirks[i]; + + if (d->device == q->device && + (d->subsystem_vendor == q->subsystem_vendor || + q->subsystem_vendor == PCI_ANY_ID) && + (d->subsystem_device == q->subsystem_device || + q->subsystem_device == PCI_ANY_ID)) + q->hook(i915); + } + for (i = 0; i < ARRAY_SIZE(intel_dmi_quirks); i++) { + if (dmi_check_system(*intel_dmi_quirks[i].dmi_id_list) != 0) + intel_dmi_quirks[i].hook(i915); + } +} diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c index 1f8d2a66c791..7f841dba87b3 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.c +++ b/drivers/gpu/drm/i915/intel_ringbuffer.c @@ -29,11 +29,11 @@ #include <linux/log2.h> -#include <drm/drmP.h> #include <drm/i915_drm.h> #include "i915_drv.h" #include "i915_gem_render_state.h" +#include "i915_reset.h" #include "i915_trace.h" #include "intel_drv.h" #include "intel_workarounds.h" @@ -43,17 +43,10 @@ */ #define LEGACY_REQUEST_SIZE 200 -static unsigned int __intel_ring_space(unsigned int head, - unsigned int tail, - unsigned int size) +static inline u32 intel_hws_seqno_address(struct intel_engine_cs *engine) { - /* - * "If the Ring Buffer Head Pointer and the Tail Pointer are on the - * same cacheline, the Head Pointer must not be greater than the Tail - * Pointer." - */ - GEM_BUG_ON(!is_power_of_2(size)); - return (head - tail - CACHELINE_BYTES) & (size - 1); + return (i915_ggtt_offset(engine->status_page.vma) + + I915_GEM_HWS_INDEX_ADDR); } unsigned int intel_ring_update_space(struct intel_ring *ring) @@ -133,7 +126,7 @@ gen4_render_ring_flush(struct i915_request *rq, u32 mode) cmd = MI_FLUSH; if (mode & EMIT_INVALIDATE) { cmd |= MI_EXE_FLUSH; - if (IS_G4X(rq->i915) || IS_GEN5(rq->i915)) + if (IS_G4X(rq->i915) || IS_GEN(rq->i915, 5)) cmd |= MI_INVALIDATE_ISP; } @@ -217,7 +210,7 @@ gen4_render_ring_flush(struct i915_request *rq, u32 mode) * really our business. That leaves only stall at scoreboard. */ static int -intel_emit_post_sync_nonzero_flush(struct i915_request *rq) +gen6_emit_post_sync_nonzero_flush(struct i915_request *rq) { u32 scratch_addr = i915_scratch_offset(rq->i915) + 2 * CACHELINE_BYTES; u32 *cs; @@ -257,7 +250,7 @@ gen6_render_ring_flush(struct i915_request *rq, u32 mode) int ret; /* Force SNB workarounds for PIPE_CONTROL flushes */ - ret = intel_emit_post_sync_nonzero_flush(rq); + ret = gen6_emit_post_sync_nonzero_flush(rq); if (ret) return ret; @@ -300,6 +293,43 @@ gen6_render_ring_flush(struct i915_request *rq, u32 mode) return 0; } +static u32 *gen6_rcs_emit_breadcrumb(struct i915_request *rq, u32 *cs) +{ + /* First we do the gen6_emit_post_sync_nonzero_flush w/a */ + *cs++ = GFX_OP_PIPE_CONTROL(4); + *cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD; + *cs++ = 0; + *cs++ = 0; + + *cs++ = GFX_OP_PIPE_CONTROL(4); + *cs++ = PIPE_CONTROL_QW_WRITE; + *cs++ = i915_scratch_offset(rq->i915) | PIPE_CONTROL_GLOBAL_GTT; + *cs++ = 0; + + /* Finally we can flush and with it emit the breadcrumb */ + *cs++ = GFX_OP_PIPE_CONTROL(4); + *cs++ = (PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | + PIPE_CONTROL_DEPTH_CACHE_FLUSH | + PIPE_CONTROL_DC_FLUSH_ENABLE | + PIPE_CONTROL_QW_WRITE | + PIPE_CONTROL_CS_STALL); + *cs++ = rq->timeline->hwsp_offset | PIPE_CONTROL_GLOBAL_GTT; + *cs++ = rq->fence.seqno; + + *cs++ = GFX_OP_PIPE_CONTROL(4); + *cs++ = PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL; + *cs++ = intel_hws_seqno_address(rq->engine) | PIPE_CONTROL_GLOBAL_GTT; + *cs++ = rq->global_seqno; + + *cs++ = MI_USER_INTERRUPT; + *cs++ = MI_NOOP; + + rq->tail = intel_ring_offset(rq, cs); + assert_ring_tail_valid(rq->ring, rq->tail); + + return cs; +} + static int gen7_render_ring_cs_stall_wa(struct i915_request *rq) { @@ -379,11 +409,111 @@ gen7_render_ring_flush(struct i915_request *rq, u32 mode) return 0; } -static void ring_setup_phys_status_page(struct intel_engine_cs *engine) +static u32 *gen7_rcs_emit_breadcrumb(struct i915_request *rq, u32 *cs) +{ + *cs++ = GFX_OP_PIPE_CONTROL(4); + *cs++ = (PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | + PIPE_CONTROL_DEPTH_CACHE_FLUSH | + PIPE_CONTROL_DC_FLUSH_ENABLE | + PIPE_CONTROL_FLUSH_ENABLE | + PIPE_CONTROL_QW_WRITE | + PIPE_CONTROL_GLOBAL_GTT_IVB | + PIPE_CONTROL_CS_STALL); + *cs++ = rq->timeline->hwsp_offset; + *cs++ = rq->fence.seqno; + + *cs++ = GFX_OP_PIPE_CONTROL(4); + *cs++ = (PIPE_CONTROL_QW_WRITE | + PIPE_CONTROL_GLOBAL_GTT_IVB | + PIPE_CONTROL_CS_STALL); + *cs++ = intel_hws_seqno_address(rq->engine); + *cs++ = rq->global_seqno; + + *cs++ = MI_USER_INTERRUPT; + *cs++ = MI_NOOP; + + rq->tail = intel_ring_offset(rq, cs); + assert_ring_tail_valid(rq->ring, rq->tail); + + return cs; +} + +static u32 *gen6_xcs_emit_breadcrumb(struct i915_request *rq, u32 *cs) +{ + GEM_BUG_ON(rq->timeline->hwsp_ggtt != rq->engine->status_page.vma); + GEM_BUG_ON(offset_in_page(rq->timeline->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR); + + *cs++ = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX; + *cs++ = I915_GEM_HWS_SEQNO_ADDR | MI_FLUSH_DW_USE_GTT; + *cs++ = rq->fence.seqno; + + *cs++ = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX; + *cs++ = I915_GEM_HWS_INDEX_ADDR | MI_FLUSH_DW_USE_GTT; + *cs++ = rq->global_seqno; + + *cs++ = MI_USER_INTERRUPT; + *cs++ = MI_NOOP; + + rq->tail = intel_ring_offset(rq, cs); + assert_ring_tail_valid(rq->ring, rq->tail); + + return cs; +} + +#define GEN7_XCS_WA 32 +static u32 *gen7_xcs_emit_breadcrumb(struct i915_request *rq, u32 *cs) +{ + int i; + + GEM_BUG_ON(rq->timeline->hwsp_ggtt != rq->engine->status_page.vma); + GEM_BUG_ON(offset_in_page(rq->timeline->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR); + + *cs++ = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX; + *cs++ = I915_GEM_HWS_SEQNO_ADDR | MI_FLUSH_DW_USE_GTT; + *cs++ = rq->fence.seqno; + + *cs++ = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX; + *cs++ = I915_GEM_HWS_INDEX_ADDR | MI_FLUSH_DW_USE_GTT; + *cs++ = rq->global_seqno; + + for (i = 0; i < GEN7_XCS_WA; i++) { + *cs++ = MI_STORE_DWORD_INDEX; + *cs++ = I915_GEM_HWS_SEQNO_ADDR; + *cs++ = rq->fence.seqno; + } + + *cs++ = MI_FLUSH_DW; + *cs++ = 0; + *cs++ = 0; + + *cs++ = MI_USER_INTERRUPT; + + rq->tail = intel_ring_offset(rq, cs); + assert_ring_tail_valid(rq->ring, rq->tail); + + return cs; +} +#undef GEN7_XCS_WA + +static void set_hwstam(struct intel_engine_cs *engine, u32 mask) +{ + /* + * Keep the render interrupt unmasked as this papers over + * lost interrupts following a reset. + */ + if (engine->class == RENDER_CLASS) { + if (INTEL_GEN(engine->i915) >= 6) + mask &= ~BIT(0); + else + mask &= ~I915_USER_INTERRUPT; + } + + intel_engine_set_hwsp_writemask(engine, mask); +} + +static void set_hws_pga(struct intel_engine_cs *engine, phys_addr_t phys) { struct drm_i915_private *dev_priv = engine->i915; - struct page *page = virt_to_page(engine->status_page.page_addr); - phys_addr_t phys = PFN_PHYS(page_to_pfn(page)); u32 addr; addr = lower_32_bits(phys); @@ -393,15 +523,30 @@ static void ring_setup_phys_status_page(struct intel_engine_cs *engine) I915_WRITE(HWS_PGA, addr); } -static void intel_ring_setup_status_page(struct intel_engine_cs *engine) +static struct page *status_page(struct intel_engine_cs *engine) +{ + struct drm_i915_gem_object *obj = engine->status_page.vma->obj; + + GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); + return sg_page(obj->mm.pages->sgl); +} + +static void ring_setup_phys_status_page(struct intel_engine_cs *engine) +{ + set_hws_pga(engine, PFN_PHYS(page_to_pfn(status_page(engine)))); + set_hwstam(engine, ~0u); +} + +static void set_hwsp(struct intel_engine_cs *engine, u32 offset) { struct drm_i915_private *dev_priv = engine->i915; - i915_reg_t mmio; + i915_reg_t hwsp; - /* The ring status page addresses are no longer next to the rest of + /* + * The ring status page addresses are no longer next to the rest of * the ring registers as of gen7. */ - if (IS_GEN7(dev_priv)) { + if (IS_GEN(dev_priv, 7)) { switch (engine->id) { /* * No more rings exist on Gen7. Default case is only to shut up @@ -410,56 +555,55 @@ static void intel_ring_setup_status_page(struct intel_engine_cs *engine) default: GEM_BUG_ON(engine->id); case RCS: - mmio = RENDER_HWS_PGA_GEN7; + hwsp = RENDER_HWS_PGA_GEN7; break; case BCS: - mmio = BLT_HWS_PGA_GEN7; + hwsp = BLT_HWS_PGA_GEN7; break; case VCS: - mmio = BSD_HWS_PGA_GEN7; + hwsp = BSD_HWS_PGA_GEN7; break; case VECS: - mmio = VEBOX_HWS_PGA_GEN7; + hwsp = VEBOX_HWS_PGA_GEN7; break; } - } else if (IS_GEN6(dev_priv)) { - mmio = RING_HWS_PGA_GEN6(engine->mmio_base); + } else if (IS_GEN(dev_priv, 6)) { + hwsp = RING_HWS_PGA_GEN6(engine->mmio_base); } else { - mmio = RING_HWS_PGA(engine->mmio_base); + hwsp = RING_HWS_PGA(engine->mmio_base); } - if (INTEL_GEN(dev_priv) >= 6) { - u32 mask = ~0u; + I915_WRITE(hwsp, offset); + POSTING_READ(hwsp); +} - /* - * Keep the render interrupt unmasked as this papers over - * lost interrupts following a reset. - */ - if (engine->id == RCS) - mask &= ~BIT(0); +static void flush_cs_tlb(struct intel_engine_cs *engine) +{ + struct drm_i915_private *dev_priv = engine->i915; + i915_reg_t instpm = RING_INSTPM(engine->mmio_base); - I915_WRITE(RING_HWSTAM(engine->mmio_base), mask); - } + if (!IS_GEN_RANGE(dev_priv, 6, 7)) + return; - I915_WRITE(mmio, engine->status_page.ggtt_offset); - POSTING_READ(mmio); + /* ring should be idle before issuing a sync flush*/ + WARN_ON((I915_READ_MODE(engine) & MODE_IDLE) == 0); - /* Flush the TLB for this page */ - if (IS_GEN(dev_priv, 6, 7)) { - i915_reg_t reg = RING_INSTPM(engine->mmio_base); + I915_WRITE(instpm, + _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE | + INSTPM_SYNC_FLUSH)); + if (intel_wait_for_register(dev_priv, + instpm, INSTPM_SYNC_FLUSH, 0, + 1000)) + DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n", + engine->name); +} - /* ring should be idle before issuing a sync flush*/ - WARN_ON((I915_READ_MODE(engine) & MODE_IDLE) == 0); +static void ring_setup_status_page(struct intel_engine_cs *engine) +{ + set_hwsp(engine, i915_ggtt_offset(engine->status_page.vma)); + set_hwstam(engine, ~0u); - I915_WRITE(reg, - _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE | - INSTPM_SYNC_FLUSH)); - if (intel_wait_for_register(dev_priv, - reg, INSTPM_SYNC_FLUSH, 0, - 1000)) - DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n", - engine->name); - } + flush_cs_tlb(engine); } static bool stop_ring(struct intel_engine_cs *engine) @@ -529,7 +673,7 @@ static int init_ring_common(struct intel_engine_cs *engine) if (HWS_NEEDS_PHYSICAL(dev_priv)) ring_setup_phys_status_page(engine); else - intel_ring_setup_status_page(engine); + ring_setup_status_page(engine); intel_engine_reset_breadcrumbs(engine); @@ -550,10 +694,11 @@ static int init_ring_common(struct intel_engine_cs *engine) /* Check that the ring offsets point within the ring! */ GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->head)); GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->tail)); - intel_ring_update_space(ring); + + /* First wake the ring up to an empty/idle ring */ I915_WRITE_HEAD(engine, ring->head); - I915_WRITE_TAIL(engine, ring->tail); + I915_WRITE_TAIL(engine, ring->head); (void)I915_READ_TAIL(engine); I915_WRITE_CTL(engine, RING_CTL_SIZE(ring->size) | RING_VALID); @@ -578,62 +723,94 @@ static int init_ring_common(struct intel_engine_cs *engine) if (INTEL_GEN(dev_priv) > 2) I915_WRITE_MODE(engine, _MASKED_BIT_DISABLE(STOP_RING)); + /* Now awake, let it get started */ + if (ring->tail != ring->head) { + I915_WRITE_TAIL(engine, ring->tail); + (void)I915_READ_TAIL(engine); + } + /* Papering over lost _interrupts_ immediately following the restart */ - intel_engine_wakeup(engine); + intel_engine_queue_breadcrumbs(engine); out: intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); return ret; } -static struct i915_request *reset_prepare(struct intel_engine_cs *engine) +static void reset_prepare(struct intel_engine_cs *engine) { intel_engine_stop_cs(engine); - - if (engine->irq_seqno_barrier) - engine->irq_seqno_barrier(engine); - - return i915_gem_find_active_request(engine); } -static void skip_request(struct i915_request *rq) +static void reset_ring(struct intel_engine_cs *engine, bool stalled) { - void *vaddr = rq->ring->vaddr; + struct i915_timeline *tl = &engine->timeline; + struct i915_request *pos, *rq; + unsigned long flags; u32 head; - head = rq->infix; - if (rq->postfix < head) { - memset32(vaddr + head, MI_NOOP, - (rq->ring->size - head) / sizeof(u32)); - head = 0; + rq = NULL; + spin_lock_irqsave(&tl->lock, flags); + list_for_each_entry(pos, &tl->requests, link) { + if (!i915_request_completed(pos)) { + rq = pos; + break; + } } - memset32(vaddr + head, MI_NOOP, (rq->postfix - head) / sizeof(u32)); -} - -static void reset_ring(struct intel_engine_cs *engine, struct i915_request *rq) -{ - GEM_TRACE("%s seqno=%x\n", engine->name, rq ? rq->global_seqno : 0); + GEM_TRACE("%s seqno=%d, current=%d, stalled? %s\n", + engine->name, + rq ? rq->global_seqno : 0, + intel_engine_get_seqno(engine), + yesno(stalled)); /* - * Try to restore the logical GPU state to match the continuation - * of the request queue. If we skip the context/PD restore, then - * the next request may try to execute assuming that its context - * is valid and loaded on the GPU and so may try to access invalid - * memory, prompting repeated GPU hangs. + * The guilty request will get skipped on a hung engine. * - * If the request was guilty, we still restore the logical state - * in case the next request requires it (e.g. the aliasing ppgtt), - * but skip over the hung batch. + * Users of client default contexts do not rely on logical + * state preserved between batches so it is safe to execute + * queued requests following the hang. Non default contexts + * rely on preserved state, so skipping a batch loses the + * evolution of the state and it needs to be considered corrupted. + * Executing more queued batches on top of corrupted state is + * risky. But we take the risk by trying to advance through + * the queued requests in order to make the client behaviour + * more predictable around resets, by not throwing away random + * amount of batches it has prepared for execution. Sophisticated + * clients can use gem_reset_stats_ioctl and dma fence status + * (exported via sync_file info ioctl on explicit fences) to observe + * when it loses the context state and should rebuild accordingly. * - * If the request was innocent, we try to replay the request with - * the restored context. + * The context ban, and ultimately the client ban, mechanism are safety + * valves if client submission ends up resulting in nothing more than + * subsequent hangs. */ + if (rq) { - /* If the rq hung, jump to its breadcrumb and skip the batch */ - rq->ring->head = intel_ring_wrap(rq->ring, rq->head); - if (rq->fence.error == -EIO) - skip_request(rq); + /* + * Try to restore the logical GPU state to match the + * continuation of the request queue. If we skip the + * context/PD restore, then the next request may try to execute + * assuming that its context is valid and loaded on the GPU and + * so may try to access invalid memory, prompting repeated GPU + * hangs. + * + * If the request was guilty, we still restore the logical + * state in case the next request requires it (e.g. the + * aliasing ppgtt), but skip over the hung batch. + * + * If the request was innocent, we try to replay the request + * with the restored context. + */ + i915_reset_request(rq, stalled); + + GEM_BUG_ON(rq->ring != engine->buffer); + head = rq->head; + } else { + head = engine->buffer->tail; } + engine->buffer->head = intel_ring_wrap(engine->buffer, head); + + spin_unlock_irqrestore(&tl->lock, flags); } static void reset_finish(struct intel_engine_cs *engine) @@ -644,7 +821,7 @@ static int intel_rcs_ctx_init(struct i915_request *rq) { int ret; - ret = intel_ctx_workarounds_emit(rq); + ret = intel_engine_emit_ctx_wa(rq); if (ret != 0) return ret; @@ -662,10 +839,8 @@ static int init_render_ring(struct intel_engine_cs *engine) if (ret) return ret; - intel_whitelist_workarounds_apply(engine); - /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */ - if (IS_GEN(dev_priv, 4, 6)) + if (IS_GEN_RANGE(dev_priv, 4, 6)) I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH)); /* We need to disable the AsyncFlip performance optimisations in order @@ -674,22 +849,22 @@ static int init_render_ring(struct intel_engine_cs *engine) * * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv */ - if (IS_GEN(dev_priv, 6, 7)) + if (IS_GEN_RANGE(dev_priv, 6, 7)) I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE)); /* Required for the hardware to program scanline values for waiting */ /* WaEnableFlushTlbInvalidationMode:snb */ - if (IS_GEN6(dev_priv)) + if (IS_GEN(dev_priv, 6)) I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT)); /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */ - if (IS_GEN7(dev_priv)) + if (IS_GEN(dev_priv, 7)) I915_WRITE(GFX_MODE_GEN7, _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) | _MASKED_BIT_ENABLE(GFX_REPLAY_MODE)); - if (IS_GEN6(dev_priv)) { + if (IS_GEN(dev_priv, 6)) { /* From the Sandybridge PRM, volume 1 part 3, page 24: * "If this bit is set, STCunit will have LRA as replacement * policy. [...] This bit must be reset. LRA replacement @@ -699,7 +874,7 @@ static int init_render_ring(struct intel_engine_cs *engine) _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB)); } - if (IS_GEN(dev_priv, 6, 7)) + if (IS_GEN_RANGE(dev_priv, 6, 7)) I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING)); if (INTEL_GEN(dev_priv) >= 6) @@ -708,33 +883,6 @@ static int init_render_ring(struct intel_engine_cs *engine) return 0; } -static u32 *gen6_signal(struct i915_request *rq, u32 *cs) -{ - struct drm_i915_private *dev_priv = rq->i915; - struct intel_engine_cs *engine; - enum intel_engine_id id; - int num_rings = 0; - - for_each_engine(engine, dev_priv, id) { - i915_reg_t mbox_reg; - - if (!(BIT(engine->hw_id) & GEN6_SEMAPHORES_MASK)) - continue; - - mbox_reg = rq->engine->semaphore.mbox.signal[engine->hw_id]; - if (i915_mmio_reg_valid(mbox_reg)) { - *cs++ = MI_LOAD_REGISTER_IMM(1); - *cs++ = i915_mmio_reg_offset(mbox_reg); - *cs++ = rq->global_seqno; - num_rings++; - } - } - if (num_rings & 1) - *cs++ = MI_NOOP; - - return cs; -} - static void cancel_requests(struct intel_engine_cs *engine) { struct i915_request *request; @@ -745,9 +893,17 @@ static void cancel_requests(struct intel_engine_cs *engine) /* Mark all submitted requests as skipped. */ list_for_each_entry(request, &engine->timeline.requests, link) { GEM_BUG_ON(!request->global_seqno); - if (!i915_request_completed(request)) + + if (!i915_request_signaled(request)) dma_fence_set_error(&request->fence, -EIO); + + i915_request_mark_complete(request); } + + intel_write_status_page(engine, + I915_GEM_HWS_INDEX, + intel_engine_last_submit(engine)); + /* Remaining _unready_ requests will be nop'ed when submitted */ spin_unlock_irqrestore(&engine->timeline.lock, flags); @@ -763,94 +919,59 @@ static void i9xx_submit_request(struct i915_request *request) intel_ring_set_tail(request->ring, request->tail)); } -static void i9xx_emit_breadcrumb(struct i915_request *rq, u32 *cs) +static u32 *i9xx_emit_breadcrumb(struct i915_request *rq, u32 *cs) { + GEM_BUG_ON(rq->timeline->hwsp_ggtt != rq->engine->status_page.vma); + GEM_BUG_ON(offset_in_page(rq->timeline->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR); + + *cs++ = MI_FLUSH; + + *cs++ = MI_STORE_DWORD_INDEX; + *cs++ = I915_GEM_HWS_SEQNO_ADDR; + *cs++ = rq->fence.seqno; + *cs++ = MI_STORE_DWORD_INDEX; - *cs++ = I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT; + *cs++ = I915_GEM_HWS_INDEX_ADDR; *cs++ = rq->global_seqno; + *cs++ = MI_USER_INTERRUPT; rq->tail = intel_ring_offset(rq, cs); assert_ring_tail_valid(rq->ring, rq->tail); -} - -static const int i9xx_emit_breadcrumb_sz = 4; -static void gen6_sema_emit_breadcrumb(struct i915_request *rq, u32 *cs) -{ - return i9xx_emit_breadcrumb(rq, rq->engine->semaphore.signal(rq, cs)); + return cs; } -static int -gen6_ring_sync_to(struct i915_request *rq, struct i915_request *signal) +#define GEN5_WA_STORES 8 /* must be at least 1! */ +static u32 *gen5_emit_breadcrumb(struct i915_request *rq, u32 *cs) { - u32 dw1 = MI_SEMAPHORE_MBOX | - MI_SEMAPHORE_COMPARE | - MI_SEMAPHORE_REGISTER; - u32 wait_mbox = signal->engine->semaphore.mbox.wait[rq->engine->hw_id]; - u32 *cs; - - WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID); + int i; - cs = intel_ring_begin(rq, 4); - if (IS_ERR(cs)) - return PTR_ERR(cs); + GEM_BUG_ON(rq->timeline->hwsp_ggtt != rq->engine->status_page.vma); + GEM_BUG_ON(offset_in_page(rq->timeline->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR); - *cs++ = dw1 | wait_mbox; - /* Throughout all of the GEM code, seqno passed implies our current - * seqno is >= the last seqno executed. However for hardware the - * comparison is strictly greater than. - */ - *cs++ = signal->global_seqno - 1; - *cs++ = 0; - *cs++ = MI_NOOP; - intel_ring_advance(rq, cs); + *cs++ = MI_FLUSH; - return 0; -} + *cs++ = MI_STORE_DWORD_INDEX; + *cs++ = I915_GEM_HWS_SEQNO_ADDR; + *cs++ = rq->fence.seqno; + + BUILD_BUG_ON(GEN5_WA_STORES < 1); + for (i = 0; i < GEN5_WA_STORES; i++) { + *cs++ = MI_STORE_DWORD_INDEX; + *cs++ = I915_GEM_HWS_INDEX_ADDR; + *cs++ = rq->global_seqno; + } -static void -gen5_seqno_barrier(struct intel_engine_cs *engine) -{ - /* MI_STORE are internally buffered by the GPU and not flushed - * either by MI_FLUSH or SyncFlush or any other combination of - * MI commands. - * - * "Only the submission of the store operation is guaranteed. - * The write result will be complete (coherent) some time later - * (this is practically a finite period but there is no guaranteed - * latency)." - * - * Empirically, we observe that we need a delay of at least 75us to - * be sure that the seqno write is visible by the CPU. - */ - usleep_range(125, 250); -} + *cs++ = MI_USER_INTERRUPT; + *cs++ = MI_NOOP; -static void -gen6_seqno_barrier(struct intel_engine_cs *engine) -{ - struct drm_i915_private *dev_priv = engine->i915; + rq->tail = intel_ring_offset(rq, cs); + assert_ring_tail_valid(rq->ring, rq->tail); - /* Workaround to force correct ordering between irq and seqno writes on - * ivb (and maybe also on snb) by reading from a CS register (like - * ACTHD) before reading the status page. - * - * Note that this effectively stalls the read by the time it takes to - * do a memory transaction, which more or less ensures that the write - * from the GPU has sufficient time to invalidate the CPU cacheline. - * Alternatively we could delay the interrupt from the CS ring to give - * the write time to land, but that would incur a delay after every - * batch i.e. much more frequent than a delay when waiting for the - * interrupt (with the same net latency). - * - * Also note that to prevent whole machine hangs on gen7, we have to - * take the spinlock to guard against concurrent cacheline access. - */ - spin_lock_irq(&dev_priv->uncore.lock); - POSTING_READ_FW(RING_ACTHD(engine->mmio_base)); - spin_unlock_irq(&dev_priv->uncore.lock); + return cs; } +#undef GEN5_WA_STORES static void gen5_irq_enable(struct intel_engine_cs *engine) @@ -925,6 +1046,10 @@ gen6_irq_enable(struct intel_engine_cs *engine) I915_WRITE_IMR(engine, ~(engine->irq_enable_mask | engine->irq_keep_mask)); + + /* Flush/delay to ensure the RING_IMR is active before the GT IMR */ + POSTING_READ_FW(RING_IMR(engine->mmio_base)); + gen5_enable_gt_irq(dev_priv, engine->irq_enable_mask); } @@ -943,6 +1068,10 @@ hsw_vebox_irq_enable(struct intel_engine_cs *engine) struct drm_i915_private *dev_priv = engine->i915; I915_WRITE_IMR(engine, ~engine->irq_enable_mask); + + /* Flush/delay to ensure the RING_IMR is active before the GT IMR */ + POSTING_READ_FW(RING_IMR(engine->mmio_base)); + gen6_unmask_pm_irq(dev_priv, engine->irq_enable_mask); } @@ -1061,14 +1190,17 @@ i915_emit_bb_start(struct i915_request *rq, int intel_ring_pin(struct intel_ring *ring) { struct i915_vma *vma = ring->vma; - enum i915_map_type map = - HAS_LLC(vma->vm->i915) ? I915_MAP_WB : I915_MAP_WC; + enum i915_map_type map = i915_coherent_map_type(vma->vm->i915); unsigned int flags; void *addr; int ret; GEM_BUG_ON(ring->vaddr); + ret = i915_timeline_pin(ring->timeline); + if (ret) + return ret; + flags = PIN_GLOBAL; /* Ring wraparound at offset 0 sometimes hangs. No idea why. */ @@ -1085,28 +1217,32 @@ int intel_ring_pin(struct intel_ring *ring) else ret = i915_gem_object_set_to_cpu_domain(vma->obj, true); if (unlikely(ret)) - return ret; + goto unpin_timeline; } ret = i915_vma_pin(vma, 0, 0, flags); if (unlikely(ret)) - return ret; + goto unpin_timeline; if (i915_vma_is_map_and_fenceable(vma)) addr = (void __force *)i915_vma_pin_iomap(vma); else addr = i915_gem_object_pin_map(vma->obj, map); - if (IS_ERR(addr)) - goto err; + if (IS_ERR(addr)) { + ret = PTR_ERR(addr); + goto unpin_ring; + } vma->obj->pin_global++; ring->vaddr = addr; return 0; -err: +unpin_ring: i915_vma_unpin(vma); - return PTR_ERR(addr); +unpin_timeline: + i915_timeline_unpin(ring->timeline); + return ret; } void intel_ring_reset(struct intel_ring *ring, u32 tail) @@ -1135,6 +1271,8 @@ void intel_ring_unpin(struct intel_ring *ring) ring->vma->obj->pin_global--; i915_vma_unpin(ring->vma); + + i915_timeline_unpin(ring->timeline); } static struct i915_vma * @@ -1445,13 +1583,18 @@ static int intel_init_ring_buffer(struct intel_engine_cs *engine) struct intel_ring *ring; int err; - intel_engine_setup_common(engine); + err = intel_engine_setup_common(engine); + if (err) + return err; - timeline = i915_timeline_create(engine->i915, engine->name); + timeline = i915_timeline_create(engine->i915, + engine->name, + engine->status_page.vma); if (IS_ERR(timeline)) { err = PTR_ERR(timeline); goto err; } + GEM_BUG_ON(timeline->has_initial_breadcrumb); ring = intel_engine_create_ring(engine, timeline, 32 * PAGE_SIZE); i915_timeline_put(timeline); @@ -1471,6 +1614,8 @@ static int intel_init_ring_buffer(struct intel_engine_cs *engine) if (err) goto err_unpin; + GEM_BUG_ON(ring->timeline->hwsp_ggtt != engine->status_page.vma); + return 0; err_unpin: @@ -1559,10 +1704,7 @@ static inline int mi_set_context(struct i915_request *rq, u32 flags) struct intel_engine_cs *engine = rq->engine; enum intel_engine_id id; const int num_rings = - /* Use an extended w/a on gen7 if signalling from other rings */ - (HAS_LEGACY_SEMAPHORES(i915) && IS_GEN7(i915)) ? - INTEL_INFO(i915)->num_rings - 1 : - 0; + IS_HSW_GT1(i915) ? RUNTIME_INFO(i915)->num_rings - 1 : 0; bool force_restore = false; int len; u32 *cs; @@ -1575,7 +1717,7 @@ static inline int mi_set_context(struct i915_request *rq, u32 flags) flags |= MI_SAVE_EXT_STATE_EN | MI_RESTORE_EXT_STATE_EN; len = 4; - if (IS_GEN7(i915)) + if (IS_GEN(i915, 7)) len += 2 + (num_rings ? 4*num_rings + 6 : 0); if (flags & MI_FORCE_RESTORE) { GEM_BUG_ON(flags & MI_RESTORE_INHIBIT); @@ -1589,7 +1731,7 @@ static inline int mi_set_context(struct i915_request *rq, u32 flags) return PTR_ERR(cs); /* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */ - if (IS_GEN7(i915)) { + if (IS_GEN(i915, 7)) { *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; if (num_rings) { struct intel_engine_cs *signaller; @@ -1636,7 +1778,7 @@ static inline int mi_set_context(struct i915_request *rq, u32 flags) */ *cs++ = MI_NOOP; - if (IS_GEN7(i915)) { + if (IS_GEN(i915, 7)) { if (num_rings) { struct intel_engine_cs *signaller; i915_reg_t last_reg = {}; /* keep gcc quiet */ @@ -1806,18 +1948,21 @@ static int ring_request_alloc(struct i915_request *request) int ret; GEM_BUG_ON(!request->hw_context->pin_count); + GEM_BUG_ON(request->timeline->has_initial_breadcrumb); - /* Flush enough space to reduce the likelihood of waiting after + /* + * Flush enough space to reduce the likelihood of waiting after * we start building the request - in which case we will just * have to repeat work. */ request->reserved_space += LEGACY_REQUEST_SIZE; - ret = intel_ring_wait_for_space(request->ring, request->reserved_space); + ret = switch_context(request); if (ret) return ret; - ret = switch_context(request); + /* Unconditionally invalidate GPU caches and TLBs. */ + ret = request->engine->emit_flush(request, EMIT_INVALIDATE); if (ret) return ret; @@ -1859,22 +2004,6 @@ static noinline int wait_for_space(struct intel_ring *ring, unsigned int bytes) return 0; } -int intel_ring_wait_for_space(struct intel_ring *ring, unsigned int bytes) -{ - GEM_BUG_ON(bytes > ring->effective_size); - if (unlikely(bytes > ring->effective_size - ring->emit)) - bytes += ring->size - ring->emit; - - if (unlikely(bytes > ring->space)) { - int ret = wait_for_space(ring, bytes); - if (unlikely(ret)) - return ret; - } - - GEM_BUG_ON(ring->space < bytes); - return 0; -} - u32 *intel_ring_begin(struct i915_request *rq, unsigned int num_dwords) { struct intel_ring *ring = rq->ring; @@ -2107,77 +2236,15 @@ static int gen6_ring_flush(struct i915_request *rq, u32 mode) return gen6_flush_dw(rq, mode, MI_INVALIDATE_TLB); } -static void intel_ring_init_semaphores(struct drm_i915_private *dev_priv, - struct intel_engine_cs *engine) -{ - int i; - - if (!HAS_LEGACY_SEMAPHORES(dev_priv)) - return; - - GEM_BUG_ON(INTEL_GEN(dev_priv) < 6); - engine->semaphore.sync_to = gen6_ring_sync_to; - engine->semaphore.signal = gen6_signal; - - /* - * The current semaphore is only applied on pre-gen8 - * platform. And there is no VCS2 ring on the pre-gen8 - * platform. So the semaphore between RCS and VCS2 is - * initialized as INVALID. - */ - for (i = 0; i < GEN6_NUM_SEMAPHORES; i++) { - static const struct { - u32 wait_mbox; - i915_reg_t mbox_reg; - } sem_data[GEN6_NUM_SEMAPHORES][GEN6_NUM_SEMAPHORES] = { - [RCS_HW] = { - [VCS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_RV, .mbox_reg = GEN6_VRSYNC }, - [BCS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_RB, .mbox_reg = GEN6_BRSYNC }, - [VECS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_RVE, .mbox_reg = GEN6_VERSYNC }, - }, - [VCS_HW] = { - [RCS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_VR, .mbox_reg = GEN6_RVSYNC }, - [BCS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_VB, .mbox_reg = GEN6_BVSYNC }, - [VECS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_VVE, .mbox_reg = GEN6_VEVSYNC }, - }, - [BCS_HW] = { - [RCS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_BR, .mbox_reg = GEN6_RBSYNC }, - [VCS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_BV, .mbox_reg = GEN6_VBSYNC }, - [VECS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_BVE, .mbox_reg = GEN6_VEBSYNC }, - }, - [VECS_HW] = { - [RCS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_VER, .mbox_reg = GEN6_RVESYNC }, - [VCS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_VEV, .mbox_reg = GEN6_VVESYNC }, - [BCS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_VEB, .mbox_reg = GEN6_BVESYNC }, - }, - }; - u32 wait_mbox; - i915_reg_t mbox_reg; - - if (i == engine->hw_id) { - wait_mbox = MI_SEMAPHORE_SYNC_INVALID; - mbox_reg = GEN6_NOSYNC; - } else { - wait_mbox = sem_data[engine->hw_id][i].wait_mbox; - mbox_reg = sem_data[engine->hw_id][i].mbox_reg; - } - - engine->semaphore.mbox.wait[i] = wait_mbox; - engine->semaphore.mbox.signal[i] = mbox_reg; - } -} - static void intel_ring_init_irq(struct drm_i915_private *dev_priv, struct intel_engine_cs *engine) { if (INTEL_GEN(dev_priv) >= 6) { engine->irq_enable = gen6_irq_enable; engine->irq_disable = gen6_irq_disable; - engine->irq_seqno_barrier = gen6_seqno_barrier; } else if (INTEL_GEN(dev_priv) >= 5) { engine->irq_enable = gen5_irq_enable; engine->irq_disable = gen5_irq_disable; - engine->irq_seqno_barrier = gen5_seqno_barrier; } else if (INTEL_GEN(dev_priv) >= 3) { engine->irq_enable = i9xx_irq_enable; engine->irq_disable = i9xx_irq_disable; @@ -2209,7 +2276,6 @@ static void intel_ring_default_vfuncs(struct drm_i915_private *dev_priv, GEM_BUG_ON(INTEL_GEN(dev_priv) >= 8); intel_ring_init_irq(dev_priv, engine); - intel_ring_init_semaphores(dev_priv, engine); engine->init_hw = init_ring_common; engine->reset.prepare = reset_prepare; @@ -2219,18 +2285,14 @@ static void intel_ring_default_vfuncs(struct drm_i915_private *dev_priv, engine->context_pin = intel_ring_context_pin; engine->request_alloc = ring_request_alloc; - engine->emit_breadcrumb = i9xx_emit_breadcrumb; - engine->emit_breadcrumb_sz = i9xx_emit_breadcrumb_sz; - if (HAS_LEGACY_SEMAPHORES(dev_priv)) { - int num_rings; - - engine->emit_breadcrumb = gen6_sema_emit_breadcrumb; - - num_rings = INTEL_INFO(dev_priv)->num_rings - 1; - engine->emit_breadcrumb_sz += num_rings * 3; - if (num_rings & 1) - engine->emit_breadcrumb_sz++; - } + /* + * Using a global execution timeline; the previous final breadcrumb is + * equivalent to our next initial bread so we can elide + * engine->emit_init_breadcrumb(). + */ + engine->emit_fini_breadcrumb = i9xx_emit_breadcrumb; + if (IS_GEN(dev_priv, 5)) + engine->emit_fini_breadcrumb = gen5_emit_breadcrumb; engine->set_default_submission = i9xx_set_default_submission; @@ -2256,12 +2318,15 @@ int intel_init_render_ring_buffer(struct intel_engine_cs *engine) engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT; - if (INTEL_GEN(dev_priv) >= 6) { + if (INTEL_GEN(dev_priv) >= 7) { engine->init_context = intel_rcs_ctx_init; engine->emit_flush = gen7_render_ring_flush; - if (IS_GEN6(dev_priv)) - engine->emit_flush = gen6_render_ring_flush; - } else if (IS_GEN5(dev_priv)) { + engine->emit_fini_breadcrumb = gen7_rcs_emit_breadcrumb; + } else if (IS_GEN(dev_priv, 6)) { + engine->init_context = intel_rcs_ctx_init; + engine->emit_flush = gen6_render_ring_flush; + engine->emit_fini_breadcrumb = gen6_rcs_emit_breadcrumb; + } else if (IS_GEN(dev_priv, 5)) { engine->emit_flush = gen4_render_ring_flush; } else { if (INTEL_GEN(dev_priv) < 4) @@ -2291,13 +2356,18 @@ int intel_init_bsd_ring_buffer(struct intel_engine_cs *engine) if (INTEL_GEN(dev_priv) >= 6) { /* gen6 bsd needs a special wa for tail updates */ - if (IS_GEN6(dev_priv)) + if (IS_GEN(dev_priv, 6)) engine->set_default_submission = gen6_bsd_set_default_submission; engine->emit_flush = gen6_bsd_ring_flush; engine->irq_enable_mask = GT_BSD_USER_INTERRUPT; + + if (IS_GEN(dev_priv, 6)) + engine->emit_fini_breadcrumb = gen6_xcs_emit_breadcrumb; + else + engine->emit_fini_breadcrumb = gen7_xcs_emit_breadcrumb; } else { engine->emit_flush = bsd_ring_flush; - if (IS_GEN5(dev_priv)) + if (IS_GEN(dev_priv, 5)) engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT; else engine->irq_enable_mask = I915_BSD_USER_INTERRUPT; @@ -2310,11 +2380,18 @@ int intel_init_blt_ring_buffer(struct intel_engine_cs *engine) { struct drm_i915_private *dev_priv = engine->i915; + GEM_BUG_ON(INTEL_GEN(dev_priv) < 6); + intel_ring_default_vfuncs(dev_priv, engine); engine->emit_flush = gen6_ring_flush; engine->irq_enable_mask = GT_BLT_USER_INTERRUPT; + if (IS_GEN(dev_priv, 6)) + engine->emit_fini_breadcrumb = gen6_xcs_emit_breadcrumb; + else + engine->emit_fini_breadcrumb = gen7_xcs_emit_breadcrumb; + return intel_init_ring_buffer(engine); } @@ -2322,6 +2399,8 @@ int intel_init_vebox_ring_buffer(struct intel_engine_cs *engine) { struct drm_i915_private *dev_priv = engine->i915; + GEM_BUG_ON(INTEL_GEN(dev_priv) < 7); + intel_ring_default_vfuncs(dev_priv, engine); engine->emit_flush = gen6_ring_flush; @@ -2329,5 +2408,7 @@ int intel_init_vebox_ring_buffer(struct intel_engine_cs *engine) engine->irq_enable = hsw_vebox_irq_enable; engine->irq_disable = hsw_vebox_irq_disable; + engine->emit_fini_breadcrumb = gen7_xcs_emit_breadcrumb; + return intel_init_ring_buffer(engine); } diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h index 767a7192c969..710ffb221775 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.h +++ b/drivers/gpu/drm/i915/intel_ringbuffer.h @@ -1,10 +1,11 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: MIT */ #ifndef _INTEL_RINGBUFFER_H_ #define _INTEL_RINGBUFFER_H_ #include <drm/drm_util.h> #include <linux/hashtable.h> +#include <linux/irq_work.h> #include <linux/seqlock.h> #include "i915_gem_batch_pool.h" @@ -28,12 +29,11 @@ struct i915_sched_attr; * workarounds! */ #define CACHELINE_BYTES 64 -#define CACHELINE_DWORDS (CACHELINE_BYTES / sizeof(uint32_t)) +#define CACHELINE_DWORDS (CACHELINE_BYTES / sizeof(u32)) struct intel_hw_status_page { struct i915_vma *vma; - u32 *page_addr; - u32 ggtt_offset; + u32 *addr; }; #define I915_READ_TAIL(engine) I915_READ(RING_TAIL((engine)->mmio_base)) @@ -94,12 +94,12 @@ hangcheck_action_to_str(const enum intel_engine_hangcheck_action a) #define I915_MAX_SUBSLICES 8 #define instdone_slice_mask(dev_priv__) \ - (INTEL_GEN(dev_priv__) == 7 ? \ - 1 : INTEL_INFO(dev_priv__)->sseu.slice_mask) + (IS_GEN(dev_priv__, 7) ? \ + 1 : RUNTIME_INFO(dev_priv__)->sseu.slice_mask) #define instdone_subslice_mask(dev_priv__) \ - (INTEL_GEN(dev_priv__) == 7 ? \ - 1 : INTEL_INFO(dev_priv__)->sseu.subslice_mask[0]) + (IS_GEN(dev_priv__, 7) ? \ + 1 : RUNTIME_INFO(dev_priv__)->sseu.subslice_mask[0]) #define for_each_instdone_slice_subslice(dev_priv__, slice__, subslice__) \ for ((slice__) = 0, (subslice__) = 0; \ @@ -120,13 +120,8 @@ struct intel_instdone { struct intel_engine_hangcheck { u64 acthd; u32 seqno; - enum intel_engine_hangcheck_action action; unsigned long action_timestamp; - int deadlock; struct intel_instdone instdone; - struct i915_request *active_request; - bool stalled:1; - bool wedged:1; }; struct intel_ring { @@ -191,13 +186,25 @@ enum intel_engine_id { }; struct i915_priolist { + struct list_head requests[I915_PRIORITY_COUNT]; struct rb_node node; - struct list_head requests; + unsigned long used; int priority; }; +#define priolist_for_each_request(it, plist, idx) \ + for (idx = 0; idx < ARRAY_SIZE((plist)->requests); idx++) \ + list_for_each_entry(it, &(plist)->requests[idx], sched.link) + +#define priolist_for_each_request_consume(it, n, plist, idx) \ + for (; (idx = ffs((plist)->used)); (plist)->used &= ~BIT(idx - 1)) \ + list_for_each_entry_safe(it, n, \ + &(plist)->requests[idx - 1], \ + sched.link) + struct st_preempt_hang { struct completion completion; + unsigned int count; bool inject_hang; }; @@ -288,14 +295,18 @@ struct intel_engine_execlists { unsigned int port_mask; /** - * @queue_priority: Highest pending priority. + * @queue_priority_hint: Highest pending priority. * * When we add requests into the queue, or adjust the priority of * executing requests, we compute the maximum priority of those * pending requests. We can then use this value to determine if * we need to preempt the executing requests to service the queue. + * However, since the we may have recorded the priority of an inflight + * request we wanted to preempt but since completed, at the time of + * dequeuing the priority hint may no longer may match the highest + * available request priority. */ - int queue_priority; + int queue_priority_hint; /** * @queue: queue of requests, in priority lists @@ -303,13 +314,6 @@ struct intel_engine_execlists { struct rb_root_cached queue; /** - * @csb_read: control register for Context Switch buffer - * - * Note this register is always in mmio. - */ - u32 __iomem *csb_read; - - /** * @csb_write: control register for Context Switch buffer * * Note this register may be either mmio or HWSP shadow. @@ -329,15 +333,6 @@ struct intel_engine_execlists { u32 preempt_complete_status; /** - * @csb_write_reset: reset value for CSB write pointer - * - * As the CSB write pointer maybe either in HWSP or as a field - * inside an mmio register, we want to reprogram it slightly - * differently to avoid later confusion. - */ - u32 csb_write_reset; - - /** * @csb_head: context status buffer head */ u8 csb_head; @@ -370,9 +365,6 @@ struct intel_engine_cs { struct drm_i915_gem_object *default_state; void *pinned_default_state; - unsigned long irq_posted; -#define ENGINE_IRQ_BREADCRUMB 0 - /* Rather than have every client wait upon all user interrupts, * with the herd waking after every interrupt and each doing the * heavyweight seqno dance, we delegate the task (of being the @@ -390,23 +382,14 @@ struct intel_engine_cs { * the overhead of waking that client is much preferred. */ struct intel_breadcrumbs { - spinlock_t irq_lock; /* protects irq_*; irqsafe */ - struct intel_wait *irq_wait; /* oldest waiter by retirement */ + spinlock_t irq_lock; + struct list_head signalers; - spinlock_t rb_lock; /* protects the rb and wraps irq_lock */ - struct rb_root waiters; /* sorted by retirement, priority */ - struct list_head signals; /* sorted by retirement */ - struct task_struct *signaler; /* used for fence signalling */ + struct irq_work irq_work; /* for use from inside irq_lock */ - struct timer_list fake_irq; /* used after a missed interrupt */ - struct timer_list hangcheck; /* detect missed interrupts */ - - unsigned int hangcheck_interrupts; unsigned int irq_enabled; - unsigned int irq_count; - bool irq_armed : 1; - I915_SELFTEST_DECLARE(bool mock : 1); + bool irq_armed; } breadcrumbs; struct { @@ -420,16 +403,17 @@ struct intel_engine_cs { /** * @enable_count: Reference count for the enabled samplers. * - * Index number corresponds to the bit number from @enable. + * Index number corresponds to @enum drm_i915_pmu_engine_sample. */ - unsigned int enable_count[I915_PMU_SAMPLE_BITS]; + unsigned int enable_count[I915_ENGINE_SAMPLE_COUNT]; /** * @sample: Counter values for sampling events. * * Our internal timer stores the current counters in this field. + * + * Index number corresponds to @enum drm_i915_pmu_engine_sample. */ -#define I915_ENGINE_SAMPLE_MAX (I915_SAMPLE_SEMA + 1) - struct i915_pmu_sample sample[I915_ENGINE_SAMPLE_MAX]; + struct i915_pmu_sample sample[I915_ENGINE_SAMPLE_COUNT]; } pmu; /* @@ -441,7 +425,9 @@ struct intel_engine_cs { struct intel_hw_status_page status_page; struct i915_ctx_workarounds wa_ctx; + struct i915_wa_list ctx_wa_list; struct i915_wa_list wa_list; + struct i915_wa_list whitelist; u32 irq_keep_mask; /* always keep these interrupts */ u32 irq_enable_mask; /* bitmask to enable ring interrupt */ @@ -451,9 +437,8 @@ struct intel_engine_cs { int (*init_hw)(struct intel_engine_cs *engine); struct { - struct i915_request *(*prepare)(struct intel_engine_cs *engine); - void (*reset)(struct intel_engine_cs *engine, - struct i915_request *rq); + void (*prepare)(struct intel_engine_cs *engine); + void (*reset)(struct intel_engine_cs *engine, bool stalled); void (*finish)(struct intel_engine_cs *engine); } reset; @@ -477,8 +462,10 @@ struct intel_engine_cs { unsigned int dispatch_flags); #define I915_DISPATCH_SECURE BIT(0) #define I915_DISPATCH_PINNED BIT(1) - void (*emit_breadcrumb)(struct i915_request *rq, u32 *cs); - int emit_breadcrumb_sz; + int (*emit_init_breadcrumb)(struct i915_request *rq); + u32 *(*emit_fini_breadcrumb)(struct i915_request *rq, + u32 *cs); + unsigned int emit_fini_breadcrumb_dw; /* Pass the request to the hardware queue (e.g. directly into * the legacy ringbuffer or to the end of an execlist). @@ -488,11 +475,10 @@ struct intel_engine_cs { */ void (*submit_request)(struct i915_request *rq); - /* Call when the priority on a request has changed and it and its + /* + * Call when the priority on a request has changed and it and its * dependencies may need rescheduling. Note the request itself may * not be ready to run! - * - * Called under the struct_mutex. */ void (*schedule)(struct i915_request *request, const struct i915_sched_attr *attr); @@ -505,69 +491,8 @@ struct intel_engine_cs { */ void (*cancel_requests)(struct intel_engine_cs *engine); - /* Some chipsets are not quite as coherent as advertised and need - * an expensive kick to force a true read of the up-to-date seqno. - * However, the up-to-date seqno is not always required and the last - * seen value is good enough. Note that the seqno will always be - * monotonic, even if not coherent. - */ - void (*irq_seqno_barrier)(struct intel_engine_cs *engine); void (*cleanup)(struct intel_engine_cs *engine); - /* GEN8 signal/wait table - never trust comments! - * signal to signal to signal to signal to signal to - * RCS VCS BCS VECS VCS2 - * -------------------------------------------------------------------- - * RCS | NOP (0x00) | VCS (0x08) | BCS (0x10) | VECS (0x18) | VCS2 (0x20) | - * |------------------------------------------------------------------- - * VCS | RCS (0x28) | NOP (0x30) | BCS (0x38) | VECS (0x40) | VCS2 (0x48) | - * |------------------------------------------------------------------- - * BCS | RCS (0x50) | VCS (0x58) | NOP (0x60) | VECS (0x68) | VCS2 (0x70) | - * |------------------------------------------------------------------- - * VECS | RCS (0x78) | VCS (0x80) | BCS (0x88) | NOP (0x90) | VCS2 (0x98) | - * |------------------------------------------------------------------- - * VCS2 | RCS (0xa0) | VCS (0xa8) | BCS (0xb0) | VECS (0xb8) | NOP (0xc0) | - * |------------------------------------------------------------------- - * - * Generalization: - * f(x, y) := (x->id * NUM_RINGS * seqno_size) + (seqno_size * y->id) - * ie. transpose of g(x, y) - * - * sync from sync from sync from sync from sync from - * RCS VCS BCS VECS VCS2 - * -------------------------------------------------------------------- - * RCS | NOP (0x00) | VCS (0x28) | BCS (0x50) | VECS (0x78) | VCS2 (0xa0) | - * |------------------------------------------------------------------- - * VCS | RCS (0x08) | NOP (0x30) | BCS (0x58) | VECS (0x80) | VCS2 (0xa8) | - * |------------------------------------------------------------------- - * BCS | RCS (0x10) | VCS (0x38) | NOP (0x60) | VECS (0x88) | VCS2 (0xb0) | - * |------------------------------------------------------------------- - * VECS | RCS (0x18) | VCS (0x40) | BCS (0x68) | NOP (0x90) | VCS2 (0xb8) | - * |------------------------------------------------------------------- - * VCS2 | RCS (0x20) | VCS (0x48) | BCS (0x70) | VECS (0x98) | NOP (0xc0) | - * |------------------------------------------------------------------- - * - * Generalization: - * g(x, y) := (y->id * NUM_RINGS * seqno_size) + (seqno_size * x->id) - * ie. transpose of f(x, y) - */ - struct { -#define GEN6_SEMAPHORE_LAST VECS_HW -#define GEN6_NUM_SEMAPHORES (GEN6_SEMAPHORE_LAST + 1) -#define GEN6_SEMAPHORES_MASK GENMASK(GEN6_SEMAPHORE_LAST, 0) - struct { - /* our mbox written by others */ - u32 wait[GEN6_NUM_SEMAPHORES]; - /* mboxes this ring signals to */ - i915_reg_t signal[GEN6_NUM_SEMAPHORES]; - } mbox; - - /* AKA wait() */ - int (*sync_to)(struct i915_request *rq, - struct i915_request *signal); - u32 *(*signal)(struct i915_request *rq, u32 *cs); - } semaphore; - struct intel_engine_execlists execlists; /* Contexts are pinned whilst they are active on the GPU. The last @@ -668,7 +593,20 @@ intel_engine_has_preemption(const struct intel_engine_cs *engine) static inline bool __execlists_need_preempt(int prio, int last) { - return prio > max(0, last); + /* + * Allow preemption of low -> normal -> high, but we do + * not allow low priority tasks to preempt other low priority + * tasks under the impression that latency for low priority + * tasks does not matter (as much as background throughput), + * so kiss. + * + * More naturally we would write + * prio >= max(0, last); + * except that we wish to prevent triggering preemption at the same + * priority level: the task that is running should remain running + * to preserve FIFO ordering of dependencies. + */ + return prio > max(I915_PRIORITY_NORMAL - 1, last); } static inline void @@ -746,7 +684,7 @@ static inline u32 intel_read_status_page(const struct intel_engine_cs *engine, int reg) { /* Ensure that the compiler doesn't optimize away the load. */ - return READ_ONCE(engine->status_page.page_addr[reg]); + return READ_ONCE(engine->status_page.addr[reg]); } static inline void @@ -759,12 +697,12 @@ intel_write_status_page(struct intel_engine_cs *engine, int reg, u32 value) */ if (static_cpu_has(X86_FEATURE_CLFLUSH)) { mb(); - clflush(&engine->status_page.page_addr[reg]); - engine->status_page.page_addr[reg] = value; - clflush(&engine->status_page.page_addr[reg]); + clflush(&engine->status_page.addr[reg]); + engine->status_page.addr[reg] = value; + clflush(&engine->status_page.addr[reg]); mb(); } else { - WRITE_ONCE(engine->status_page.page_addr[reg], value); + WRITE_ONCE(engine->status_page.addr[reg], value); } } @@ -785,11 +723,13 @@ intel_write_status_page(struct intel_engine_cs *engine, int reg, u32 value) * The area from dword 0x30 to 0x3ff is available for driver usage. */ #define I915_GEM_HWS_INDEX 0x30 -#define I915_GEM_HWS_INDEX_ADDR (I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT) -#define I915_GEM_HWS_PREEMPT_INDEX 0x32 -#define I915_GEM_HWS_PREEMPT_ADDR (I915_GEM_HWS_PREEMPT_INDEX << MI_STORE_DWORD_INDEX_SHIFT) -#define I915_GEM_HWS_SCRATCH_INDEX 0x40 -#define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH_INDEX << MI_STORE_DWORD_INDEX_SHIFT) +#define I915_GEM_HWS_INDEX_ADDR (I915_GEM_HWS_INDEX * sizeof(u32)) +#define I915_GEM_HWS_PREEMPT 0x32 +#define I915_GEM_HWS_PREEMPT_ADDR (I915_GEM_HWS_PREEMPT * sizeof(u32)) +#define I915_GEM_HWS_SEQNO 0x40 +#define I915_GEM_HWS_SEQNO_ADDR (I915_GEM_HWS_SEQNO * sizeof(u32)) +#define I915_GEM_HWS_SCRATCH 0x80 +#define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH * sizeof(u32)) #define I915_HWS_CSB_BUF0_INDEX 0x10 #define I915_HWS_CSB_WRITE_INDEX 0x1f @@ -812,7 +752,6 @@ void intel_legacy_submission_resume(struct drm_i915_private *dev_priv); int __must_check intel_ring_cacheline_align(struct i915_request *rq); -int intel_ring_wait_for_space(struct intel_ring *ring, unsigned int bytes); u32 __must_check *intel_ring_begin(struct i915_request *rq, unsigned int n); static inline void intel_ring_advance(struct i915_request *rq, u32 *cs) @@ -893,9 +832,21 @@ intel_ring_set_tail(struct intel_ring *ring, unsigned int tail) return tail; } -void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno); +static inline unsigned int +__intel_ring_space(unsigned int head, unsigned int tail, unsigned int size) +{ + /* + * "If the Ring Buffer Head Pointer and the Tail Pointer are on the + * same cacheline, the Head Pointer must not be greater than the Tail + * Pointer." + */ + GEM_BUG_ON(!is_power_of_2(size)); + return (head - tail - CACHELINE_BYTES) & (size - 1); +} -void intel_engine_setup_common(struct intel_engine_cs *engine); +void intel_engine_write_global_seqno(struct intel_engine_cs *engine, u32 seqno); + +int intel_engine_setup_common(struct intel_engine_cs *engine); int intel_engine_init_common(struct intel_engine_cs *engine); void intel_engine_cleanup_common(struct intel_engine_cs *engine); @@ -907,6 +858,8 @@ int intel_init_vebox_ring_buffer(struct intel_engine_cs *engine); int intel_engine_stop_cs(struct intel_engine_cs *engine); void intel_engine_cancel_stop_cs(struct intel_engine_cs *engine); +void intel_engine_set_hwsp_writemask(struct intel_engine_cs *engine, u32 mask); + u64 intel_engine_get_active_head(const struct intel_engine_cs *engine); u64 intel_engine_get_last_batch_head(const struct intel_engine_cs *engine); @@ -951,102 +904,29 @@ static inline bool intel_engine_has_started(struct intel_engine_cs *engine, void intel_engine_get_instdone(struct intel_engine_cs *engine, struct intel_instdone *instdone); -/* - * Arbitrary size for largest possible 'add request' sequence. The code paths - * are complex and variable. Empirical measurement shows that the worst case - * is BDW at 192 bytes (6 + 6 + 36 dwords), then ILK at 136 bytes. However, - * we need to allocate double the largest single packet within that emission - * to account for tail wraparound (so 6 + 6 + 72 dwords for BDW). - */ -#define MIN_SPACE_FOR_ADD_REQUEST 336 - -static inline u32 intel_hws_seqno_address(struct intel_engine_cs *engine) -{ - return engine->status_page.ggtt_offset + I915_GEM_HWS_INDEX_ADDR; -} - -static inline u32 intel_hws_preempt_done_address(struct intel_engine_cs *engine) -{ - return engine->status_page.ggtt_offset + I915_GEM_HWS_PREEMPT_ADDR; -} - -/* intel_breadcrumbs.c -- user interrupt bottom-half for waiters */ -int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine); - -static inline void intel_wait_init(struct intel_wait *wait) -{ - wait->tsk = current; - wait->request = NULL; -} - -static inline void intel_wait_init_for_seqno(struct intel_wait *wait, u32 seqno) -{ - wait->tsk = current; - wait->seqno = seqno; -} - -static inline bool intel_wait_has_seqno(const struct intel_wait *wait) -{ - return wait->seqno; -} - -static inline bool -intel_wait_update_seqno(struct intel_wait *wait, u32 seqno) -{ - wait->seqno = seqno; - return intel_wait_has_seqno(wait); -} - -static inline bool -intel_wait_update_request(struct intel_wait *wait, - const struct i915_request *rq) -{ - return intel_wait_update_seqno(wait, i915_request_global_seqno(rq)); -} - -static inline bool -intel_wait_check_seqno(const struct intel_wait *wait, u32 seqno) -{ - return wait->seqno == seqno; -} - -static inline bool -intel_wait_check_request(const struct intel_wait *wait, - const struct i915_request *rq) -{ - return intel_wait_check_seqno(wait, i915_request_global_seqno(rq)); -} +void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine); +void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine); -static inline bool intel_wait_complete(const struct intel_wait *wait) -{ - return RB_EMPTY_NODE(&wait->node); -} +void intel_engine_pin_breadcrumbs_irq(struct intel_engine_cs *engine); +void intel_engine_unpin_breadcrumbs_irq(struct intel_engine_cs *engine); -bool intel_engine_add_wait(struct intel_engine_cs *engine, - struct intel_wait *wait); -void intel_engine_remove_wait(struct intel_engine_cs *engine, - struct intel_wait *wait); -bool intel_engine_enable_signaling(struct i915_request *request, bool wakeup); -void intel_engine_cancel_signaling(struct i915_request *request); +bool intel_engine_signal_breadcrumbs(struct intel_engine_cs *engine); +void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine); -static inline bool intel_engine_has_waiter(const struct intel_engine_cs *engine) +static inline void +intel_engine_queue_breadcrumbs(struct intel_engine_cs *engine) { - return READ_ONCE(engine->breadcrumbs.irq_wait); + irq_work_queue(&engine->breadcrumbs.irq_work); } -unsigned int intel_engine_wakeup(struct intel_engine_cs *engine); -#define ENGINE_WAKEUP_WAITER BIT(0) -#define ENGINE_WAKEUP_ASLEEP BIT(1) - -void intel_engine_pin_breadcrumbs_irq(struct intel_engine_cs *engine); -void intel_engine_unpin_breadcrumbs_irq(struct intel_engine_cs *engine); - -void __intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine); -void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine); +bool intel_engine_breadcrumbs_irq(struct intel_engine_cs *engine); void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine); void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine); +void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine, + struct drm_printer *p); + static inline u32 *gen8_emit_pipe_control(u32 *batch, u32 flags, u32 offset) { memset(batch, 0, 6 * sizeof(u32)); @@ -1059,7 +939,7 @@ static inline u32 *gen8_emit_pipe_control(u32 *batch, u32 flags, u32 offset) } static inline u32 * -gen8_emit_ggtt_write_rcs(u32 *cs, u32 value, u32 gtt_offset) +gen8_emit_ggtt_write_rcs(u32 *cs, u32 value, u32 gtt_offset, u32 flags) { /* We're using qword write, offset should be aligned to 8 bytes. */ GEM_BUG_ON(!IS_ALIGNED(gtt_offset, 8)); @@ -1069,8 +949,7 @@ gen8_emit_ggtt_write_rcs(u32 *cs, u32 value, u32 gtt_offset) * following the batch. */ *cs++ = GFX_OP_PIPE_CONTROL(6); - *cs++ = PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_CS_STALL | - PIPE_CONTROL_QW_WRITE; + *cs++ = flags | PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_GLOBAL_GTT_IVB; *cs++ = gtt_offset; *cs++ = 0; *cs++ = value; @@ -1096,7 +975,14 @@ gen8_emit_ggtt_write(u32 *cs, u32 value, u32 gtt_offset) return cs; } -void intel_engines_sanitize(struct drm_i915_private *i915); +static inline void intel_engine_reset(struct intel_engine_cs *engine, + bool stalled) +{ + if (engine->reset.reset) + engine->reset.reset(engine, stalled); +} + +void intel_engines_sanitize(struct drm_i915_private *i915, bool force); bool intel_engine_is_idle(struct intel_engine_cs *engine); bool intel_engines_are_idle(struct drm_i915_private *dev_priv); diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c index 44e4491a4918..a017a4232c0f 100644 --- a/drivers/gpu/drm/i915/intel_runtime_pm.c +++ b/drivers/gpu/drm/i915/intel_runtime_pm.c @@ -29,6 +29,8 @@ #include <linux/pm_runtime.h> #include <linux/vgaarb.h> +#include <drm/drm_print.h> + #include "i915_drv.h" #include "intel_drv.h" @@ -49,6 +51,268 @@ * present for a given platform. */ +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM) + +#include <linux/sort.h> + +#define STACKDEPTH 8 + +static noinline depot_stack_handle_t __save_depot_stack(void) +{ + unsigned long entries[STACKDEPTH]; + struct stack_trace trace = { + .entries = entries, + .max_entries = ARRAY_SIZE(entries), + .skip = 1, + }; + + save_stack_trace(&trace); + if (trace.nr_entries && + trace.entries[trace.nr_entries - 1] == ULONG_MAX) + trace.nr_entries--; + + return depot_save_stack(&trace, GFP_NOWAIT | __GFP_NOWARN); +} + +static void __print_depot_stack(depot_stack_handle_t stack, + char *buf, int sz, int indent) +{ + unsigned long entries[STACKDEPTH]; + struct stack_trace trace = { + .entries = entries, + .max_entries = ARRAY_SIZE(entries), + }; + + depot_fetch_stack(stack, &trace); + snprint_stack_trace(buf, sz, &trace, indent); +} + +static void init_intel_runtime_pm_wakeref(struct drm_i915_private *i915) +{ + struct i915_runtime_pm *rpm = &i915->runtime_pm; + + spin_lock_init(&rpm->debug.lock); +} + +static noinline depot_stack_handle_t +track_intel_runtime_pm_wakeref(struct drm_i915_private *i915) +{ + struct i915_runtime_pm *rpm = &i915->runtime_pm; + depot_stack_handle_t stack, *stacks; + unsigned long flags; + + atomic_inc(&rpm->wakeref_count); + assert_rpm_wakelock_held(i915); + + if (!HAS_RUNTIME_PM(i915)) + return -1; + + stack = __save_depot_stack(); + if (!stack) + return -1; + + spin_lock_irqsave(&rpm->debug.lock, flags); + + if (!rpm->debug.count) + rpm->debug.last_acquire = stack; + + stacks = krealloc(rpm->debug.owners, + (rpm->debug.count + 1) * sizeof(*stacks), + GFP_NOWAIT | __GFP_NOWARN); + if (stacks) { + stacks[rpm->debug.count++] = stack; + rpm->debug.owners = stacks; + } else { + stack = -1; + } + + spin_unlock_irqrestore(&rpm->debug.lock, flags); + + return stack; +} + +static void cancel_intel_runtime_pm_wakeref(struct drm_i915_private *i915, + depot_stack_handle_t stack) +{ + struct i915_runtime_pm *rpm = &i915->runtime_pm; + unsigned long flags, n; + bool found = false; + + if (unlikely(stack == -1)) + return; + + spin_lock_irqsave(&rpm->debug.lock, flags); + for (n = rpm->debug.count; n--; ) { + if (rpm->debug.owners[n] == stack) { + memmove(rpm->debug.owners + n, + rpm->debug.owners + n + 1, + (--rpm->debug.count - n) * sizeof(stack)); + found = true; + break; + } + } + spin_unlock_irqrestore(&rpm->debug.lock, flags); + + if (WARN(!found, + "Unmatched wakeref (tracking %lu), count %u\n", + rpm->debug.count, atomic_read(&rpm->wakeref_count))) { + char *buf; + + buf = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!buf) + return; + + __print_depot_stack(stack, buf, PAGE_SIZE, 2); + DRM_DEBUG_DRIVER("wakeref %x from\n%s", stack, buf); + + stack = READ_ONCE(rpm->debug.last_release); + if (stack) { + __print_depot_stack(stack, buf, PAGE_SIZE, 2); + DRM_DEBUG_DRIVER("wakeref last released at\n%s", buf); + } + + kfree(buf); + } +} + +static int cmphandle(const void *_a, const void *_b) +{ + const depot_stack_handle_t * const a = _a, * const b = _b; + + if (*a < *b) + return -1; + else if (*a > *b) + return 1; + else + return 0; +} + +static void +__print_intel_runtime_pm_wakeref(struct drm_printer *p, + const struct intel_runtime_pm_debug *dbg) +{ + unsigned long i; + char *buf; + + buf = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!buf) + return; + + if (dbg->last_acquire) { + __print_depot_stack(dbg->last_acquire, buf, PAGE_SIZE, 2); + drm_printf(p, "Wakeref last acquired:\n%s", buf); + } + + if (dbg->last_release) { + __print_depot_stack(dbg->last_release, buf, PAGE_SIZE, 2); + drm_printf(p, "Wakeref last released:\n%s", buf); + } + + drm_printf(p, "Wakeref count: %lu\n", dbg->count); + + sort(dbg->owners, dbg->count, sizeof(*dbg->owners), cmphandle, NULL); + + for (i = 0; i < dbg->count; i++) { + depot_stack_handle_t stack = dbg->owners[i]; + unsigned long rep; + + rep = 1; + while (i + 1 < dbg->count && dbg->owners[i + 1] == stack) + rep++, i++; + __print_depot_stack(stack, buf, PAGE_SIZE, 2); + drm_printf(p, "Wakeref x%lu taken at:\n%s", rep, buf); + } + + kfree(buf); +} + +static noinline void +untrack_intel_runtime_pm_wakeref(struct drm_i915_private *i915) +{ + struct i915_runtime_pm *rpm = &i915->runtime_pm; + struct intel_runtime_pm_debug dbg = {}; + struct drm_printer p; + unsigned long flags; + + assert_rpm_wakelock_held(i915); + if (atomic_dec_and_lock_irqsave(&rpm->wakeref_count, + &rpm->debug.lock, + flags)) { + dbg = rpm->debug; + + rpm->debug.owners = NULL; + rpm->debug.count = 0; + rpm->debug.last_release = __save_depot_stack(); + + spin_unlock_irqrestore(&rpm->debug.lock, flags); + } + if (!dbg.count) + return; + + p = drm_debug_printer("i915"); + __print_intel_runtime_pm_wakeref(&p, &dbg); + + kfree(dbg.owners); +} + +void print_intel_runtime_pm_wakeref(struct drm_i915_private *i915, + struct drm_printer *p) +{ + struct intel_runtime_pm_debug dbg = {}; + + do { + struct i915_runtime_pm *rpm = &i915->runtime_pm; + unsigned long alloc = dbg.count; + depot_stack_handle_t *s; + + spin_lock_irq(&rpm->debug.lock); + dbg.count = rpm->debug.count; + if (dbg.count <= alloc) { + memcpy(dbg.owners, + rpm->debug.owners, + dbg.count * sizeof(*s)); + } + dbg.last_acquire = rpm->debug.last_acquire; + dbg.last_release = rpm->debug.last_release; + spin_unlock_irq(&rpm->debug.lock); + if (dbg.count <= alloc) + break; + + s = krealloc(dbg.owners, dbg.count * sizeof(*s), GFP_KERNEL); + if (!s) + goto out; + + dbg.owners = s; + } while (1); + + __print_intel_runtime_pm_wakeref(p, &dbg); + +out: + kfree(dbg.owners); +} + +#else + +static void init_intel_runtime_pm_wakeref(struct drm_i915_private *i915) +{ +} + +static depot_stack_handle_t +track_intel_runtime_pm_wakeref(struct drm_i915_private *i915) +{ + atomic_inc(&i915->runtime_pm.wakeref_count); + assert_rpm_wakelock_held(i915); + return -1; +} + +static void untrack_intel_runtime_pm_wakeref(struct drm_i915_private *i915) +{ + assert_rpm_wakelock_held(i915); + atomic_dec(&i915->runtime_pm.wakeref_count); +} + +#endif + bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv, enum i915_power_well_id power_well_id); @@ -76,6 +340,8 @@ intel_display_power_domain_str(enum intel_display_power_domain domain) return "TRANSCODER_C"; case POWER_DOMAIN_TRANSCODER_EDP: return "TRANSCODER_EDP"; + case POWER_DOMAIN_TRANSCODER_EDP_VDSC: + return "TRANSCODER_EDP_VDSC"; case POWER_DOMAIN_TRANSCODER_DSI_A: return "TRANSCODER_DSI_A"; case POWER_DOMAIN_TRANSCODER_DSI_C: @@ -208,7 +474,7 @@ bool __intel_display_power_is_enabled(struct drm_i915_private *dev_priv, is_enabled = true; - for_each_power_domain_well_rev(dev_priv, power_well, BIT_ULL(domain)) { + for_each_power_domain_well_reverse(dev_priv, power_well, BIT_ULL(domain)) { if (power_well->desc->always_on) continue; @@ -436,6 +702,15 @@ icl_combo_phy_aux_power_well_enable(struct drm_i915_private *dev_priv, I915_WRITE(ICL_PORT_CL_DW12(port), val | ICL_LANE_ENABLE_AUX); hsw_wait_for_power_well_enable(dev_priv, power_well); + + /* Display WA #1178: icl */ + if (IS_ICELAKE(dev_priv) && + pw_idx >= ICL_PW_CTL_IDX_AUX_A && pw_idx <= ICL_PW_CTL_IDX_AUX_B && + !intel_bios_is_port_edp(dev_priv, port)) { + val = I915_READ(ICL_AUX_ANAOVRD1(pw_idx)); + val |= ICL_AUX_ANAOVRD1_ENABLE | ICL_AUX_ANAOVRD1_LDO_BYPASS; + I915_WRITE(ICL_AUX_ANAOVRD1(pw_idx), val); + } } static void @@ -456,6 +731,25 @@ icl_combo_phy_aux_power_well_disable(struct drm_i915_private *dev_priv, hsw_wait_for_power_well_disable(dev_priv, power_well); } +#define ICL_AUX_PW_TO_CH(pw_idx) \ + ((pw_idx) - ICL_PW_CTL_IDX_AUX_A + AUX_CH_A) + +static void +icl_tc_phy_aux_power_well_enable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + enum aux_ch aux_ch = ICL_AUX_PW_TO_CH(power_well->desc->hsw.idx); + u32 val; + + val = I915_READ(DP_AUX_CH_CTL(aux_ch)); + val &= ~DP_AUX_CH_CTL_TBT_IO; + if (power_well->desc->hsw.is_tc_tbt) + val |= DP_AUX_CH_CTL_TBT_IO; + I915_WRITE(DP_AUX_CH_CTL(aux_ch), val); + + hsw_power_well_enable(dev_priv, power_well); +} + /* * We should only use the power well if we explicitly asked the hardware to * enable it, so check if it's enabled and also check if we've requested it to @@ -465,11 +759,25 @@ static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { const struct i915_power_well_regs *regs = power_well->desc->hsw.regs; + enum i915_power_well_id id = power_well->desc->id; int pw_idx = power_well->desc->hsw.idx; u32 mask = HSW_PWR_WELL_CTL_REQ(pw_idx) | HSW_PWR_WELL_CTL_STATE(pw_idx); + u32 val; + + val = I915_READ(regs->driver); + + /* + * On GEN9 big core due to a DMC bug the driver's request bits for PW1 + * and the MISC_IO PW will be not restored, so check instead for the + * BIOS's own request bits, which are forced-on for these power wells + * when exiting DC5/6. + */ + if (IS_GEN(dev_priv, 9) && !IS_GEN9_LP(dev_priv) && + (id == SKL_DISP_PW_1 || id == SKL_DISP_PW_MISC_IO)) + val |= I915_READ(regs->bios); - return (I915_READ(regs->driver) & mask) == mask; + return (val & mask) == mask; } static void assert_can_enable_dc9(struct drm_i915_private *dev_priv) @@ -551,7 +859,9 @@ static u32 gen9_dc_mask(struct drm_i915_private *dev_priv) u32 mask; mask = DC_STATE_EN_UPTO_DC5; - if (IS_GEN9_LP(dev_priv)) + if (INTEL_GEN(dev_priv) >= 11) + mask |= DC_STATE_EN_UPTO_DC6 | DC_STATE_EN_DC9; + else if (IS_GEN9_LP(dev_priv)) mask |= DC_STATE_EN_DC9; else mask |= DC_STATE_EN_UPTO_DC6; @@ -593,10 +903,10 @@ void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv) * back on and register state is restored. This is guaranteed by the MMIO write * to DC_STATE_EN blocking until the state is restored. */ -static void gen9_set_dc_state(struct drm_i915_private *dev_priv, uint32_t state) +static void gen9_set_dc_state(struct drm_i915_private *dev_priv, u32 state) { - uint32_t val; - uint32_t mask; + u32 val; + u32 mask; if (WARN_ON_ONCE(state & ~dev_priv->csr.allowed_dc_mask)) state &= dev_priv->csr.allowed_dc_mask; @@ -624,8 +934,13 @@ void bxt_enable_dc9(struct drm_i915_private *dev_priv) assert_can_enable_dc9(dev_priv); DRM_DEBUG_KMS("Enabling DC9\n"); - - intel_power_sequencer_reset(dev_priv); + /* + * Power sequencer reset is not needed on + * platforms with South Display Engine on PCH, + * because PPS registers are always on. + */ + if (!HAS_PCH_SPLIT(dev_priv)) + intel_power_sequencer_reset(dev_priv); gen9_set_dc_state(dev_priv, DC_STATE_EN_DC9); } @@ -707,7 +1022,7 @@ static void assert_can_enable_dc6(struct drm_i915_private *dev_priv) assert_csr_loaded(dev_priv); } -static void skl_enable_dc6(struct drm_i915_private *dev_priv) +void skl_enable_dc6(struct drm_i915_private *dev_priv) { assert_can_enable_dc6(dev_priv); @@ -808,6 +1123,14 @@ static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv, if (IS_GEN9_LP(dev_priv)) bxt_verify_ddi_phy_power_wells(dev_priv); + + if (INTEL_GEN(dev_priv) >= 11) + /* + * DMC retains HW context only for port A, the other combo + * PHY's HW context for port B is lost after DC transitions, + * so we need to restore it manually. + */ + icl_combo_phys_init(dev_priv); } static void gen9_dc_off_power_well_disable(struct drm_i915_private *dev_priv, @@ -1215,7 +1538,7 @@ static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv, { enum dpio_phy phy; enum pipe pipe; - uint32_t tmp; + u32 tmp; WARN_ON_ONCE(power_well->desc->id != VLV_DISP_PW_DPIO_CMN_BC && power_well->desc->id != CHV_DISP_PW_DPIO_CMN_D); @@ -1532,18 +1855,19 @@ __intel_display_power_get_domain(struct drm_i915_private *dev_priv, * Any power domain reference obtained by this function must have a symmetric * call to intel_display_power_put() to release the reference again. */ -void intel_display_power_get(struct drm_i915_private *dev_priv, - enum intel_display_power_domain domain) +intel_wakeref_t intel_display_power_get(struct drm_i915_private *dev_priv, + enum intel_display_power_domain domain) { struct i915_power_domains *power_domains = &dev_priv->power_domains; - - intel_runtime_pm_get(dev_priv); + intel_wakeref_t wakeref = intel_runtime_pm_get(dev_priv); mutex_lock(&power_domains->lock); __intel_display_power_get_domain(dev_priv, domain); mutex_unlock(&power_domains->lock); + + return wakeref; } /** @@ -1558,13 +1882,16 @@ void intel_display_power_get(struct drm_i915_private *dev_priv, * Any power domain reference obtained by this function must have a symmetric * call to intel_display_power_put() to release the reference again. */ -bool intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv, - enum intel_display_power_domain domain) +intel_wakeref_t +intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv, + enum intel_display_power_domain domain) { struct i915_power_domains *power_domains = &dev_priv->power_domains; + intel_wakeref_t wakeref; bool is_enabled; - if (!intel_runtime_pm_get_if_in_use(dev_priv)) + wakeref = intel_runtime_pm_get_if_in_use(dev_priv); + if (!wakeref) return false; mutex_lock(&power_domains->lock); @@ -1578,23 +1905,16 @@ bool intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv, mutex_unlock(&power_domains->lock); - if (!is_enabled) - intel_runtime_pm_put(dev_priv); + if (!is_enabled) { + intel_runtime_pm_put(dev_priv, wakeref); + wakeref = 0; + } - return is_enabled; + return wakeref; } -/** - * intel_display_power_put - release a power domain reference - * @dev_priv: i915 device instance - * @domain: power domain to reference - * - * This function drops the power domain reference obtained by - * intel_display_power_get() and might power down the corresponding hardware - * block right away if this is the last reference. - */ -void intel_display_power_put(struct drm_i915_private *dev_priv, - enum intel_display_power_domain domain) +static void __intel_display_power_put(struct drm_i915_private *dev_priv, + enum intel_display_power_domain domain) { struct i915_power_domains *power_domains; struct i915_power_well *power_well; @@ -1608,13 +1928,37 @@ void intel_display_power_put(struct drm_i915_private *dev_priv, intel_display_power_domain_str(domain)); power_domains->domain_use_count[domain]--; - for_each_power_domain_well_rev(dev_priv, power_well, BIT_ULL(domain)) + for_each_power_domain_well_reverse(dev_priv, power_well, BIT_ULL(domain)) intel_power_well_put(dev_priv, power_well); mutex_unlock(&power_domains->lock); +} + +/** + * intel_display_power_put - release a power domain reference + * @dev_priv: i915 device instance + * @domain: power domain to reference + * + * This function drops the power domain reference obtained by + * intel_display_power_get() and might power down the corresponding hardware + * block right away if this is the last reference. + */ +void intel_display_power_put_unchecked(struct drm_i915_private *dev_priv, + enum intel_display_power_domain domain) +{ + __intel_display_power_put(dev_priv, domain); + intel_runtime_pm_put_unchecked(dev_priv); +} - intel_runtime_pm_put(dev_priv); +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM) +void intel_display_power_put(struct drm_i915_private *dev_priv, + enum intel_display_power_domain domain, + intel_wakeref_t wakeref) +{ + __intel_display_power_put(dev_priv, domain); + intel_runtime_pm_put(dev_priv, wakeref); } +#endif #define I830_PIPES_POWER_DOMAINS ( \ BIT_ULL(POWER_DOMAIN_PIPE_A) | \ @@ -1971,9 +2315,9 @@ void intel_display_power_put(struct drm_i915_private *dev_priv, */ #define ICL_PW_2_POWER_DOMAINS ( \ ICL_PW_3_POWER_DOMAINS | \ + BIT_ULL(POWER_DOMAIN_TRANSCODER_EDP_VDSC) | \ BIT_ULL(POWER_DOMAIN_INIT)) /* - * - eDP/DSI VDSC * - KVMR (HW control) */ #define ICL_DISPLAY_DC_OFF_POWER_DOMAINS ( \ @@ -2041,7 +2385,7 @@ static const struct i915_power_well_ops chv_dpio_cmn_power_well_ops = { static const struct i915_power_well_desc i9xx_always_on_power_well[] = { { .name = "always-on", - .always_on = 1, + .always_on = true, .domains = POWER_DOMAIN_MASK, .ops = &i9xx_always_on_power_well_ops, .id = DISP_PW_ID_NONE, @@ -2058,7 +2402,7 @@ static const struct i915_power_well_ops i830_pipes_power_well_ops = { static const struct i915_power_well_desc i830_power_wells[] = { { .name = "always-on", - .always_on = 1, + .always_on = true, .domains = POWER_DOMAIN_MASK, .ops = &i9xx_always_on_power_well_ops, .id = DISP_PW_ID_NONE, @@ -2102,7 +2446,7 @@ static const struct i915_power_well_regs hsw_power_well_regs = { static const struct i915_power_well_desc hsw_power_wells[] = { { .name = "always-on", - .always_on = 1, + .always_on = true, .domains = POWER_DOMAIN_MASK, .ops = &i9xx_always_on_power_well_ops, .id = DISP_PW_ID_NONE, @@ -2123,7 +2467,7 @@ static const struct i915_power_well_desc hsw_power_wells[] = { static const struct i915_power_well_desc bdw_power_wells[] = { { .name = "always-on", - .always_on = 1, + .always_on = true, .domains = POWER_DOMAIN_MASK, .ops = &i9xx_always_on_power_well_ops, .id = DISP_PW_ID_NONE, @@ -2166,7 +2510,7 @@ static const struct i915_power_well_ops vlv_dpio_power_well_ops = { static const struct i915_power_well_desc vlv_power_wells[] = { { .name = "always-on", - .always_on = 1, + .always_on = true, .domains = POWER_DOMAIN_MASK, .ops = &i9xx_always_on_power_well_ops, .id = DISP_PW_ID_NONE, @@ -2242,7 +2586,7 @@ static const struct i915_power_well_desc vlv_power_wells[] = { static const struct i915_power_well_desc chv_power_wells[] = { { .name = "always-on", - .always_on = 1, + .always_on = true, .domains = POWER_DOMAIN_MASK, .ops = &i9xx_always_on_power_well_ops, .id = DISP_PW_ID_NONE, @@ -2293,7 +2637,7 @@ bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv, static const struct i915_power_well_desc skl_power_wells[] = { { .name = "always-on", - .always_on = 1, + .always_on = true, .domains = POWER_DOMAIN_MASK, .ops = &i9xx_always_on_power_well_ops, .id = DISP_PW_ID_NONE, @@ -2301,6 +2645,7 @@ static const struct i915_power_well_desc skl_power_wells[] = { { .name = "power well 1", /* Handled by the DMC firmware */ + .always_on = true, .domains = 0, .ops = &hsw_power_well_ops, .id = SKL_DISP_PW_1, @@ -2313,6 +2658,7 @@ static const struct i915_power_well_desc skl_power_wells[] = { { .name = "MISC IO power well", /* Handled by the DMC firmware */ + .always_on = true, .domains = 0, .ops = &hsw_power_well_ops, .id = SKL_DISP_PW_MISC_IO, @@ -2385,13 +2731,15 @@ static const struct i915_power_well_desc skl_power_wells[] = { static const struct i915_power_well_desc bxt_power_wells[] = { { .name = "always-on", - .always_on = 1, + .always_on = true, .domains = POWER_DOMAIN_MASK, .ops = &i9xx_always_on_power_well_ops, .id = DISP_PW_ID_NONE, }, { .name = "power well 1", + /* Handled by the DMC firmware */ + .always_on = true, .domains = 0, .ops = &hsw_power_well_ops, .id = SKL_DISP_PW_1, @@ -2443,7 +2791,7 @@ static const struct i915_power_well_desc bxt_power_wells[] = { static const struct i915_power_well_desc glk_power_wells[] = { { .name = "always-on", - .always_on = 1, + .always_on = true, .domains = POWER_DOMAIN_MASK, .ops = &i9xx_always_on_power_well_ops, .id = DISP_PW_ID_NONE, @@ -2451,6 +2799,7 @@ static const struct i915_power_well_desc glk_power_wells[] = { { .name = "power well 1", /* Handled by the DMC firmware */ + .always_on = true, .domains = 0, .ops = &hsw_power_well_ops, .id = SKL_DISP_PW_1, @@ -2571,7 +2920,7 @@ static const struct i915_power_well_desc glk_power_wells[] = { static const struct i915_power_well_desc cnl_power_wells[] = { { .name = "always-on", - .always_on = 1, + .always_on = true, .domains = POWER_DOMAIN_MASK, .ops = &i9xx_always_on_power_well_ops, .id = DISP_PW_ID_NONE, @@ -2579,6 +2928,7 @@ static const struct i915_power_well_desc cnl_power_wells[] = { { .name = "power well 1", /* Handled by the DMC firmware */ + .always_on = true, .domains = 0, .ops = &hsw_power_well_ops, .id = SKL_DISP_PW_1, @@ -2716,6 +3066,13 @@ static const struct i915_power_well_ops icl_combo_phy_aux_power_well_ops = { .is_enabled = hsw_power_well_enabled, }; +static const struct i915_power_well_ops icl_tc_phy_aux_power_well_ops = { + .sync_hw = hsw_power_well_sync_hw, + .enable = icl_tc_phy_aux_power_well_enable, + .disable = hsw_power_well_disable, + .is_enabled = hsw_power_well_enabled, +}; + static const struct i915_power_well_regs icl_aux_power_well_regs = { .bios = ICL_PWR_WELL_CTL_AUX1, .driver = ICL_PWR_WELL_CTL_AUX2, @@ -2731,7 +3088,7 @@ static const struct i915_power_well_regs icl_ddi_power_well_regs = { static const struct i915_power_well_desc icl_power_wells[] = { { .name = "always-on", - .always_on = 1, + .always_on = true, .domains = POWER_DOMAIN_MASK, .ops = &i9xx_always_on_power_well_ops, .id = DISP_PW_ID_NONE, @@ -2739,6 +3096,7 @@ static const struct i915_power_well_desc icl_power_wells[] = { { .name = "power well 1", /* Handled by the DMC firmware */ + .always_on = true, .domains = 0, .ops = &hsw_power_well_ops, .id = SKL_DISP_PW_1, @@ -2861,81 +3219,89 @@ static const struct i915_power_well_desc icl_power_wells[] = { { .name = "AUX C", .domains = ICL_AUX_C_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, + .ops = &icl_tc_phy_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, .hsw.idx = ICL_PW_CTL_IDX_AUX_C, + .hsw.is_tc_tbt = false, }, }, { .name = "AUX D", .domains = ICL_AUX_D_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, + .ops = &icl_tc_phy_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, .hsw.idx = ICL_PW_CTL_IDX_AUX_D, + .hsw.is_tc_tbt = false, }, }, { .name = "AUX E", .domains = ICL_AUX_E_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, + .ops = &icl_tc_phy_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, .hsw.idx = ICL_PW_CTL_IDX_AUX_E, + .hsw.is_tc_tbt = false, }, }, { .name = "AUX F", .domains = ICL_AUX_F_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, + .ops = &icl_tc_phy_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, .hsw.idx = ICL_PW_CTL_IDX_AUX_F, + .hsw.is_tc_tbt = false, }, }, { .name = "AUX TBT1", .domains = ICL_AUX_TBT1_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, + .ops = &icl_tc_phy_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, .hsw.idx = ICL_PW_CTL_IDX_AUX_TBT1, + .hsw.is_tc_tbt = true, }, }, { .name = "AUX TBT2", .domains = ICL_AUX_TBT2_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, + .ops = &icl_tc_phy_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, .hsw.idx = ICL_PW_CTL_IDX_AUX_TBT2, + .hsw.is_tc_tbt = true, }, }, { .name = "AUX TBT3", .domains = ICL_AUX_TBT3_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, + .ops = &icl_tc_phy_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, .hsw.idx = ICL_PW_CTL_IDX_AUX_TBT3, + .hsw.is_tc_tbt = true, }, }, { .name = "AUX TBT4", .domains = ICL_AUX_TBT4_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, + .ops = &icl_tc_phy_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, .hsw.idx = ICL_PW_CTL_IDX_AUX_TBT4, + .hsw.is_tc_tbt = true, }, }, { @@ -2962,24 +3328,27 @@ sanitize_disable_power_well_option(const struct drm_i915_private *dev_priv, return 1; } -static uint32_t get_allowed_dc_mask(const struct drm_i915_private *dev_priv, - int enable_dc) +static u32 get_allowed_dc_mask(const struct drm_i915_private *dev_priv, + int enable_dc) { - uint32_t mask; + u32 mask; int requested_dc; int max_dc; - if (IS_GEN9_BC(dev_priv) || INTEL_INFO(dev_priv)->gen >= 10) { + if (INTEL_GEN(dev_priv) >= 11) { max_dc = 2; - mask = 0; - } else if (IS_GEN9_LP(dev_priv)) { - max_dc = 1; /* * DC9 has a separate HW flow from the rest of the DC states, * not depending on the DMC firmware. It's needed by system * suspend/resume, so allow it unconditionally. */ mask = DC_STATE_EN_DC9; + } else if (IS_GEN(dev_priv, 10) || IS_GEN9_BC(dev_priv)) { + max_dc = 2; + mask = 0; + } else if (IS_GEN9_LP(dev_priv)) { + max_dc = 1; + mask = DC_STATE_EN_DC9; } else { max_dc = 0; mask = 0; @@ -3075,12 +3444,6 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv) */ if (IS_ICELAKE(dev_priv)) { err = set_power_wells(power_domains, icl_power_wells); - } else if (IS_HASWELL(dev_priv)) { - err = set_power_wells(power_domains, hsw_power_wells); - } else if (IS_BROADWELL(dev_priv)) { - err = set_power_wells(power_domains, bdw_power_wells); - } else if (IS_GEN9_BC(dev_priv)) { - err = set_power_wells(power_domains, skl_power_wells); } else if (IS_CANNONLAKE(dev_priv)) { err = set_power_wells(power_domains, cnl_power_wells); @@ -3092,13 +3455,18 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv) */ if (!IS_CNL_WITH_PORT_F(dev_priv)) power_domains->power_well_count -= 2; - - } else if (IS_BROXTON(dev_priv)) { - err = set_power_wells(power_domains, bxt_power_wells); } else if (IS_GEMINILAKE(dev_priv)) { err = set_power_wells(power_domains, glk_power_wells); + } else if (IS_BROXTON(dev_priv)) { + err = set_power_wells(power_domains, bxt_power_wells); + } else if (IS_GEN9_BC(dev_priv)) { + err = set_power_wells(power_domains, skl_power_wells); } else if (IS_CHERRYVIEW(dev_priv)) { err = set_power_wells(power_domains, chv_power_wells); + } else if (IS_BROADWELL(dev_priv)) { + err = set_power_wells(power_domains, bdw_power_wells); + } else if (IS_HASWELL(dev_priv)) { + err = set_power_wells(power_domains, hsw_power_wells); } else if (IS_VALLEYVIEW(dev_priv)) { err = set_power_wells(power_domains, vlv_power_wells); } else if (IS_I830(dev_priv)) { @@ -3228,7 +3596,7 @@ static void icl_dbuf_disable(struct drm_i915_private *dev_priv) static void icl_mbus_init(struct drm_i915_private *dev_priv) { - uint32_t val; + u32 val; val = MBUS_ABOX_BT_CREDIT_POOL1(16) | MBUS_ABOX_BT_CREDIT_POOL2(16) | @@ -3238,18 +3606,40 @@ static void icl_mbus_init(struct drm_i915_private *dev_priv) I915_WRITE(MBUS_ABOX_CTL, val); } +static void intel_pch_reset_handshake(struct drm_i915_private *dev_priv, + bool enable) +{ + i915_reg_t reg; + u32 reset_bits, val; + + if (IS_IVYBRIDGE(dev_priv)) { + reg = GEN7_MSG_CTL; + reset_bits = WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK; + } else { + reg = HSW_NDE_RSTWRN_OPT; + reset_bits = RESET_PCH_HANDSHAKE_ENABLE; + } + + val = I915_READ(reg); + + if (enable) + val |= reset_bits; + else + val &= ~reset_bits; + + I915_WRITE(reg, val); +} + static void skl_display_core_init(struct drm_i915_private *dev_priv, bool resume) { struct i915_power_domains *power_domains = &dev_priv->power_domains; struct i915_power_well *well; - uint32_t val; gen9_set_dc_state(dev_priv, DC_STATE_DISABLE); /* enable PCH reset handshake */ - val = I915_READ(HSW_NDE_RSTWRN_OPT); - I915_WRITE(HSW_NDE_RSTWRN_OPT, val | RESET_PCH_HANDSHAKE_ENABLE); + intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv)); /* enable PG1 and Misc I/O */ mutex_lock(&power_domains->lock); @@ -3305,7 +3695,6 @@ void bxt_display_core_init(struct drm_i915_private *dev_priv, { struct i915_power_domains *power_domains = &dev_priv->power_domains; struct i915_power_well *well; - uint32_t val; gen9_set_dc_state(dev_priv, DC_STATE_DISABLE); @@ -3315,9 +3704,7 @@ void bxt_display_core_init(struct drm_i915_private *dev_priv, * Move the handshake programming to initialization sequence. * Previously was left up to BIOS. */ - val = I915_READ(HSW_NDE_RSTWRN_OPT); - val &= ~RESET_PCH_HANDSHAKE_ENABLE; - I915_WRITE(HSW_NDE_RSTWRN_OPT, val); + intel_pch_reset_handshake(dev_priv, false); /* Enable PG1 */ mutex_lock(&power_domains->lock); @@ -3363,101 +3750,18 @@ void bxt_display_core_uninit(struct drm_i915_private *dev_priv) usleep_range(10, 30); /* 10 us delay per Bspec */ } -enum { - PROCMON_0_85V_DOT_0, - PROCMON_0_95V_DOT_0, - PROCMON_0_95V_DOT_1, - PROCMON_1_05V_DOT_0, - PROCMON_1_05V_DOT_1, -}; - -static const struct cnl_procmon { - u32 dw1, dw9, dw10; -} cnl_procmon_values[] = { - [PROCMON_0_85V_DOT_0] = - { .dw1 = 0x00000000, .dw9 = 0x62AB67BB, .dw10 = 0x51914F96, }, - [PROCMON_0_95V_DOT_0] = - { .dw1 = 0x00000000, .dw9 = 0x86E172C7, .dw10 = 0x77CA5EAB, }, - [PROCMON_0_95V_DOT_1] = - { .dw1 = 0x00000000, .dw9 = 0x93F87FE1, .dw10 = 0x8AE871C5, }, - [PROCMON_1_05V_DOT_0] = - { .dw1 = 0x00000000, .dw9 = 0x98FA82DD, .dw10 = 0x89E46DC1, }, - [PROCMON_1_05V_DOT_1] = - { .dw1 = 0x00440000, .dw9 = 0x9A00AB25, .dw10 = 0x8AE38FF1, }, -}; - -/* - * CNL has just one set of registers, while ICL has two sets: one for port A and - * the other for port B. The CNL registers are equivalent to the ICL port A - * registers, that's why we call the ICL macros even though the function has CNL - * on its name. - */ -static void cnl_set_procmon_ref_values(struct drm_i915_private *dev_priv, - enum port port) -{ - const struct cnl_procmon *procmon; - u32 val; - - val = I915_READ(ICL_PORT_COMP_DW3(port)); - switch (val & (PROCESS_INFO_MASK | VOLTAGE_INFO_MASK)) { - default: - MISSING_CASE(val); - /* fall through */ - case VOLTAGE_INFO_0_85V | PROCESS_INFO_DOT_0: - procmon = &cnl_procmon_values[PROCMON_0_85V_DOT_0]; - break; - case VOLTAGE_INFO_0_95V | PROCESS_INFO_DOT_0: - procmon = &cnl_procmon_values[PROCMON_0_95V_DOT_0]; - break; - case VOLTAGE_INFO_0_95V | PROCESS_INFO_DOT_1: - procmon = &cnl_procmon_values[PROCMON_0_95V_DOT_1]; - break; - case VOLTAGE_INFO_1_05V | PROCESS_INFO_DOT_0: - procmon = &cnl_procmon_values[PROCMON_1_05V_DOT_0]; - break; - case VOLTAGE_INFO_1_05V | PROCESS_INFO_DOT_1: - procmon = &cnl_procmon_values[PROCMON_1_05V_DOT_1]; - break; - } - - val = I915_READ(ICL_PORT_COMP_DW1(port)); - val &= ~((0xff << 16) | 0xff); - val |= procmon->dw1; - I915_WRITE(ICL_PORT_COMP_DW1(port), val); - - I915_WRITE(ICL_PORT_COMP_DW9(port), procmon->dw9); - I915_WRITE(ICL_PORT_COMP_DW10(port), procmon->dw10); -} - static void cnl_display_core_init(struct drm_i915_private *dev_priv, bool resume) { struct i915_power_domains *power_domains = &dev_priv->power_domains; struct i915_power_well *well; - u32 val; gen9_set_dc_state(dev_priv, DC_STATE_DISABLE); /* 1. Enable PCH Reset Handshake */ - val = I915_READ(HSW_NDE_RSTWRN_OPT); - val |= RESET_PCH_HANDSHAKE_ENABLE; - I915_WRITE(HSW_NDE_RSTWRN_OPT, val); - - /* 2. Enable Comp */ - val = I915_READ(CHICKEN_MISC_2); - val &= ~CNL_COMP_PWR_DOWN; - I915_WRITE(CHICKEN_MISC_2, val); - - /* Dummy PORT_A to get the correct CNL register from the ICL macro */ - cnl_set_procmon_ref_values(dev_priv, PORT_A); + intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv)); - val = I915_READ(CNL_PORT_COMP_DW0); - val |= COMP_INIT; - I915_WRITE(CNL_PORT_COMP_DW0, val); - - /* 3. */ - val = I915_READ(CNL_PORT_CL1CM_DW5); - val |= CL_POWER_DOWN_ENABLE; - I915_WRITE(CNL_PORT_CL1CM_DW5, val); + /* 2-3. */ + cnl_combo_phys_init(dev_priv); /* * 4. Enable Power Well 1 (PG1). @@ -3482,7 +3786,6 @@ static void cnl_display_core_uninit(struct drm_i915_private *dev_priv) { struct i915_power_domains *power_domains = &dev_priv->power_domains; struct i915_power_well *well; - u32 val; gen9_set_dc_state(dev_priv, DC_STATE_DISABLE); @@ -3506,44 +3809,23 @@ static void cnl_display_core_uninit(struct drm_i915_private *dev_priv) usleep_range(10, 30); /* 10 us delay per Bspec */ - /* 5. Disable Comp */ - val = I915_READ(CHICKEN_MISC_2); - val |= CNL_COMP_PWR_DOWN; - I915_WRITE(CHICKEN_MISC_2, val); + /* 5. */ + cnl_combo_phys_uninit(dev_priv); } -static void icl_display_core_init(struct drm_i915_private *dev_priv, - bool resume) +void icl_display_core_init(struct drm_i915_private *dev_priv, + bool resume) { struct i915_power_domains *power_domains = &dev_priv->power_domains; struct i915_power_well *well; - enum port port; - u32 val; gen9_set_dc_state(dev_priv, DC_STATE_DISABLE); /* 1. Enable PCH reset handshake. */ - val = I915_READ(HSW_NDE_RSTWRN_OPT); - val |= RESET_PCH_HANDSHAKE_ENABLE; - I915_WRITE(HSW_NDE_RSTWRN_OPT, val); - - for (port = PORT_A; port <= PORT_B; port++) { - /* 2. Enable DDI combo PHY comp. */ - val = I915_READ(ICL_PHY_MISC(port)); - val &= ~ICL_PHY_MISC_DE_IO_COMP_PWR_DOWN; - I915_WRITE(ICL_PHY_MISC(port), val); - - cnl_set_procmon_ref_values(dev_priv, port); - - val = I915_READ(ICL_PORT_COMP_DW0(port)); - val |= COMP_INIT; - I915_WRITE(ICL_PORT_COMP_DW0(port), val); - - /* 3. Set power down enable. */ - val = I915_READ(ICL_PORT_CL_DW5(port)); - val |= CL_POWER_DOWN_ENABLE; - I915_WRITE(ICL_PORT_CL_DW5(port), val); - } + intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv)); + + /* 2-3. */ + icl_combo_phys_init(dev_priv); /* * 4. Enable Power Well 1 (PG1). @@ -3567,12 +3849,10 @@ static void icl_display_core_init(struct drm_i915_private *dev_priv, intel_csr_load_program(dev_priv); } -static void icl_display_core_uninit(struct drm_i915_private *dev_priv) +void icl_display_core_uninit(struct drm_i915_private *dev_priv) { struct i915_power_domains *power_domains = &dev_priv->power_domains; struct i915_power_well *well; - enum port port; - u32 val; gen9_set_dc_state(dev_priv, DC_STATE_DISABLE); @@ -3594,12 +3874,8 @@ static void icl_display_core_uninit(struct drm_i915_private *dev_priv) intel_power_well_disable(dev_priv, well); mutex_unlock(&power_domains->lock); - /* 5. Disable Comp */ - for (port = PORT_A; port <= PORT_B; port++) { - val = I915_READ(ICL_PHY_MISC(port)); - val |= ICL_PHY_MISC_DE_IO_COMP_PWR_DOWN; - I915_WRITE(ICL_PHY_MISC(port), val); - } + /* 5. */ + icl_combo_phys_uninit(dev_priv); } static void chv_phy_control_init(struct drm_i915_private *dev_priv) @@ -3631,7 +3907,7 @@ static void chv_phy_control_init(struct drm_i915_private *dev_priv) * current lane status. */ if (cmn_bc->desc->ops->is_enabled(dev_priv, cmn_bc)) { - uint32_t status = I915_READ(DPLL(PIPE_A)); + u32 status = I915_READ(DPLL(PIPE_A)); unsigned int mask; mask = status & DPLL_PORTB_READY_MASK; @@ -3662,7 +3938,7 @@ static void chv_phy_control_init(struct drm_i915_private *dev_priv) } if (cmn_d->desc->ops->is_enabled(dev_priv, cmn_d)) { - uint32_t status = I915_READ(DPIO_PHY_STATUS); + u32 status = I915_READ(DPIO_PHY_STATUS); unsigned int mask; mask = status & DPLL_PORTD_READY_MASK; @@ -3721,7 +3997,7 @@ static void intel_power_domains_verify_state(struct drm_i915_private *dev_priv); /** * intel_power_domains_init_hw - initialize hardware power domain state - * @dev_priv: i915 device instance + * @i915: i915 device instance * @resume: Called from resume code paths or not * * This function initializes the hardware power domain state and enables all @@ -3735,28 +4011,30 @@ static void intel_power_domains_verify_state(struct drm_i915_private *dev_priv); * intel_power_domains_enable()) and must be paired with * intel_power_domains_fini_hw(). */ -void intel_power_domains_init_hw(struct drm_i915_private *dev_priv, bool resume) +void intel_power_domains_init_hw(struct drm_i915_private *i915, bool resume) { - struct i915_power_domains *power_domains = &dev_priv->power_domains; + struct i915_power_domains *power_domains = &i915->power_domains; power_domains->initializing = true; - if (IS_ICELAKE(dev_priv)) { - icl_display_core_init(dev_priv, resume); - } else if (IS_CANNONLAKE(dev_priv)) { - cnl_display_core_init(dev_priv, resume); - } else if (IS_GEN9_BC(dev_priv)) { - skl_display_core_init(dev_priv, resume); - } else if (IS_GEN9_LP(dev_priv)) { - bxt_display_core_init(dev_priv, resume); - } else if (IS_CHERRYVIEW(dev_priv)) { + if (IS_ICELAKE(i915)) { + icl_display_core_init(i915, resume); + } else if (IS_CANNONLAKE(i915)) { + cnl_display_core_init(i915, resume); + } else if (IS_GEN9_BC(i915)) { + skl_display_core_init(i915, resume); + } else if (IS_GEN9_LP(i915)) { + bxt_display_core_init(i915, resume); + } else if (IS_CHERRYVIEW(i915)) { mutex_lock(&power_domains->lock); - chv_phy_control_init(dev_priv); + chv_phy_control_init(i915); mutex_unlock(&power_domains->lock); - } else if (IS_VALLEYVIEW(dev_priv)) { + } else if (IS_VALLEYVIEW(i915)) { mutex_lock(&power_domains->lock); - vlv_cmnlane_wa(dev_priv); + vlv_cmnlane_wa(i915); mutex_unlock(&power_domains->lock); + } else if (IS_IVYBRIDGE(i915) || INTEL_GEN(i915) >= 7) { + intel_pch_reset_handshake(i915, !HAS_PCH_NOP(i915)); } /* @@ -3765,18 +4043,20 @@ void intel_power_domains_init_hw(struct drm_i915_private *dev_priv, bool resume) * resources powered until display HW readout is complete. We drop * this reference in intel_power_domains_enable(). */ - intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); + power_domains->wakeref = + intel_display_power_get(i915, POWER_DOMAIN_INIT); + /* Disable power support if the user asked so. */ if (!i915_modparams.disable_power_well) - intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); - intel_power_domains_sync_hw(dev_priv); + intel_display_power_get(i915, POWER_DOMAIN_INIT); + intel_power_domains_sync_hw(i915); power_domains->initializing = false; } /** * intel_power_domains_fini_hw - deinitialize hw power domain state - * @dev_priv: i915 device instance + * @i915: i915 device instance * * De-initializes the display power domain HW state. It also ensures that the * device stays powered up so that the driver can be reloaded. @@ -3785,21 +4065,24 @@ void intel_power_domains_init_hw(struct drm_i915_private *dev_priv, bool resume) * intel_power_domains_disable()) and must be paired with * intel_power_domains_init_hw(). */ -void intel_power_domains_fini_hw(struct drm_i915_private *dev_priv) +void intel_power_domains_fini_hw(struct drm_i915_private *i915) { - /* Keep the power well enabled, but cancel its rpm wakeref. */ - intel_runtime_pm_put(dev_priv); + intel_wakeref_t wakeref __maybe_unused = + fetch_and_zero(&i915->power_domains.wakeref); /* Remove the refcount we took to keep power well support disabled. */ if (!i915_modparams.disable_power_well) - intel_display_power_put(dev_priv, POWER_DOMAIN_INIT); + intel_display_power_put_unchecked(i915, POWER_DOMAIN_INIT); - intel_power_domains_verify_state(dev_priv); + intel_power_domains_verify_state(i915); + + /* Keep the power well enabled, but cancel its rpm wakeref. */ + intel_runtime_pm_put(i915, wakeref); } /** * intel_power_domains_enable - enable toggling of display power wells - * @dev_priv: i915 device instance + * @i915: i915 device instance * * Enable the ondemand enabling/disabling of the display power wells. Note that * power wells not belonging to POWER_DOMAIN_INIT are allowed to be toggled @@ -3809,30 +4092,36 @@ void intel_power_domains_fini_hw(struct drm_i915_private *dev_priv) * of display HW readout (which will acquire the power references reflecting * the current HW state). */ -void intel_power_domains_enable(struct drm_i915_private *dev_priv) +void intel_power_domains_enable(struct drm_i915_private *i915) { - intel_display_power_put(dev_priv, POWER_DOMAIN_INIT); + intel_wakeref_t wakeref __maybe_unused = + fetch_and_zero(&i915->power_domains.wakeref); - intel_power_domains_verify_state(dev_priv); + intel_display_power_put(i915, POWER_DOMAIN_INIT, wakeref); + intel_power_domains_verify_state(i915); } /** * intel_power_domains_disable - disable toggling of display power wells - * @dev_priv: i915 device instance + * @i915: i915 device instance * * Disable the ondemand enabling/disabling of the display power wells. See * intel_power_domains_enable() for which power wells this call controls. */ -void intel_power_domains_disable(struct drm_i915_private *dev_priv) +void intel_power_domains_disable(struct drm_i915_private *i915) { - intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); + struct i915_power_domains *power_domains = &i915->power_domains; - intel_power_domains_verify_state(dev_priv); + WARN_ON(power_domains->wakeref); + power_domains->wakeref = + intel_display_power_get(i915, POWER_DOMAIN_INIT); + + intel_power_domains_verify_state(i915); } /** * intel_power_domains_suspend - suspend power domain state - * @dev_priv: i915 device instance + * @i915: i915 device instance * @suspend_mode: specifies the target suspend state (idle, mem, hibernation) * * This function prepares the hardware power domain state before entering @@ -3841,12 +4130,14 @@ void intel_power_domains_disable(struct drm_i915_private *dev_priv) * It must be called with power domains already disabled (after a call to * intel_power_domains_disable()) and paired with intel_power_domains_resume(). */ -void intel_power_domains_suspend(struct drm_i915_private *dev_priv, +void intel_power_domains_suspend(struct drm_i915_private *i915, enum i915_drm_suspend_mode suspend_mode) { - struct i915_power_domains *power_domains = &dev_priv->power_domains; + struct i915_power_domains *power_domains = &i915->power_domains; + intel_wakeref_t wakeref __maybe_unused = + fetch_and_zero(&power_domains->wakeref); - intel_display_power_put(dev_priv, POWER_DOMAIN_INIT); + intel_display_power_put(i915, POWER_DOMAIN_INIT, wakeref); /* * In case of suspend-to-idle (aka S0ix) on a DMC platform without DC9 @@ -3855,10 +4146,10 @@ void intel_power_domains_suspend(struct drm_i915_private *dev_priv, * resources as required and also enable deeper system power states * that would be blocked if the firmware was inactive. */ - if (!(dev_priv->csr.allowed_dc_mask & DC_STATE_EN_DC9) && + if (!(i915->csr.allowed_dc_mask & DC_STATE_EN_DC9) && suspend_mode == I915_DRM_SUSPEND_IDLE && - dev_priv->csr.dmc_payload != NULL) { - intel_power_domains_verify_state(dev_priv); + i915->csr.dmc_payload) { + intel_power_domains_verify_state(i915); return; } @@ -3867,25 +4158,25 @@ void intel_power_domains_suspend(struct drm_i915_private *dev_priv, * power wells if power domains must be deinitialized for suspend. */ if (!i915_modparams.disable_power_well) { - intel_display_power_put(dev_priv, POWER_DOMAIN_INIT); - intel_power_domains_verify_state(dev_priv); + intel_display_power_put_unchecked(i915, POWER_DOMAIN_INIT); + intel_power_domains_verify_state(i915); } - if (IS_ICELAKE(dev_priv)) - icl_display_core_uninit(dev_priv); - else if (IS_CANNONLAKE(dev_priv)) - cnl_display_core_uninit(dev_priv); - else if (IS_GEN9_BC(dev_priv)) - skl_display_core_uninit(dev_priv); - else if (IS_GEN9_LP(dev_priv)) - bxt_display_core_uninit(dev_priv); + if (IS_ICELAKE(i915)) + icl_display_core_uninit(i915); + else if (IS_CANNONLAKE(i915)) + cnl_display_core_uninit(i915); + else if (IS_GEN9_BC(i915)) + skl_display_core_uninit(i915); + else if (IS_GEN9_LP(i915)) + bxt_display_core_uninit(i915); power_domains->display_core_suspended = true; } /** * intel_power_domains_resume - resume power domain state - * @dev_priv: i915 device instance + * @i915: i915 device instance * * This function resume the hardware power domain state during system resume. * @@ -3893,28 +4184,30 @@ void intel_power_domains_suspend(struct drm_i915_private *dev_priv, * intel_power_domains_enable()) and must be paired with * intel_power_domains_suspend(). */ -void intel_power_domains_resume(struct drm_i915_private *dev_priv) +void intel_power_domains_resume(struct drm_i915_private *i915) { - struct i915_power_domains *power_domains = &dev_priv->power_domains; + struct i915_power_domains *power_domains = &i915->power_domains; if (power_domains->display_core_suspended) { - intel_power_domains_init_hw(dev_priv, true); + intel_power_domains_init_hw(i915, true); power_domains->display_core_suspended = false; } else { - intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); + WARN_ON(power_domains->wakeref); + power_domains->wakeref = + intel_display_power_get(i915, POWER_DOMAIN_INIT); } - intel_power_domains_verify_state(dev_priv); + intel_power_domains_verify_state(i915); } #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM) -static void intel_power_domains_dump_info(struct drm_i915_private *dev_priv) +static void intel_power_domains_dump_info(struct drm_i915_private *i915) { - struct i915_power_domains *power_domains = &dev_priv->power_domains; + struct i915_power_domains *power_domains = &i915->power_domains; struct i915_power_well *power_well; - for_each_power_well(dev_priv, power_well) { + for_each_power_well(i915, power_well) { enum intel_display_power_domain domain; DRM_DEBUG_DRIVER("%-25s %d\n", @@ -3929,7 +4222,7 @@ static void intel_power_domains_dump_info(struct drm_i915_private *dev_priv) /** * intel_power_domains_verify_state - verify the HW/SW state for all power wells - * @dev_priv: i915 device instance + * @i915: i915 device instance * * Verify if the reference count of each power well matches its HW enabled * state and the total refcount of the domains it belongs to. This must be @@ -3937,30 +4230,21 @@ static void intel_power_domains_dump_info(struct drm_i915_private *dev_priv) * acquiring reference counts for any power wells in use and disabling the * ones left on by BIOS but not required by any active output. */ -static void intel_power_domains_verify_state(struct drm_i915_private *dev_priv) +static void intel_power_domains_verify_state(struct drm_i915_private *i915) { - struct i915_power_domains *power_domains = &dev_priv->power_domains; + struct i915_power_domains *power_domains = &i915->power_domains; struct i915_power_well *power_well; bool dump_domain_info; mutex_lock(&power_domains->lock); dump_domain_info = false; - for_each_power_well(dev_priv, power_well) { + for_each_power_well(i915, power_well) { enum intel_display_power_domain domain; int domains_count; bool enabled; - /* - * Power wells not belonging to any domain (like the MISC_IO - * and PW1 power wells) are under FW control, so ignore them, - * since their state can change asynchronously. - */ - if (!power_well->desc->domains) - continue; - - enabled = power_well->desc->ops->is_enabled(dev_priv, - power_well); + enabled = power_well->desc->ops->is_enabled(i915, power_well); if ((power_well->count || power_well->desc->always_on) != enabled) DRM_ERROR("power well %s state mismatch (refcount %d/enabled %d)", @@ -3984,7 +4268,7 @@ static void intel_power_domains_verify_state(struct drm_i915_private *dev_priv) static bool dumped; if (!dumped) { - intel_power_domains_dump_info(dev_priv); + intel_power_domains_dump_info(i915); dumped = true; } } @@ -3994,7 +4278,7 @@ static void intel_power_domains_verify_state(struct drm_i915_private *dev_priv) #else -static void intel_power_domains_verify_state(struct drm_i915_private *dev_priv) +static void intel_power_domains_verify_state(struct drm_i915_private *i915) { } @@ -4002,30 +4286,31 @@ static void intel_power_domains_verify_state(struct drm_i915_private *dev_priv) /** * intel_runtime_pm_get - grab a runtime pm reference - * @dev_priv: i915 device instance + * @i915: i915 device instance * * This function grabs a device-level runtime pm reference (mostly used for GEM * code to ensure the GTT or GT is on) and ensures that it is powered up. * * Any runtime pm reference obtained by this function must have a symmetric * call to intel_runtime_pm_put() to release the reference again. + * + * Returns: the wakeref cookie to pass to intel_runtime_pm_put() */ -void intel_runtime_pm_get(struct drm_i915_private *dev_priv) +intel_wakeref_t intel_runtime_pm_get(struct drm_i915_private *i915) { - struct pci_dev *pdev = dev_priv->drm.pdev; + struct pci_dev *pdev = i915->drm.pdev; struct device *kdev = &pdev->dev; int ret; ret = pm_runtime_get_sync(kdev); WARN_ONCE(ret < 0, "pm_runtime_get_sync() failed: %d\n", ret); - atomic_inc(&dev_priv->runtime_pm.wakeref_count); - assert_rpm_wakelock_held(dev_priv); + return track_intel_runtime_pm_wakeref(i915); } /** * intel_runtime_pm_get_if_in_use - grab a runtime pm reference if device in use - * @dev_priv: i915 device instance + * @i915: i915 device instance * * This function grabs a device-level runtime pm reference if the device is * already in use and ensures that it is powered up. It is illegal to try @@ -4034,12 +4319,13 @@ void intel_runtime_pm_get(struct drm_i915_private *dev_priv) * Any runtime pm reference obtained by this function must have a symmetric * call to intel_runtime_pm_put() to release the reference again. * - * Returns: True if the wakeref was acquired, or False otherwise. + * Returns: the wakeref cookie to pass to intel_runtime_pm_put(), evaluates + * as True if the wakeref was acquired, or False otherwise. */ -bool intel_runtime_pm_get_if_in_use(struct drm_i915_private *dev_priv) +intel_wakeref_t intel_runtime_pm_get_if_in_use(struct drm_i915_private *i915) { if (IS_ENABLED(CONFIG_PM)) { - struct pci_dev *pdev = dev_priv->drm.pdev; + struct pci_dev *pdev = i915->drm.pdev; struct device *kdev = &pdev->dev; /* @@ -4049,18 +4335,15 @@ bool intel_runtime_pm_get_if_in_use(struct drm_i915_private *dev_priv) * atm to the late/early system suspend/resume handlers. */ if (pm_runtime_get_if_in_use(kdev) <= 0) - return false; + return 0; } - atomic_inc(&dev_priv->runtime_pm.wakeref_count); - assert_rpm_wakelock_held(dev_priv); - - return true; + return track_intel_runtime_pm_wakeref(i915); } /** * intel_runtime_pm_get_noresume - grab a runtime pm reference - * @dev_priv: i915 device instance + * @i915: i915 device instance * * This function grabs a device-level runtime pm reference (mostly used for GEM * code to ensure the GTT or GT is on). @@ -4074,41 +4357,50 @@ bool intel_runtime_pm_get_if_in_use(struct drm_i915_private *dev_priv) * * Any runtime pm reference obtained by this function must have a symmetric * call to intel_runtime_pm_put() to release the reference again. + * + * Returns: the wakeref cookie to pass to intel_runtime_pm_put() */ -void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv) +intel_wakeref_t intel_runtime_pm_get_noresume(struct drm_i915_private *i915) { - struct pci_dev *pdev = dev_priv->drm.pdev; + struct pci_dev *pdev = i915->drm.pdev; struct device *kdev = &pdev->dev; - assert_rpm_wakelock_held(dev_priv); + assert_rpm_wakelock_held(i915); pm_runtime_get_noresume(kdev); - atomic_inc(&dev_priv->runtime_pm.wakeref_count); + return track_intel_runtime_pm_wakeref(i915); } /** * intel_runtime_pm_put - release a runtime pm reference - * @dev_priv: i915 device instance + * @i915: i915 device instance * * This function drops the device-level runtime pm reference obtained by * intel_runtime_pm_get() and might power down the corresponding * hardware block right away if this is the last reference. */ -void intel_runtime_pm_put(struct drm_i915_private *dev_priv) +void intel_runtime_pm_put_unchecked(struct drm_i915_private *i915) { - struct pci_dev *pdev = dev_priv->drm.pdev; + struct pci_dev *pdev = i915->drm.pdev; struct device *kdev = &pdev->dev; - assert_rpm_wakelock_held(dev_priv); - atomic_dec(&dev_priv->runtime_pm.wakeref_count); + untrack_intel_runtime_pm_wakeref(i915); pm_runtime_mark_last_busy(kdev); pm_runtime_put_autosuspend(kdev); } +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM) +void intel_runtime_pm_put(struct drm_i915_private *i915, intel_wakeref_t wref) +{ + cancel_intel_runtime_pm_wakeref(i915, wref); + intel_runtime_pm_put_unchecked(i915); +} +#endif + /** * intel_runtime_pm_enable - enable runtime pm - * @dev_priv: i915 device instance + * @i915: i915 device instance * * This function enables runtime pm at the end of the driver load sequence. * @@ -4116,9 +4408,9 @@ void intel_runtime_pm_put(struct drm_i915_private *dev_priv) * subordinate display power domains. That is done by * intel_power_domains_enable(). */ -void intel_runtime_pm_enable(struct drm_i915_private *dev_priv) +void intel_runtime_pm_enable(struct drm_i915_private *i915) { - struct pci_dev *pdev = dev_priv->drm.pdev; + struct pci_dev *pdev = i915->drm.pdev; struct device *kdev = &pdev->dev; /* @@ -4140,7 +4432,7 @@ void intel_runtime_pm_enable(struct drm_i915_private *dev_priv) * so the driver's own RPM reference tracking asserts also work on * platforms without RPM support. */ - if (!HAS_RUNTIME_PM(dev_priv)) { + if (!HAS_RUNTIME_PM(i915)) { int ret; pm_runtime_dont_use_autosuspend(kdev); @@ -4158,17 +4450,35 @@ void intel_runtime_pm_enable(struct drm_i915_private *dev_priv) pm_runtime_put_autosuspend(kdev); } -void intel_runtime_pm_disable(struct drm_i915_private *dev_priv) +void intel_runtime_pm_disable(struct drm_i915_private *i915) { - struct pci_dev *pdev = dev_priv->drm.pdev; + struct pci_dev *pdev = i915->drm.pdev; struct device *kdev = &pdev->dev; /* Transfer rpm ownership back to core */ - WARN(pm_runtime_get_sync(&dev_priv->drm.pdev->dev) < 0, + WARN(pm_runtime_get_sync(kdev) < 0, "Failed to pass rpm ownership back to core\n"); pm_runtime_dont_use_autosuspend(kdev); - if (!HAS_RUNTIME_PM(dev_priv)) + if (!HAS_RUNTIME_PM(i915)) pm_runtime_put(kdev); } + +void intel_runtime_pm_cleanup(struct drm_i915_private *i915) +{ + struct i915_runtime_pm *rpm = &i915->runtime_pm; + int count; + + count = atomic_fetch_inc(&rpm->wakeref_count); /* balance untrack */ + WARN(count, + "i915->runtime_pm.wakeref_count=%d on cleanup\n", + count); + + untrack_intel_runtime_pm_wakeref(i915); +} + +void intel_runtime_pm_init_early(struct drm_i915_private *i915) +{ + init_intel_runtime_pm_wakeref(i915); +} diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c index 701372e512a8..e7b0884ba5a5 100644 --- a/drivers/gpu/drm/i915/intel_sdvo.c +++ b/drivers/gpu/drm/i915/intel_sdvo.c @@ -29,7 +29,6 @@ #include <linux/slab.h> #include <linux/delay.h> #include <linux/export.h> -#include <drm/drmP.h> #include <drm/drm_atomic_helper.h> #include <drm/drm_crtc.h> #include <drm/drm_edid.h> @@ -77,7 +76,7 @@ struct intel_sdvo { i915_reg_t sdvo_reg; /* Active outputs controlled by this SDVO output */ - uint16_t controlled_output; + u16 controlled_output; /* * Capabilities of the SDVO device returned by @@ -92,38 +91,32 @@ struct intel_sdvo { * For multiple function SDVO device, * this is for current attached outputs. */ - uint16_t attached_output; + u16 attached_output; /* * Hotplug activation bits for this device */ - uint16_t hotplug_active; + u16 hotplug_active; enum port port; bool has_hdmi_monitor; bool has_hdmi_audio; - bool rgb_quant_range_selectable; - - /** - * This is sdvo fixed pannel mode pointer - */ - struct drm_display_mode *sdvo_lvds_fixed_mode; /* DDC bus used by this SDVO encoder */ - uint8_t ddc_bus; + u8 ddc_bus; /* * the sdvo flag gets lost in round trip: dtd->adjusted_mode->dtd */ - uint8_t dtd_sdvo_flags; + u8 dtd_sdvo_flags; }; struct intel_sdvo_connector { struct intel_connector base; /* Mark the type of connector */ - uint16_t output_flag; + u16 output_flag; /* This contains all current supported TV format */ u8 tv_format_supported[TV_FORMAT_NUM]; @@ -191,7 +184,7 @@ to_intel_sdvo_connector(struct drm_connector *connector) container_of((conn_state), struct intel_sdvo_connector_state, base.base) static bool -intel_sdvo_output_setup(struct intel_sdvo *intel_sdvo, uint16_t flags); +intel_sdvo_output_setup(struct intel_sdvo *intel_sdvo, u16 flags); static bool intel_sdvo_tv_create_property(struct intel_sdvo *intel_sdvo, struct intel_sdvo_connector *intel_sdvo_connector, @@ -753,9 +746,9 @@ static bool intel_sdvo_get_input_timing(struct intel_sdvo *intel_sdvo, static bool intel_sdvo_create_preferred_input_timing(struct intel_sdvo *intel_sdvo, struct intel_sdvo_connector *intel_sdvo_connector, - uint16_t clock, - uint16_t width, - uint16_t height) + u16 clock, + u16 width, + u16 height) { struct intel_sdvo_preferred_input_timing_args args; @@ -765,10 +758,14 @@ intel_sdvo_create_preferred_input_timing(struct intel_sdvo *intel_sdvo, args.height = height; args.interlace = 0; - if (IS_LVDS(intel_sdvo_connector) && - (intel_sdvo->sdvo_lvds_fixed_mode->hdisplay != width || - intel_sdvo->sdvo_lvds_fixed_mode->vdisplay != height)) - args.scaled = 1; + if (IS_LVDS(intel_sdvo_connector)) { + const struct drm_display_mode *fixed_mode = + intel_sdvo_connector->base.panel.fixed_mode; + + if (fixed_mode->hdisplay != width || + fixed_mode->vdisplay != height) + args.scaled = 1; + } return intel_sdvo_set_value(intel_sdvo, SDVO_CMD_CREATE_PREFERRED_INPUT_TIMING, @@ -794,9 +791,9 @@ static bool intel_sdvo_set_clock_rate_mult(struct intel_sdvo *intel_sdvo, u8 val static void intel_sdvo_get_dtd_from_mode(struct intel_sdvo_dtd *dtd, const struct drm_display_mode *mode) { - uint16_t width, height; - uint16_t h_blank_len, h_sync_len, v_blank_len, v_sync_len; - uint16_t h_sync_offset, v_sync_offset; + u16 width, height; + u16 h_blank_len, h_sync_len, v_blank_len, v_sync_len; + u16 h_sync_offset, v_sync_offset; int mode_clock; memset(dtd, 0, sizeof(*dtd)); @@ -901,13 +898,13 @@ static bool intel_sdvo_check_supp_encode(struct intel_sdvo *intel_sdvo) } static bool intel_sdvo_set_encode(struct intel_sdvo *intel_sdvo, - uint8_t mode) + u8 mode) { return intel_sdvo_set_value(intel_sdvo, SDVO_CMD_SET_ENCODE, &mode, 1); } static bool intel_sdvo_set_colorimetry(struct intel_sdvo *intel_sdvo, - uint8_t mode) + u8 mode) { return intel_sdvo_set_value(intel_sdvo, SDVO_CMD_SET_COLORIMETRY, &mode, 1); } @@ -916,11 +913,11 @@ static bool intel_sdvo_set_colorimetry(struct intel_sdvo *intel_sdvo, static void intel_sdvo_dump_hdmi_buf(struct intel_sdvo *intel_sdvo) { int i, j; - uint8_t set_buf_index[2]; - uint8_t av_split; - uint8_t buf_size; - uint8_t buf[48]; - uint8_t *pos; + u8 set_buf_index[2]; + u8 av_split; + u8 buf_size; + u8 buf[48]; + u8 *pos; intel_sdvo_get_value(encoder, SDVO_CMD_GET_HBUF_AV_SPLIT, &av_split, 1); @@ -943,11 +940,11 @@ static void intel_sdvo_dump_hdmi_buf(struct intel_sdvo *intel_sdvo) #endif static bool intel_sdvo_write_infoframe(struct intel_sdvo *intel_sdvo, - unsigned if_index, uint8_t tx_rate, - const uint8_t *data, unsigned length) + unsigned int if_index, u8 tx_rate, + const u8 *data, unsigned int length) { - uint8_t set_buf_index[2] = { if_index, 0 }; - uint8_t hbuf_size, tmp[8]; + u8 set_buf_index[2] = { if_index, 0 }; + u8 hbuf_size, tmp[8]; int i; if (!intel_sdvo_set_value(intel_sdvo, @@ -982,29 +979,30 @@ static bool intel_sdvo_write_infoframe(struct intel_sdvo *intel_sdvo, } static bool intel_sdvo_set_avi_infoframe(struct intel_sdvo *intel_sdvo, - const struct intel_crtc_state *pipe_config) + const struct intel_crtc_state *pipe_config, + const struct drm_connector_state *conn_state) { - uint8_t sdvo_data[HDMI_INFOFRAME_SIZE(AVI)]; + const struct drm_display_mode *adjusted_mode = + &pipe_config->base.adjusted_mode; + u8 sdvo_data[HDMI_INFOFRAME_SIZE(AVI)]; union hdmi_infoframe frame; int ret; ssize_t len; ret = drm_hdmi_avi_infoframe_from_display_mode(&frame.avi, - &pipe_config->base.adjusted_mode, - false); + conn_state->connector, + adjusted_mode); if (ret < 0) { DRM_ERROR("couldn't fill AVI infoframe\n"); return false; } - if (intel_sdvo->rgb_quant_range_selectable) { - if (pipe_config->limited_color_range) - frame.avi.quantization_range = - HDMI_QUANTIZATION_RANGE_LIMITED; - else - frame.avi.quantization_range = - HDMI_QUANTIZATION_RANGE_FULL; - } + drm_hdmi_avi_infoframe_quant_range(&frame.avi, + conn_state->connector, + adjusted_mode, + pipe_config->limited_color_range ? + HDMI_QUANTIZATION_RANGE_LIMITED : + HDMI_QUANTIZATION_RANGE_FULL); len = hdmi_infoframe_pack(&frame, sdvo_data, sizeof(sdvo_data)); if (len < 0) @@ -1019,7 +1017,7 @@ static bool intel_sdvo_set_tv_format(struct intel_sdvo *intel_sdvo, const struct drm_connector_state *conn_state) { struct intel_sdvo_tv_format format; - uint32_t format_map; + u32 format_map; format_map = 1 << conn_state->tv.mode; memset(&format, 0, sizeof(format)); @@ -1109,9 +1107,9 @@ static void i9xx_adjust_sdvo_tv_clock(struct intel_crtc_state *pipe_config) pipe_config->clock_set = true; } -static bool intel_sdvo_compute_config(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config, - struct drm_connector_state *conn_state) +static int intel_sdvo_compute_config(struct intel_encoder *encoder, + struct intel_crtc_state *pipe_config, + struct drm_connector_state *conn_state) { struct intel_sdvo *intel_sdvo = to_sdvo(encoder); struct intel_sdvo_connector_state *intel_sdvo_state = @@ -1123,6 +1121,7 @@ static bool intel_sdvo_compute_config(struct intel_encoder *encoder, DRM_DEBUG_KMS("forcing bpc to 8 for SDVO\n"); pipe_config->pipe_bpp = 8*3; + pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; if (HAS_PCH_SPLIT(to_i915(encoder->base.dev))) pipe_config->has_pch_encoder = true; @@ -1135,7 +1134,7 @@ static bool intel_sdvo_compute_config(struct intel_encoder *encoder, */ if (IS_TV(intel_sdvo_connector)) { if (!intel_sdvo_set_output_timings_from_mode(intel_sdvo, mode)) - return false; + return -EINVAL; (void) intel_sdvo_get_preferred_input_mode(intel_sdvo, intel_sdvo_connector, @@ -1144,8 +1143,8 @@ static bool intel_sdvo_compute_config(struct intel_encoder *encoder, pipe_config->sdvo_tv_clock = true; } else if (IS_LVDS(intel_sdvo_connector)) { if (!intel_sdvo_set_output_timings_from_mode(intel_sdvo, - intel_sdvo->sdvo_lvds_fixed_mode)) - return false; + intel_sdvo_connector->base.panel.fixed_mode)) + return -EINVAL; (void) intel_sdvo_get_preferred_input_mode(intel_sdvo, intel_sdvo_connector, @@ -1154,7 +1153,7 @@ static bool intel_sdvo_compute_config(struct intel_encoder *encoder, } if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) - return false; + return -EINVAL; /* * Make the CRTC code factor in the SDVO pixel multiplier. The @@ -1194,7 +1193,7 @@ static bool intel_sdvo_compute_config(struct intel_encoder *encoder, if (intel_sdvo_connector->is_hdmi) adjusted_mode->picture_aspect_ratio = conn_state->picture_aspect_ratio; - return true; + return 0; } #define UPDATE_PROPERTY(input, NAME) \ @@ -1209,7 +1208,7 @@ static void intel_sdvo_update_props(struct intel_sdvo *intel_sdvo, const struct drm_connector_state *conn_state = &sdvo_state->base.base; struct intel_sdvo_connector *intel_sdvo_conn = to_intel_sdvo_connector(conn_state->connector); - uint16_t val; + u16 val; if (intel_sdvo_conn->left) UPDATE_PROPERTY(sdvo_state->tv.overscan_h, OVERSCAN_H); @@ -1301,7 +1300,7 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder, /* lvds has a special fixed output timing. */ if (IS_LVDS(intel_sdvo_connector)) intel_sdvo_get_dtd_from_mode(&output_dtd, - intel_sdvo->sdvo_lvds_fixed_mode); + intel_sdvo_connector->base.panel.fixed_mode); else intel_sdvo_get_dtd_from_mode(&output_dtd, mode); if (!intel_sdvo_set_output_timing(intel_sdvo, &output_dtd)) @@ -1316,7 +1315,8 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder, intel_sdvo_set_encode(intel_sdvo, SDVO_ENCODE_HDMI); intel_sdvo_set_colorimetry(intel_sdvo, SDVO_COLORIMETRY_RGB256); - intel_sdvo_set_avi_infoframe(intel_sdvo, crtc_state); + intel_sdvo_set_avi_infoframe(intel_sdvo, + crtc_state, conn_state); } else intel_sdvo_set_encode(intel_sdvo, SDVO_ENCODE_DVI); @@ -1642,10 +1642,13 @@ intel_sdvo_mode_valid(struct drm_connector *connector, return MODE_CLOCK_HIGH; if (IS_LVDS(intel_sdvo_connector)) { - if (mode->hdisplay > intel_sdvo->sdvo_lvds_fixed_mode->hdisplay) + const struct drm_display_mode *fixed_mode = + intel_sdvo_connector->base.panel.fixed_mode; + + if (mode->hdisplay > fixed_mode->hdisplay) return MODE_PANEL; - if (mode->vdisplay > intel_sdvo->sdvo_lvds_fixed_mode->vdisplay) + if (mode->vdisplay > fixed_mode->vdisplay) return MODE_PANEL; } @@ -1689,10 +1692,10 @@ static bool intel_sdvo_get_capabilities(struct intel_sdvo *intel_sdvo, struct in return true; } -static uint16_t intel_sdvo_get_hotplug_support(struct intel_sdvo *intel_sdvo) +static u16 intel_sdvo_get_hotplug_support(struct intel_sdvo *intel_sdvo) { struct drm_i915_private *dev_priv = to_i915(intel_sdvo->base.base.dev); - uint16_t hotplug; + u16 hotplug; if (!I915_HAS_HOTPLUG(dev_priv)) return 0; @@ -1799,8 +1802,6 @@ intel_sdvo_tmds_sink_detect(struct drm_connector *connector) if (intel_sdvo_connector->is_hdmi) { intel_sdvo->has_hdmi_monitor = drm_detect_hdmi_monitor(edid); intel_sdvo->has_hdmi_audio = drm_detect_monitor_audio(edid); - intel_sdvo->rgb_quant_range_selectable = - drm_rgb_quant_range_selectable(edid); } } else status = connector_status_disconnected; @@ -1825,7 +1826,7 @@ intel_sdvo_connector_matches_edid(struct intel_sdvo_connector *sdvo, static enum drm_connector_status intel_sdvo_detect(struct drm_connector *connector, bool force) { - uint16_t response; + u16 response; struct intel_sdvo *intel_sdvo = intel_attached_sdvo(connector); struct intel_sdvo_connector *intel_sdvo_connector = to_intel_sdvo_connector(connector); enum drm_connector_status ret; @@ -1849,7 +1850,6 @@ intel_sdvo_detect(struct drm_connector *connector, bool force) intel_sdvo->has_hdmi_monitor = false; intel_sdvo->has_hdmi_audio = false; - intel_sdvo->rgb_quant_range_selectable = false; if ((intel_sdvo_connector->output_flag & response) == 0) ret = connector_status_disconnected; @@ -1977,7 +1977,7 @@ static void intel_sdvo_get_tv_modes(struct drm_connector *connector) struct intel_sdvo *intel_sdvo = intel_attached_sdvo(connector); const struct drm_connector_state *conn_state = connector->state; struct intel_sdvo_sdtv_resolution_request tv_res; - uint32_t reply = 0, format_map = 0; + u32 reply = 0, format_map = 0; int i; DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", @@ -2058,19 +2058,11 @@ static int intel_sdvo_get_modes(struct drm_connector *connector) return !list_empty(&connector->probed_modes); } -static void intel_sdvo_destroy(struct drm_connector *connector) -{ - struct intel_sdvo_connector *intel_sdvo_connector = to_intel_sdvo_connector(connector); - - drm_connector_cleanup(connector); - kfree(intel_sdvo_connector); -} - static int intel_sdvo_connector_atomic_get_property(struct drm_connector *connector, const struct drm_connector_state *state, struct drm_property *property, - uint64_t *val) + u64 *val) { struct intel_sdvo_connector *intel_sdvo_connector = to_intel_sdvo_connector(connector); const struct intel_sdvo_connector_state *sdvo_state = to_intel_sdvo_connector_state((void *)state); @@ -2129,7 +2121,7 @@ static int intel_sdvo_connector_atomic_set_property(struct drm_connector *connector, struct drm_connector_state *state, struct drm_property *property, - uint64_t val) + u64 val) { struct intel_sdvo_connector *intel_sdvo_connector = to_intel_sdvo_connector(connector); struct intel_sdvo_connector_state *sdvo_state = to_intel_sdvo_connector_state(state); @@ -2228,7 +2220,7 @@ static const struct drm_connector_funcs intel_sdvo_connector_funcs = { .atomic_set_property = intel_sdvo_connector_atomic_set_property, .late_register = intel_sdvo_connector_register, .early_unregister = intel_sdvo_connector_unregister, - .destroy = intel_sdvo_destroy, + .destroy = intel_connector_destroy, .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, .atomic_duplicate_state = intel_sdvo_connector_duplicate_state, }; @@ -2267,10 +2259,6 @@ static void intel_sdvo_enc_destroy(struct drm_encoder *encoder) { struct intel_sdvo *intel_sdvo = to_sdvo(to_intel_encoder(encoder)); - if (intel_sdvo->sdvo_lvds_fixed_mode != NULL) - drm_mode_destroy(encoder->dev, - intel_sdvo->sdvo_lvds_fixed_mode); - i2c_del_adapter(&intel_sdvo->ddc); intel_encoder_destroy(encoder); } @@ -2282,7 +2270,7 @@ static const struct drm_encoder_funcs intel_sdvo_enc_funcs = { static void intel_sdvo_guess_ddc_bus(struct intel_sdvo *sdvo) { - uint16_t mask = 0; + u16 mask = 0; unsigned int num_bits; /* @@ -2583,7 +2571,7 @@ intel_sdvo_tv_init(struct intel_sdvo *intel_sdvo, int type) return true; err: - intel_sdvo_destroy(connector); + intel_connector_destroy(connector); return false; } @@ -2663,24 +2651,27 @@ intel_sdvo_lvds_init(struct intel_sdvo *intel_sdvo, int device) list_for_each_entry(mode, &connector->probed_modes, head) { if (mode->type & DRM_MODE_TYPE_PREFERRED) { - intel_sdvo->sdvo_lvds_fixed_mode = + struct drm_display_mode *fixed_mode = drm_mode_duplicate(connector->dev, mode); + + intel_panel_init(&intel_connector->panel, + fixed_mode, NULL); break; } } - if (!intel_sdvo->sdvo_lvds_fixed_mode) + if (!intel_connector->panel.fixed_mode) goto err; return true; err: - intel_sdvo_destroy(connector); + intel_connector_destroy(connector); return false; } static bool -intel_sdvo_output_setup(struct intel_sdvo *intel_sdvo, uint16_t flags) +intel_sdvo_output_setup(struct intel_sdvo *intel_sdvo, u16 flags) { /* SDVO requires XXX1 function may not exist unless it has XXX0 function.*/ @@ -2745,7 +2736,7 @@ static void intel_sdvo_output_cleanup(struct intel_sdvo *intel_sdvo) &dev->mode_config.connector_list, head) { if (intel_attached_encoder(connector) == &intel_sdvo->base) { drm_connector_unregister(connector); - intel_sdvo_destroy(connector); + intel_connector_destroy(connector); } } } @@ -2756,7 +2747,7 @@ static bool intel_sdvo_tv_create_property(struct intel_sdvo *intel_sdvo, { struct drm_device *dev = intel_sdvo->base.base.dev; struct intel_sdvo_tv_format format; - uint32_t format_map, i; + u32 format_map, i; if (!intel_sdvo_set_target_output(intel_sdvo, type)) return false; @@ -2823,7 +2814,7 @@ intel_sdvo_create_enhance_property_tv(struct intel_sdvo *intel_sdvo, struct drm_connector_state *conn_state = connector->state; struct intel_sdvo_connector_state *sdvo_state = to_intel_sdvo_connector_state(conn_state); - uint16_t response, data_value[2]; + u16 response, data_value[2]; /* when horizontal overscan is supported, Add the left/right property */ if (enhancements.overscan_h) { @@ -2934,7 +2925,7 @@ intel_sdvo_create_enhance_property_lvds(struct intel_sdvo *intel_sdvo, { struct drm_device *dev = intel_sdvo->base.base.dev; struct drm_connector *connector = &intel_sdvo_connector->base.base; - uint16_t response, data_value[2]; + u16 response, data_value[2]; ENHANCEMENT(&connector->state->tv, brightness, BRIGHTNESS); @@ -2948,7 +2939,7 @@ static bool intel_sdvo_create_enhance_property(struct intel_sdvo *intel_sdvo, { union { struct intel_sdvo_enhancements_reply reply; - uint16_t response; + u16 response; } enhancements; BUILD_BUG_ON(sizeof(enhancements) != 2); diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c index d3090a7537bb..b56a1a9ad01d 100644 --- a/drivers/gpu/drm/i915/intel_sprite.c +++ b/drivers/gpu/drm/i915/intel_sprite.c @@ -29,7 +29,6 @@ * registers; newer ones are much simpler and we can use the new DRM plane * support. */ -#include <drm/drmP.h> #include <drm/drm_atomic_helper.h> #include <drm/drm_crtc.h> #include <drm/drm_fourcc.h> @@ -40,6 +39,7 @@ #include "intel_frontbuffer.h" #include <drm/i915_drm.h> #include "i915_drv.h" +#include <drm/drm_color_mgmt.h> int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode, int usecs) @@ -275,17 +275,24 @@ int intel_plane_check_src_coordinates(struct intel_plane_state *plane_state) src->y2 = (src_y + src_h) << 16; if (fb->format->is_yuv && - fb->format->format != DRM_FORMAT_NV12 && (src_x & 1 || src_w & 1)) { DRM_DEBUG_KMS("src x/w (%u, %u) must be a multiple of 2 for YUV planes\n", src_x, src_w); return -EINVAL; } + if (fb->format->is_yuv && + fb->format->num_planes > 1 && + (src_y & 1 || src_h & 1)) { + DRM_DEBUG_KMS("src y/h (%u, %u) must be a multiple of 2 for planar YUV planes\n", + src_y, src_h); + return -EINVAL; + } + return 0; } -unsigned int +static unsigned int skl_plane_max_stride(struct intel_plane *plane, u32 pixel_format, u64 modifier, unsigned int rotation) @@ -314,8 +321,8 @@ skl_program_scaler(struct intel_plane *plane, &crtc_state->scaler_state.scalers[scaler_id]; int crtc_x = plane_state->base.dst.x1; int crtc_y = plane_state->base.dst.y1; - uint32_t crtc_w = drm_rect_width(&plane_state->base.dst); - uint32_t crtc_h = drm_rect_height(&plane_state->base.dst); + u32 crtc_w = drm_rect_width(&plane_state->base.dst); + u32 crtc_h = drm_rect_height(&plane_state->base.dst); u16 y_hphase, uv_rgb_hphase; u16 y_vphase, uv_rgb_vphase; int hscale, vscale; @@ -328,7 +335,8 @@ skl_program_scaler(struct intel_plane *plane, 0, INT_MAX); /* TODO: handle sub-pixel coordinates */ - if (plane_state->base.fb->format->format == DRM_FORMAT_NV12) { + if (plane_state->base.fb->format->format == DRM_FORMAT_NV12 && + !icl_is_hdr_plane(plane)) { y_hphase = skl_scaler_calc_phase(1, hscale, false); y_vphase = skl_scaler_calc_phase(1, vscale, false); @@ -346,7 +354,6 @@ skl_program_scaler(struct intel_plane *plane, I915_WRITE_FW(SKL_PS_CTRL(pipe, scaler_id), PS_SCALER_EN | PS_PLANE_SEL(plane->id) | scaler->mode); - I915_WRITE_FW(SKL_PS_PWR_GATE(pipe, scaler_id), 0); I915_WRITE_FW(SKL_PS_VPHASE(pipe, scaler_id), PS_Y_PHASE(y_vphase) | PS_UV_RGB_PHASE(uv_rgb_vphase)); I915_WRITE_FW(SKL_PS_HPHASE(pipe, scaler_id), @@ -355,70 +362,245 @@ skl_program_scaler(struct intel_plane *plane, I915_WRITE_FW(SKL_PS_WIN_SZ(pipe, scaler_id), (crtc_w << 16) | crtc_h); } -void -skl_update_plane(struct intel_plane *plane, - const struct intel_crtc_state *crtc_state, - const struct intel_plane_state *plane_state) +/* Preoffset values for YUV to RGB Conversion */ +#define PREOFF_YUV_TO_RGB_HI 0x1800 +#define PREOFF_YUV_TO_RGB_ME 0x1F00 +#define PREOFF_YUV_TO_RGB_LO 0x1800 + +#define ROFF(x) (((x) & 0xffff) << 16) +#define GOFF(x) (((x) & 0xffff) << 0) +#define BOFF(x) (((x) & 0xffff) << 16) + +static void +icl_program_input_csc(struct intel_plane *plane, + const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state) +{ + struct drm_i915_private *dev_priv = to_i915(plane->base.dev); + enum pipe pipe = plane->pipe; + enum plane_id plane_id = plane->id; + + static const u16 input_csc_matrix[][9] = { + /* + * BT.601 full range YCbCr -> full range RGB + * The matrix required is : + * [1.000, 0.000, 1.371, + * 1.000, -0.336, -0.698, + * 1.000, 1.732, 0.0000] + */ + [DRM_COLOR_YCBCR_BT601] = { + 0x7AF8, 0x7800, 0x0, + 0x8B28, 0x7800, 0x9AC0, + 0x0, 0x7800, 0x7DD8, + }, + /* + * BT.709 full range YCbCr -> full range RGB + * The matrix required is : + * [1.000, 0.000, 1.574, + * 1.000, -0.187, -0.468, + * 1.000, 1.855, 0.0000] + */ + [DRM_COLOR_YCBCR_BT709] = { + 0x7C98, 0x7800, 0x0, + 0x9EF8, 0x7800, 0xABF8, + 0x0, 0x7800, 0x7ED8, + }, + }; + + /* Matrix for Limited Range to Full Range Conversion */ + static const u16 input_csc_matrix_lr[][9] = { + /* + * BT.601 Limted range YCbCr -> full range RGB + * The matrix required is : + * [1.164384, 0.000, 1.596370, + * 1.138393, -0.382500, -0.794598, + * 1.138393, 1.971696, 0.0000] + */ + [DRM_COLOR_YCBCR_BT601] = { + 0x7CC8, 0x7950, 0x0, + 0x8CB8, 0x7918, 0x9C40, + 0x0, 0x7918, 0x7FC8, + }, + /* + * BT.709 Limited range YCbCr -> full range RGB + * The matrix required is : + * [1.164, 0.000, 1.833671, + * 1.138393, -0.213249, -0.532909, + * 1.138393, 2.112402, 0.0000] + */ + [DRM_COLOR_YCBCR_BT709] = { + 0x7EA8, 0x7950, 0x0, + 0x8888, 0x7918, 0xADA8, + 0x0, 0x7918, 0x6870, + }, + }; + const u16 *csc; + + if (plane_state->base.color_range == DRM_COLOR_YCBCR_FULL_RANGE) + csc = input_csc_matrix[plane_state->base.color_encoding]; + else + csc = input_csc_matrix_lr[plane_state->base.color_encoding]; + + I915_WRITE_FW(PLANE_INPUT_CSC_COEFF(pipe, plane_id, 0), ROFF(csc[0]) | + GOFF(csc[1])); + I915_WRITE_FW(PLANE_INPUT_CSC_COEFF(pipe, plane_id, 1), BOFF(csc[2])); + I915_WRITE_FW(PLANE_INPUT_CSC_COEFF(pipe, plane_id, 2), ROFF(csc[3]) | + GOFF(csc[4])); + I915_WRITE_FW(PLANE_INPUT_CSC_COEFF(pipe, plane_id, 3), BOFF(csc[5])); + I915_WRITE_FW(PLANE_INPUT_CSC_COEFF(pipe, plane_id, 4), ROFF(csc[6]) | + GOFF(csc[7])); + I915_WRITE_FW(PLANE_INPUT_CSC_COEFF(pipe, plane_id, 5), BOFF(csc[8])); + + I915_WRITE_FW(PLANE_INPUT_CSC_PREOFF(pipe, plane_id, 0), + PREOFF_YUV_TO_RGB_HI); + I915_WRITE_FW(PLANE_INPUT_CSC_PREOFF(pipe, plane_id, 1), + PREOFF_YUV_TO_RGB_ME); + I915_WRITE_FW(PLANE_INPUT_CSC_PREOFF(pipe, plane_id, 2), + PREOFF_YUV_TO_RGB_LO); + I915_WRITE_FW(PLANE_INPUT_CSC_POSTOFF(pipe, plane_id, 0), 0x0); + I915_WRITE_FW(PLANE_INPUT_CSC_POSTOFF(pipe, plane_id, 1), 0x0); + I915_WRITE_FW(PLANE_INPUT_CSC_POSTOFF(pipe, plane_id, 2), 0x0); +} + +static void +skl_program_plane(struct intel_plane *plane, + const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state, + int color_plane, bool slave, u32 plane_ctl) { struct drm_i915_private *dev_priv = to_i915(plane->base.dev); enum plane_id plane_id = plane->id; enum pipe pipe = plane->pipe; - u32 plane_ctl = plane_state->ctl; const struct drm_intel_sprite_colorkey *key = &plane_state->ckey; - u32 surf_addr = plane_state->color_plane[0].offset; - u32 stride = skl_plane_stride(plane_state, 0); + u32 surf_addr = plane_state->color_plane[color_plane].offset; + u32 stride = skl_plane_stride(plane_state, color_plane); u32 aux_stride = skl_plane_stride(plane_state, 1); int crtc_x = plane_state->base.dst.x1; int crtc_y = plane_state->base.dst.y1; - uint32_t x = plane_state->color_plane[0].x; - uint32_t y = plane_state->color_plane[0].y; - uint32_t src_w = drm_rect_width(&plane_state->base.src) >> 16; - uint32_t src_h = drm_rect_height(&plane_state->base.src) >> 16; + u32 x = plane_state->color_plane[color_plane].x; + u32 y = plane_state->color_plane[color_plane].y; + u32 src_w = drm_rect_width(&plane_state->base.src) >> 16; + u32 src_h = drm_rect_height(&plane_state->base.src) >> 16; + struct intel_plane *linked = plane_state->linked_plane; + const struct drm_framebuffer *fb = plane_state->base.fb; + u8 alpha = plane_state->base.alpha >> 8; + u32 plane_color_ctl = 0; unsigned long irqflags; + u32 keymsk, keymax; + + plane_ctl |= skl_plane_ctl_crtc(crtc_state); + + if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) + plane_color_ctl = plane_state->color_ctl | + glk_plane_color_ctl_crtc(crtc_state); /* Sizes are 0 based */ src_w--; src_h--; - spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); + keymax = (key->max_value & 0xffffff) | PLANE_KEYMAX_ALPHA(alpha); - if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) - I915_WRITE_FW(PLANE_COLOR_CTL(pipe, plane_id), - plane_state->color_ctl); + keymsk = key->channel_mask & 0x7ffffff; + if (alpha < 0xff) + keymsk |= PLANE_KEYMSK_ALPHA_ENABLE; - if (key->flags) { - I915_WRITE_FW(PLANE_KEYVAL(pipe, plane_id), key->min_value); - I915_WRITE_FW(PLANE_KEYMAX(pipe, plane_id), key->max_value); - I915_WRITE_FW(PLANE_KEYMSK(pipe, plane_id), key->channel_mask); + /* The scaler will handle the output position */ + if (plane_state->scaler_id >= 0) { + crtc_x = 0; + crtc_y = 0; } - I915_WRITE_FW(PLANE_OFFSET(pipe, plane_id), (y << 16) | x); + spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); + I915_WRITE_FW(PLANE_STRIDE(pipe, plane_id), stride); + I915_WRITE_FW(PLANE_POS(pipe, plane_id), (crtc_y << 16) | crtc_x); I915_WRITE_FW(PLANE_SIZE(pipe, plane_id), (src_h << 16) | src_w); I915_WRITE_FW(PLANE_AUX_DIST(pipe, plane_id), (plane_state->color_plane[1].offset - surf_addr) | aux_stride); - I915_WRITE_FW(PLANE_AUX_OFFSET(pipe, plane_id), - (plane_state->color_plane[1].y << 16) | - plane_state->color_plane[1].x); - if (plane_state->scaler_id >= 0) { - skl_program_scaler(plane, crtc_state, plane_state); + if (icl_is_hdr_plane(plane)) { + u32 cus_ctl = 0; + + if (linked) { + /* Enable and use MPEG-2 chroma siting */ + cus_ctl = PLANE_CUS_ENABLE | + PLANE_CUS_HPHASE_0 | + PLANE_CUS_VPHASE_SIGN_NEGATIVE | + PLANE_CUS_VPHASE_0_25; + + if (linked->id == PLANE_SPRITE5) + cus_ctl |= PLANE_CUS_PLANE_7; + else if (linked->id == PLANE_SPRITE4) + cus_ctl |= PLANE_CUS_PLANE_6; + else + MISSING_CASE(linked->id); + } - I915_WRITE_FW(PLANE_POS(pipe, plane_id), 0); - } else { - I915_WRITE_FW(PLANE_POS(pipe, plane_id), (crtc_y << 16) | crtc_x); + I915_WRITE_FW(PLANE_CUS_CTL(pipe, plane_id), cus_ctl); } + if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) + I915_WRITE_FW(PLANE_COLOR_CTL(pipe, plane_id), plane_color_ctl); + + if (fb->format->is_yuv && icl_is_hdr_plane(plane)) + icl_program_input_csc(plane, crtc_state, plane_state); + + skl_write_plane_wm(plane, crtc_state); + + I915_WRITE_FW(PLANE_KEYVAL(pipe, plane_id), key->min_value); + I915_WRITE_FW(PLANE_KEYMSK(pipe, plane_id), keymsk); + I915_WRITE_FW(PLANE_KEYMAX(pipe, plane_id), keymax); + + I915_WRITE_FW(PLANE_OFFSET(pipe, plane_id), (y << 16) | x); + + if (INTEL_GEN(dev_priv) < 11) + I915_WRITE_FW(PLANE_AUX_OFFSET(pipe, plane_id), + (plane_state->color_plane[1].y << 16) | + plane_state->color_plane[1].x); + + /* + * The control register self-arms if the plane was previously + * disabled. Try to make the plane enable atomic by writing + * the control register just before the surface register. + */ I915_WRITE_FW(PLANE_CTL(pipe, plane_id), plane_ctl); I915_WRITE_FW(PLANE_SURF(pipe, plane_id), intel_plane_ggtt_offset(plane_state) + surf_addr); - POSTING_READ_FW(PLANE_SURF(pipe, plane_id)); + + if (!slave && plane_state->scaler_id >= 0) + skl_program_scaler(plane, crtc_state, plane_state); spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); } -void -skl_disable_plane(struct intel_plane *plane, struct intel_crtc *crtc) +static void +skl_update_plane(struct intel_plane *plane, + const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state) +{ + int color_plane = 0; + + if (plane_state->linked_plane) { + /* Program the UV plane */ + color_plane = 1; + } + + skl_program_plane(plane, crtc_state, plane_state, + color_plane, false, plane_state->ctl); +} + +static void +icl_update_slave(struct intel_plane *plane, + const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state) +{ + skl_program_plane(plane, crtc_state, plane_state, 0, true, + plane_state->ctl | PLANE_CTL_YUV420_Y_PLANE); +} + +static void +skl_disable_plane(struct intel_plane *plane, + const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(plane->base.dev); enum plane_id plane_id = plane->id; @@ -427,32 +609,34 @@ skl_disable_plane(struct intel_plane *plane, struct intel_crtc *crtc) spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); - I915_WRITE_FW(PLANE_CTL(pipe, plane_id), 0); + skl_write_plane_wm(plane, crtc_state); + I915_WRITE_FW(PLANE_CTL(pipe, plane_id), 0); I915_WRITE_FW(PLANE_SURF(pipe, plane_id), 0); - POSTING_READ_FW(PLANE_SURF(pipe, plane_id)); spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); } -bool +static bool skl_plane_get_hw_state(struct intel_plane *plane, enum pipe *pipe) { struct drm_i915_private *dev_priv = to_i915(plane->base.dev); enum intel_display_power_domain power_domain; enum plane_id plane_id = plane->id; + intel_wakeref_t wakeref; bool ret; power_domain = POWER_DOMAIN_PIPE(plane->pipe); - if (!intel_display_power_get_if_enabled(dev_priv, power_domain)) + wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); + if (!wakeref) return false; ret = I915_READ(PLANE_CTL(plane->pipe, plane_id)) & PLANE_CTL_ENABLE; *pipe = plane->pipe; - intel_display_power_put(dev_priv, power_domain); + intel_display_power_put(dev_priv, power_domain, wakeref); return ret; } @@ -555,6 +739,11 @@ vlv_update_clrc(const struct intel_plane_state *plane_state) SP_SH_SIN(sh_sin) | SP_SH_COS(sh_cos)); } +static u32 vlv_sprite_ctl_crtc(const struct intel_crtc_state *crtc_state) +{ + return SP_GAMMA_ENABLE; +} + static u32 vlv_sprite_ctl(const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state) { @@ -563,7 +752,7 @@ static u32 vlv_sprite_ctl(const struct intel_crtc_state *crtc_state, const struct drm_intel_sprite_colorkey *key = &plane_state->ckey; u32 sprctl; - sprctl = SP_ENABLE | SP_GAMMA_ENABLE; + sprctl = SP_ENABLE; switch (fb->format->format) { case DRM_FORMAT_YUYV: @@ -628,20 +817,21 @@ vlv_update_plane(struct intel_plane *plane, const struct intel_plane_state *plane_state) { struct drm_i915_private *dev_priv = to_i915(plane->base.dev); - const struct drm_framebuffer *fb = plane_state->base.fb; enum pipe pipe = plane->pipe; enum plane_id plane_id = plane->id; - u32 sprctl = plane_state->ctl; u32 sprsurf_offset = plane_state->color_plane[0].offset; u32 linear_offset; const struct drm_intel_sprite_colorkey *key = &plane_state->ckey; int crtc_x = plane_state->base.dst.x1; int crtc_y = plane_state->base.dst.y1; - uint32_t crtc_w = drm_rect_width(&plane_state->base.dst); - uint32_t crtc_h = drm_rect_height(&plane_state->base.dst); - uint32_t x = plane_state->color_plane[0].x; - uint32_t y = plane_state->color_plane[0].y; + u32 crtc_w = drm_rect_width(&plane_state->base.dst); + u32 crtc_h = drm_rect_height(&plane_state->base.dst); + u32 x = plane_state->color_plane[0].x; + u32 y = plane_state->color_plane[0].y; unsigned long irqflags; + u32 sprctl; + + sprctl = plane_state->ctl | vlv_sprite_ctl_crtc(crtc_state); /* Sizes are 0 based */ crtc_w--; @@ -651,38 +841,41 @@ vlv_update_plane(struct intel_plane *plane, spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); - vlv_update_clrc(plane_state); + I915_WRITE_FW(SPSTRIDE(pipe, plane_id), + plane_state->color_plane[0].stride); + I915_WRITE_FW(SPPOS(pipe, plane_id), (crtc_y << 16) | crtc_x); + I915_WRITE_FW(SPSIZE(pipe, plane_id), (crtc_h << 16) | crtc_w); + I915_WRITE_FW(SPCONSTALPHA(pipe, plane_id), 0); if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) chv_update_csc(plane_state); if (key->flags) { I915_WRITE_FW(SPKEYMINVAL(pipe, plane_id), key->min_value); - I915_WRITE_FW(SPKEYMAXVAL(pipe, plane_id), key->max_value); I915_WRITE_FW(SPKEYMSK(pipe, plane_id), key->channel_mask); + I915_WRITE_FW(SPKEYMAXVAL(pipe, plane_id), key->max_value); } - I915_WRITE_FW(SPSTRIDE(pipe, plane_id), - plane_state->color_plane[0].stride); - I915_WRITE_FW(SPPOS(pipe, plane_id), (crtc_y << 16) | crtc_x); - if (fb->modifier == I915_FORMAT_MOD_X_TILED) - I915_WRITE_FW(SPTILEOFF(pipe, plane_id), (y << 16) | x); - else - I915_WRITE_FW(SPLINOFF(pipe, plane_id), linear_offset); - - I915_WRITE_FW(SPCONSTALPHA(pipe, plane_id), 0); + I915_WRITE_FW(SPLINOFF(pipe, plane_id), linear_offset); + I915_WRITE_FW(SPTILEOFF(pipe, plane_id), (y << 16) | x); - I915_WRITE_FW(SPSIZE(pipe, plane_id), (crtc_h << 16) | crtc_w); + /* + * The control register self-arms if the plane was previously + * disabled. Try to make the plane enable atomic by writing + * the control register just before the surface register. + */ I915_WRITE_FW(SPCNTR(pipe, plane_id), sprctl); I915_WRITE_FW(SPSURF(pipe, plane_id), intel_plane_ggtt_offset(plane_state) + sprsurf_offset); - POSTING_READ_FW(SPSURF(pipe, plane_id)); + + vlv_update_clrc(plane_state); spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); } static void -vlv_disable_plane(struct intel_plane *plane, struct intel_crtc *crtc) +vlv_disable_plane(struct intel_plane *plane, + const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(plane->base.dev); enum pipe pipe = plane->pipe; @@ -692,9 +885,7 @@ vlv_disable_plane(struct intel_plane *plane, struct intel_crtc *crtc) spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); I915_WRITE_FW(SPCNTR(pipe, plane_id), 0); - I915_WRITE_FW(SPSURF(pipe, plane_id), 0); - POSTING_READ_FW(SPSURF(pipe, plane_id)); spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); } @@ -706,21 +897,36 @@ vlv_plane_get_hw_state(struct intel_plane *plane, struct drm_i915_private *dev_priv = to_i915(plane->base.dev); enum intel_display_power_domain power_domain; enum plane_id plane_id = plane->id; + intel_wakeref_t wakeref; bool ret; power_domain = POWER_DOMAIN_PIPE(plane->pipe); - if (!intel_display_power_get_if_enabled(dev_priv, power_domain)) + wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); + if (!wakeref) return false; ret = I915_READ(SPCNTR(plane->pipe, plane_id)) & SP_ENABLE; *pipe = plane->pipe; - intel_display_power_put(dev_priv, power_domain); + intel_display_power_put(dev_priv, power_domain, wakeref); return ret; } +static u32 ivb_sprite_ctl_crtc(const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); + u32 sprctl = 0; + + sprctl |= SPRITE_GAMMA_ENABLE; + + if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) + sprctl |= SPRITE_PIPE_CSC_ENABLE; + + return sprctl; +} + static u32 ivb_sprite_ctl(const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state) { @@ -731,14 +937,11 @@ static u32 ivb_sprite_ctl(const struct intel_crtc_state *crtc_state, const struct drm_intel_sprite_colorkey *key = &plane_state->ckey; u32 sprctl; - sprctl = SPRITE_ENABLE | SPRITE_GAMMA_ENABLE; + sprctl = SPRITE_ENABLE; if (IS_IVYBRIDGE(dev_priv)) sprctl |= SPRITE_TRICKLE_FEED_DISABLE; - if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) - sprctl |= SPRITE_PIPE_CSC_ENABLE; - switch (fb->format->format) { case DRM_FORMAT_XBGR8888: sprctl |= SPRITE_FORMAT_RGBX888 | SPRITE_RGB_ORDER_RGBX; @@ -789,22 +992,23 @@ ivb_update_plane(struct intel_plane *plane, const struct intel_plane_state *plane_state) { struct drm_i915_private *dev_priv = to_i915(plane->base.dev); - const struct drm_framebuffer *fb = plane_state->base.fb; enum pipe pipe = plane->pipe; - u32 sprctl = plane_state->ctl, sprscale = 0; u32 sprsurf_offset = plane_state->color_plane[0].offset; u32 linear_offset; const struct drm_intel_sprite_colorkey *key = &plane_state->ckey; int crtc_x = plane_state->base.dst.x1; int crtc_y = plane_state->base.dst.y1; - uint32_t crtc_w = drm_rect_width(&plane_state->base.dst); - uint32_t crtc_h = drm_rect_height(&plane_state->base.dst); - uint32_t x = plane_state->color_plane[0].x; - uint32_t y = plane_state->color_plane[0].y; - uint32_t src_w = drm_rect_width(&plane_state->base.src) >> 16; - uint32_t src_h = drm_rect_height(&plane_state->base.src) >> 16; + u32 crtc_w = drm_rect_width(&plane_state->base.dst); + u32 crtc_h = drm_rect_height(&plane_state->base.dst); + u32 x = plane_state->color_plane[0].x; + u32 y = plane_state->color_plane[0].y; + u32 src_w = drm_rect_width(&plane_state->base.src) >> 16; + u32 src_h = drm_rect_height(&plane_state->base.src) >> 16; + u32 sprctl, sprscale = 0; unsigned long irqflags; + sprctl = plane_state->ctl | ivb_sprite_ctl_crtc(crtc_state); + /* Sizes are 0 based */ src_w--; src_h--; @@ -818,37 +1022,42 @@ ivb_update_plane(struct intel_plane *plane, spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); + I915_WRITE_FW(SPRSTRIDE(pipe), plane_state->color_plane[0].stride); + I915_WRITE_FW(SPRPOS(pipe), (crtc_y << 16) | crtc_x); + I915_WRITE_FW(SPRSIZE(pipe), (crtc_h << 16) | crtc_w); + if (IS_IVYBRIDGE(dev_priv)) + I915_WRITE_FW(SPRSCALE(pipe), sprscale); + if (key->flags) { I915_WRITE_FW(SPRKEYVAL(pipe), key->min_value); - I915_WRITE_FW(SPRKEYMAX(pipe), key->max_value); I915_WRITE_FW(SPRKEYMSK(pipe), key->channel_mask); + I915_WRITE_FW(SPRKEYMAX(pipe), key->max_value); } - I915_WRITE_FW(SPRSTRIDE(pipe), plane_state->color_plane[0].stride); - I915_WRITE_FW(SPRPOS(pipe), (crtc_y << 16) | crtc_x); - /* HSW consolidates SPRTILEOFF and SPRLINOFF into a single SPROFFSET * register */ - if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) + if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { I915_WRITE_FW(SPROFFSET(pipe), (y << 16) | x); - else if (fb->modifier == I915_FORMAT_MOD_X_TILED) - I915_WRITE_FW(SPRTILEOFF(pipe), (y << 16) | x); - else + } else { I915_WRITE_FW(SPRLINOFF(pipe), linear_offset); + I915_WRITE_FW(SPRTILEOFF(pipe), (y << 16) | x); + } - I915_WRITE_FW(SPRSIZE(pipe), (crtc_h << 16) | crtc_w); - if (IS_IVYBRIDGE(dev_priv)) - I915_WRITE_FW(SPRSCALE(pipe), sprscale); + /* + * The control register self-arms if the plane was previously + * disabled. Try to make the plane enable atomic by writing + * the control register just before the surface register. + */ I915_WRITE_FW(SPRCTL(pipe), sprctl); I915_WRITE_FW(SPRSURF(pipe), intel_plane_ggtt_offset(plane_state) + sprsurf_offset); - POSTING_READ_FW(SPRSURF(pipe)); spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); } static void -ivb_disable_plane(struct intel_plane *plane, struct intel_crtc *crtc) +ivb_disable_plane(struct intel_plane *plane, + const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(plane->base.dev); enum pipe pipe = plane->pipe; @@ -857,12 +1066,10 @@ ivb_disable_plane(struct intel_plane *plane, struct intel_crtc *crtc) spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); I915_WRITE_FW(SPRCTL(pipe), 0); - /* Can't leave the scaler enabled... */ + /* Disable the scaler */ if (IS_IVYBRIDGE(dev_priv)) I915_WRITE_FW(SPRSCALE(pipe), 0); - I915_WRITE_FW(SPRSURF(pipe), 0); - POSTING_READ_FW(SPRSURF(pipe)); spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); } @@ -873,17 +1080,19 @@ ivb_plane_get_hw_state(struct intel_plane *plane, { struct drm_i915_private *dev_priv = to_i915(plane->base.dev); enum intel_display_power_domain power_domain; + intel_wakeref_t wakeref; bool ret; power_domain = POWER_DOMAIN_PIPE(plane->pipe); - if (!intel_display_power_get_if_enabled(dev_priv, power_domain)) + wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); + if (!wakeref) return false; ret = I915_READ(SPRCTL(plane->pipe)) & SPRITE_ENABLE; *pipe = plane->pipe; - intel_display_power_put(dev_priv, power_domain); + intel_display_power_put(dev_priv, power_domain, wakeref); return ret; } @@ -896,6 +1105,11 @@ g4x_sprite_max_stride(struct intel_plane *plane, return 16384; } +static u32 g4x_sprite_ctl_crtc(const struct intel_crtc_state *crtc_state) +{ + return DVS_GAMMA_ENABLE; +} + static u32 g4x_sprite_ctl(const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state) { @@ -906,9 +1120,9 @@ static u32 g4x_sprite_ctl(const struct intel_crtc_state *crtc_state, const struct drm_intel_sprite_colorkey *key = &plane_state->ckey; u32 dvscntr; - dvscntr = DVS_ENABLE | DVS_GAMMA_ENABLE; + dvscntr = DVS_ENABLE; - if (IS_GEN6(dev_priv)) + if (IS_GEN(dev_priv, 6)) dvscntr |= DVS_TRICKLE_FEED_DISABLE; switch (fb->format->format) { @@ -961,22 +1175,23 @@ g4x_update_plane(struct intel_plane *plane, const struct intel_plane_state *plane_state) { struct drm_i915_private *dev_priv = to_i915(plane->base.dev); - const struct drm_framebuffer *fb = plane_state->base.fb; enum pipe pipe = plane->pipe; - u32 dvscntr = plane_state->ctl, dvsscale = 0; u32 dvssurf_offset = plane_state->color_plane[0].offset; u32 linear_offset; const struct drm_intel_sprite_colorkey *key = &plane_state->ckey; int crtc_x = plane_state->base.dst.x1; int crtc_y = plane_state->base.dst.y1; - uint32_t crtc_w = drm_rect_width(&plane_state->base.dst); - uint32_t crtc_h = drm_rect_height(&plane_state->base.dst); - uint32_t x = plane_state->color_plane[0].x; - uint32_t y = plane_state->color_plane[0].y; - uint32_t src_w = drm_rect_width(&plane_state->base.src) >> 16; - uint32_t src_h = drm_rect_height(&plane_state->base.src) >> 16; + u32 crtc_w = drm_rect_width(&plane_state->base.dst); + u32 crtc_h = drm_rect_height(&plane_state->base.dst); + u32 x = plane_state->color_plane[0].x; + u32 y = plane_state->color_plane[0].y; + u32 src_w = drm_rect_width(&plane_state->base.src) >> 16; + u32 src_h = drm_rect_height(&plane_state->base.src) >> 16; + u32 dvscntr, dvsscale = 0; unsigned long irqflags; + dvscntr = plane_state->ctl | g4x_sprite_ctl_crtc(crtc_state); + /* Sizes are 0 based */ src_w--; src_h--; @@ -990,32 +1205,35 @@ g4x_update_plane(struct intel_plane *plane, spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); + I915_WRITE_FW(DVSSTRIDE(pipe), plane_state->color_plane[0].stride); + I915_WRITE_FW(DVSPOS(pipe), (crtc_y << 16) | crtc_x); + I915_WRITE_FW(DVSSIZE(pipe), (crtc_h << 16) | crtc_w); + I915_WRITE_FW(DVSSCALE(pipe), dvsscale); + if (key->flags) { I915_WRITE_FW(DVSKEYVAL(pipe), key->min_value); - I915_WRITE_FW(DVSKEYMAX(pipe), key->max_value); I915_WRITE_FW(DVSKEYMSK(pipe), key->channel_mask); + I915_WRITE_FW(DVSKEYMAX(pipe), key->max_value); } - I915_WRITE_FW(DVSSTRIDE(pipe), plane_state->color_plane[0].stride); - I915_WRITE_FW(DVSPOS(pipe), (crtc_y << 16) | crtc_x); - - if (fb->modifier == I915_FORMAT_MOD_X_TILED) - I915_WRITE_FW(DVSTILEOFF(pipe), (y << 16) | x); - else - I915_WRITE_FW(DVSLINOFF(pipe), linear_offset); + I915_WRITE_FW(DVSLINOFF(pipe), linear_offset); + I915_WRITE_FW(DVSTILEOFF(pipe), (y << 16) | x); - I915_WRITE_FW(DVSSIZE(pipe), (crtc_h << 16) | crtc_w); - I915_WRITE_FW(DVSSCALE(pipe), dvsscale); + /* + * The control register self-arms if the plane was previously + * disabled. Try to make the plane enable atomic by writing + * the control register just before the surface register. + */ I915_WRITE_FW(DVSCNTR(pipe), dvscntr); I915_WRITE_FW(DVSSURF(pipe), intel_plane_ggtt_offset(plane_state) + dvssurf_offset); - POSTING_READ_FW(DVSSURF(pipe)); spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); } static void -g4x_disable_plane(struct intel_plane *plane, struct intel_crtc *crtc) +g4x_disable_plane(struct intel_plane *plane, + const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(plane->base.dev); enum pipe pipe = plane->pipe; @@ -1026,9 +1244,7 @@ g4x_disable_plane(struct intel_plane *plane, struct intel_crtc *crtc) I915_WRITE_FW(DVSCNTR(pipe), 0); /* Disable the scaler */ I915_WRITE_FW(DVSSCALE(pipe), 0); - I915_WRITE_FW(DVSSURF(pipe), 0); - POSTING_READ_FW(DVSSURF(pipe)); spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); } @@ -1039,21 +1255,36 @@ g4x_plane_get_hw_state(struct intel_plane *plane, { struct drm_i915_private *dev_priv = to_i915(plane->base.dev); enum intel_display_power_domain power_domain; + intel_wakeref_t wakeref; bool ret; power_domain = POWER_DOMAIN_PIPE(plane->pipe); - if (!intel_display_power_get_if_enabled(dev_priv, power_domain)) + wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); + if (!wakeref) return false; ret = I915_READ(DVSCNTR(plane->pipe)) & DVS_ENABLE; *pipe = plane->pipe; - intel_display_power_put(dev_priv, power_domain); + intel_display_power_put(dev_priv, power_domain, wakeref); return ret; } +static bool intel_fb_scalable(const struct drm_framebuffer *fb) +{ + if (!fb) + return false; + + switch (fb->format->format) { + case DRM_FORMAT_C8: + return false; + default: + return true; + } +} + static int g4x_sprite_check_scaling(struct intel_crtc_state *crtc_state, struct intel_plane_state *plane_state) @@ -1121,18 +1352,18 @@ g4x_sprite_check(struct intel_crtc_state *crtc_state, { struct intel_plane *plane = to_intel_plane(plane_state->base.plane); struct drm_i915_private *dev_priv = to_i915(plane->base.dev); - int max_scale, min_scale; + int min_scale = DRM_PLANE_HELPER_NO_SCALING; + int max_scale = DRM_PLANE_HELPER_NO_SCALING; int ret; - if (INTEL_GEN(dev_priv) < 7) { - min_scale = 1; - max_scale = 16 << 16; - } else if (IS_IVYBRIDGE(dev_priv)) { - min_scale = 1; - max_scale = 2 << 16; - } else { - min_scale = DRM_PLANE_HELPER_NO_SCALING; - max_scale = DRM_PLANE_HELPER_NO_SCALING; + if (intel_fb_scalable(plane_state->base.fb)) { + if (INTEL_GEN(dev_priv) < 7) { + min_scale = 1; + max_scale = 16 << 16; + } else if (IS_IVYBRIDGE(dev_priv)) { + min_scale = 1; + max_scale = 2 << 16; + } } ret = drm_atomic_helper_check_plane_state(&plane_state->base, @@ -1219,6 +1450,8 @@ vlv_sprite_check(struct intel_crtc_state *crtc_state, static int skl_plane_check_fb(const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state) { + struct intel_plane *plane = to_intel_plane(plane_state->base.plane); + struct drm_i915_private *dev_priv = to_i915(plane->base.dev); const struct drm_framebuffer *fb = plane_state->base.fb; unsigned int rotation = plane_state->base.rotation; struct drm_format_name_buf format_name; @@ -1247,13 +1480,17 @@ static int skl_plane_check_fb(const struct intel_crtc_state *crtc_state, } /* - * 90/270 is not allowed with RGB64 16:16:16:16, - * RGB 16-bit 5:6:5, and Indexed 8-bit. - * TBD: Add RGB64 case once its added in supported format list. + * 90/270 is not allowed with RGB64 16:16:16:16 and + * Indexed 8-bit. RGB 16-bit 5:6:5 is allowed gen11 onwards. + * TBD: Add RGB64 case once its added in supported format + * list. */ switch (fb->format->format) { - case DRM_FORMAT_C8: case DRM_FORMAT_RGB565: + if (INTEL_GEN(dev_priv) >= 11) + break; + /* fall through */ + case DRM_FORMAT_C8: DRM_DEBUG_KMS("Unsupported pixel format %s for 90/270!\n", drm_get_format_name(fb->format->format, &format_name)); @@ -1307,12 +1544,31 @@ static int skl_plane_check_dst_coordinates(const struct intel_crtc_state *crtc_s return 0; } -int skl_plane_check(struct intel_crtc_state *crtc_state, - struct intel_plane_state *plane_state) +static int skl_plane_check_nv12_rotation(const struct intel_plane_state *plane_state) +{ + const struct drm_framebuffer *fb = plane_state->base.fb; + unsigned int rotation = plane_state->base.rotation; + int src_w = drm_rect_width(&plane_state->base.src) >> 16; + + /* Display WA #1106 */ + if (fb->format->format == DRM_FORMAT_NV12 && src_w & 3 && + (rotation == DRM_MODE_ROTATE_270 || + rotation == (DRM_MODE_REFLECT_X | DRM_MODE_ROTATE_90))) { + DRM_DEBUG_KMS("src width must be multiple of 4 for rotated NV12\n"); + return -EINVAL; + } + + return 0; +} + +static int skl_plane_check(struct intel_crtc_state *crtc_state, + struct intel_plane_state *plane_state) { struct intel_plane *plane = to_intel_plane(plane_state->base.plane); struct drm_i915_private *dev_priv = to_i915(plane->base.dev); - int max_scale, min_scale; + const struct drm_framebuffer *fb = plane_state->base.fb; + int min_scale = DRM_PLANE_HELPER_NO_SCALING; + int max_scale = DRM_PLANE_HELPER_NO_SCALING; int ret; ret = skl_plane_check_fb(crtc_state, plane_state); @@ -1320,15 +1576,9 @@ int skl_plane_check(struct intel_crtc_state *crtc_state, return ret; /* use scaler when colorkey is not required */ - if (!plane_state->ckey.flags) { - const struct drm_framebuffer *fb = plane_state->base.fb; - + if (!plane_state->ckey.flags && intel_fb_scalable(fb)) { min_scale = 1; - max_scale = skl_max_scale(crtc_state, - fb ? fb->format->format : 0); - } else { - min_scale = DRM_PLANE_HELPER_NO_SCALING; - max_scale = DRM_PLANE_HELPER_NO_SCALING; + max_scale = skl_max_scale(crtc_state, fb->format->format); } ret = drm_atomic_helper_check_plane_state(&plane_state->base, @@ -1349,10 +1599,18 @@ int skl_plane_check(struct intel_crtc_state *crtc_state, if (ret) return ret; + ret = skl_plane_check_nv12_rotation(plane_state); + if (ret) + return ret; + ret = skl_check_plane_surface(plane_state); if (ret) return ret; + /* HW only has 8 bits pixel precision, disable plane if invisible */ + if (!(plane_state->base.alpha >> 8)) + plane_state->base.visible = false; + plane_state->ctl = skl_plane_ctl(crtc_state, plane_state); if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) @@ -1480,7 +1738,7 @@ out: return ret; } -static const uint32_t g4x_plane_formats[] = { +static const u32 g4x_plane_formats[] = { DRM_FORMAT_XRGB8888, DRM_FORMAT_YUYV, DRM_FORMAT_YVYU, @@ -1488,13 +1746,13 @@ static const uint32_t g4x_plane_formats[] = { DRM_FORMAT_VYUY, }; -static const uint64_t i9xx_plane_format_modifiers[] = { +static const u64 i9xx_plane_format_modifiers[] = { I915_FORMAT_MOD_X_TILED, DRM_FORMAT_MOD_LINEAR, DRM_FORMAT_MOD_INVALID }; -static const uint32_t snb_plane_formats[] = { +static const u32 snb_plane_formats[] = { DRM_FORMAT_XBGR8888, DRM_FORMAT_XRGB8888, DRM_FORMAT_YUYV, @@ -1503,7 +1761,7 @@ static const uint32_t snb_plane_formats[] = { DRM_FORMAT_VYUY, }; -static const uint32_t vlv_plane_formats[] = { +static const u32 vlv_plane_formats[] = { DRM_FORMAT_RGB565, DRM_FORMAT_ABGR8888, DRM_FORMAT_ARGB8888, @@ -1517,24 +1775,30 @@ static const uint32_t vlv_plane_formats[] = { DRM_FORMAT_VYUY, }; -static uint32_t skl_plane_formats[] = { +static const u32 skl_plane_formats[] = { + DRM_FORMAT_C8, DRM_FORMAT_RGB565, - DRM_FORMAT_ABGR8888, - DRM_FORMAT_ARGB8888, - DRM_FORMAT_XBGR8888, DRM_FORMAT_XRGB8888, + DRM_FORMAT_XBGR8888, + DRM_FORMAT_ARGB8888, + DRM_FORMAT_ABGR8888, + DRM_FORMAT_XRGB2101010, + DRM_FORMAT_XBGR2101010, DRM_FORMAT_YUYV, DRM_FORMAT_YVYU, DRM_FORMAT_UYVY, DRM_FORMAT_VYUY, }; -static uint32_t skl_planar_formats[] = { +static const u32 skl_planar_formats[] = { + DRM_FORMAT_C8, DRM_FORMAT_RGB565, - DRM_FORMAT_ABGR8888, - DRM_FORMAT_ARGB8888, - DRM_FORMAT_XBGR8888, DRM_FORMAT_XRGB8888, + DRM_FORMAT_XBGR8888, + DRM_FORMAT_ARGB8888, + DRM_FORMAT_ABGR8888, + DRM_FORMAT_XRGB2101010, + DRM_FORMAT_XBGR2101010, DRM_FORMAT_YUYV, DRM_FORMAT_YVYU, DRM_FORMAT_UYVY, @@ -1542,7 +1806,7 @@ static uint32_t skl_planar_formats[] = { DRM_FORMAT_NV12, }; -static const uint64_t skl_plane_format_modifiers_noccs[] = { +static const u64 skl_plane_format_modifiers_noccs[] = { I915_FORMAT_MOD_Yf_TILED, I915_FORMAT_MOD_Y_TILED, I915_FORMAT_MOD_X_TILED, @@ -1550,7 +1814,7 @@ static const uint64_t skl_plane_format_modifiers_noccs[] = { DRM_FORMAT_MOD_INVALID }; -static const uint64_t skl_plane_format_modifiers_ccs[] = { +static const u64 skl_plane_format_modifiers_ccs[] = { I915_FORMAT_MOD_Yf_TILED_CCS, I915_FORMAT_MOD_Y_TILED_CCS, I915_FORMAT_MOD_Yf_TILED, @@ -1739,8 +2003,36 @@ static const struct drm_plane_funcs skl_plane_funcs = { .format_mod_supported = skl_plane_format_mod_supported, }; -bool skl_plane_has_ccs(struct drm_i915_private *dev_priv, - enum pipe pipe, enum plane_id plane_id) +static bool skl_plane_has_fbc(struct drm_i915_private *dev_priv, + enum pipe pipe, enum plane_id plane_id) +{ + if (!HAS_FBC(dev_priv)) + return false; + + return pipe == PIPE_A && plane_id == PLANE_PRIMARY; +} + +static bool skl_plane_has_planar(struct drm_i915_private *dev_priv, + enum pipe pipe, enum plane_id plane_id) +{ + if (INTEL_GEN(dev_priv) >= 11) + return plane_id <= PLANE_SPRITE3; + + /* Display WA #0870: skl, bxt */ + if (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv)) + return false; + + if (IS_GEN(dev_priv, 9) && !IS_GEMINILAKE(dev_priv) && pipe == PIPE_C) + return false; + + if (plane_id != PLANE_PRIMARY && plane_id != PLANE_SPRITE0) + return false; + + return true; +} + +static bool skl_plane_has_ccs(struct drm_i915_private *dev_priv, + enum pipe pipe, enum plane_id plane_id) { if (plane_id == PLANE_CURSOR) return false; @@ -1757,109 +2049,173 @@ bool skl_plane_has_ccs(struct drm_i915_private *dev_priv, } struct intel_plane * -intel_sprite_plane_create(struct drm_i915_private *dev_priv, - enum pipe pipe, int plane) +skl_universal_plane_create(struct drm_i915_private *dev_priv, + enum pipe pipe, enum plane_id plane_id) { - struct intel_plane *intel_plane = NULL; - struct intel_plane_state *state = NULL; - const struct drm_plane_funcs *plane_funcs; - unsigned long possible_crtcs; - const uint32_t *plane_formats; - const uint64_t *modifiers; + struct intel_plane *plane; + enum drm_plane_type plane_type; unsigned int supported_rotations; - int num_plane_formats; + unsigned int possible_crtcs; + const u64 *modifiers; + const u32 *formats; + int num_formats; int ret; - intel_plane = kzalloc(sizeof(*intel_plane), GFP_KERNEL); - if (!intel_plane) { - ret = -ENOMEM; - goto fail; + plane = intel_plane_alloc(); + if (IS_ERR(plane)) + return plane; + + plane->pipe = pipe; + plane->id = plane_id; + plane->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, plane_id); + + plane->has_fbc = skl_plane_has_fbc(dev_priv, pipe, plane_id); + if (plane->has_fbc) { + struct intel_fbc *fbc = &dev_priv->fbc; + + fbc->possible_framebuffer_bits |= plane->frontbuffer_bit; } - state = intel_create_plane_state(&intel_plane->base); - if (!state) { - ret = -ENOMEM; - goto fail; + plane->max_stride = skl_plane_max_stride; + plane->update_plane = skl_update_plane; + plane->disable_plane = skl_disable_plane; + plane->get_hw_state = skl_plane_get_hw_state; + plane->check_plane = skl_plane_check; + if (icl_is_nv12_y_plane(plane_id)) + plane->update_slave = icl_update_slave; + + if (skl_plane_has_planar(dev_priv, pipe, plane_id)) { + formats = skl_planar_formats; + num_formats = ARRAY_SIZE(skl_planar_formats); + } else { + formats = skl_plane_formats; + num_formats = ARRAY_SIZE(skl_plane_formats); } - intel_plane->base.state = &state->base; - if (INTEL_GEN(dev_priv) >= 9) { - state->scaler_id = -1; + plane->has_ccs = skl_plane_has_ccs(dev_priv, pipe, plane_id); + if (plane->has_ccs) + modifiers = skl_plane_format_modifiers_ccs; + else + modifiers = skl_plane_format_modifiers_noccs; - intel_plane->has_ccs = skl_plane_has_ccs(dev_priv, pipe, - PLANE_SPRITE0 + plane); + if (plane_id == PLANE_PRIMARY) + plane_type = DRM_PLANE_TYPE_PRIMARY; + else + plane_type = DRM_PLANE_TYPE_OVERLAY; - intel_plane->max_stride = skl_plane_max_stride; - intel_plane->update_plane = skl_update_plane; - intel_plane->disable_plane = skl_disable_plane; - intel_plane->get_hw_state = skl_plane_get_hw_state; - intel_plane->check_plane = skl_plane_check; + possible_crtcs = BIT(pipe); - if (skl_plane_has_planar(dev_priv, pipe, - PLANE_SPRITE0 + plane)) { - plane_formats = skl_planar_formats; - num_plane_formats = ARRAY_SIZE(skl_planar_formats); - } else { - plane_formats = skl_plane_formats; - num_plane_formats = ARRAY_SIZE(skl_plane_formats); - } + ret = drm_universal_plane_init(&dev_priv->drm, &plane->base, + possible_crtcs, &skl_plane_funcs, + formats, num_formats, modifiers, + plane_type, + "plane %d%c", plane_id + 1, + pipe_name(pipe)); + if (ret) + goto fail; + + supported_rotations = + DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_90 | + DRM_MODE_ROTATE_180 | DRM_MODE_ROTATE_270; + + if (INTEL_GEN(dev_priv) >= 10) + supported_rotations |= DRM_MODE_REFLECT_X; + + drm_plane_create_rotation_property(&plane->base, + DRM_MODE_ROTATE_0, + supported_rotations); + + drm_plane_create_color_properties(&plane->base, + BIT(DRM_COLOR_YCBCR_BT601) | + BIT(DRM_COLOR_YCBCR_BT709), + BIT(DRM_COLOR_YCBCR_LIMITED_RANGE) | + BIT(DRM_COLOR_YCBCR_FULL_RANGE), + DRM_COLOR_YCBCR_BT709, + DRM_COLOR_YCBCR_LIMITED_RANGE); + + drm_plane_create_alpha_property(&plane->base); + drm_plane_create_blend_mode_property(&plane->base, + BIT(DRM_MODE_BLEND_PIXEL_NONE) | + BIT(DRM_MODE_BLEND_PREMULTI) | + BIT(DRM_MODE_BLEND_COVERAGE)); + + drm_plane_helper_add(&plane->base, &intel_plane_helper_funcs); - if (intel_plane->has_ccs) - modifiers = skl_plane_format_modifiers_ccs; - else - modifiers = skl_plane_format_modifiers_noccs; - - plane_funcs = &skl_plane_funcs; - } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { - intel_plane->max_stride = i9xx_plane_max_stride; - intel_plane->update_plane = vlv_update_plane; - intel_plane->disable_plane = vlv_disable_plane; - intel_plane->get_hw_state = vlv_plane_get_hw_state; - intel_plane->check_plane = vlv_sprite_check; - - plane_formats = vlv_plane_formats; - num_plane_formats = ARRAY_SIZE(vlv_plane_formats); + return plane; + +fail: + intel_plane_free(plane); + + return ERR_PTR(ret); +} + +struct intel_plane * +intel_sprite_plane_create(struct drm_i915_private *dev_priv, + enum pipe pipe, int sprite) +{ + struct intel_plane *plane; + const struct drm_plane_funcs *plane_funcs; + unsigned long possible_crtcs; + unsigned int supported_rotations; + const u64 *modifiers; + const u32 *formats; + int num_formats; + int ret; + + if (INTEL_GEN(dev_priv) >= 9) + return skl_universal_plane_create(dev_priv, pipe, + PLANE_SPRITE0 + sprite); + + plane = intel_plane_alloc(); + if (IS_ERR(plane)) + return plane; + + if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { + plane->max_stride = i9xx_plane_max_stride; + plane->update_plane = vlv_update_plane; + plane->disable_plane = vlv_disable_plane; + plane->get_hw_state = vlv_plane_get_hw_state; + plane->check_plane = vlv_sprite_check; + + formats = vlv_plane_formats; + num_formats = ARRAY_SIZE(vlv_plane_formats); modifiers = i9xx_plane_format_modifiers; plane_funcs = &vlv_sprite_funcs; } else if (INTEL_GEN(dev_priv) >= 7) { - intel_plane->max_stride = g4x_sprite_max_stride; - intel_plane->update_plane = ivb_update_plane; - intel_plane->disable_plane = ivb_disable_plane; - intel_plane->get_hw_state = ivb_plane_get_hw_state; - intel_plane->check_plane = g4x_sprite_check; - - plane_formats = snb_plane_formats; - num_plane_formats = ARRAY_SIZE(snb_plane_formats); + plane->max_stride = g4x_sprite_max_stride; + plane->update_plane = ivb_update_plane; + plane->disable_plane = ivb_disable_plane; + plane->get_hw_state = ivb_plane_get_hw_state; + plane->check_plane = g4x_sprite_check; + + formats = snb_plane_formats; + num_formats = ARRAY_SIZE(snb_plane_formats); modifiers = i9xx_plane_format_modifiers; plane_funcs = &snb_sprite_funcs; } else { - intel_plane->max_stride = g4x_sprite_max_stride; - intel_plane->update_plane = g4x_update_plane; - intel_plane->disable_plane = g4x_disable_plane; - intel_plane->get_hw_state = g4x_plane_get_hw_state; - intel_plane->check_plane = g4x_sprite_check; + plane->max_stride = g4x_sprite_max_stride; + plane->update_plane = g4x_update_plane; + plane->disable_plane = g4x_disable_plane; + plane->get_hw_state = g4x_plane_get_hw_state; + plane->check_plane = g4x_sprite_check; modifiers = i9xx_plane_format_modifiers; - if (IS_GEN6(dev_priv)) { - plane_formats = snb_plane_formats; - num_plane_formats = ARRAY_SIZE(snb_plane_formats); + if (IS_GEN(dev_priv, 6)) { + formats = snb_plane_formats; + num_formats = ARRAY_SIZE(snb_plane_formats); plane_funcs = &snb_sprite_funcs; } else { - plane_formats = g4x_plane_formats; - num_plane_formats = ARRAY_SIZE(g4x_plane_formats); + formats = g4x_plane_formats; + num_formats = ARRAY_SIZE(g4x_plane_formats); plane_funcs = &g4x_sprite_funcs; } } - if (INTEL_GEN(dev_priv) >= 9) { - supported_rotations = - DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_90 | - DRM_MODE_ROTATE_180 | DRM_MODE_ROTATE_270; - } else if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) { + if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) { supported_rotations = DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180 | DRM_MODE_REFLECT_X; @@ -1868,35 +2224,25 @@ intel_sprite_plane_create(struct drm_i915_private *dev_priv, DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180; } - intel_plane->pipe = pipe; - intel_plane->i9xx_plane = plane; - intel_plane->id = PLANE_SPRITE0 + plane; - intel_plane->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, intel_plane->id); + plane->pipe = pipe; + plane->id = PLANE_SPRITE0 + sprite; + plane->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, plane->id); - possible_crtcs = (1 << pipe); + possible_crtcs = BIT(pipe); - if (INTEL_GEN(dev_priv) >= 9) - ret = drm_universal_plane_init(&dev_priv->drm, &intel_plane->base, - possible_crtcs, plane_funcs, - plane_formats, num_plane_formats, - modifiers, - DRM_PLANE_TYPE_OVERLAY, - "plane %d%c", plane + 2, pipe_name(pipe)); - else - ret = drm_universal_plane_init(&dev_priv->drm, &intel_plane->base, - possible_crtcs, plane_funcs, - plane_formats, num_plane_formats, - modifiers, - DRM_PLANE_TYPE_OVERLAY, - "sprite %c", sprite_name(pipe, plane)); + ret = drm_universal_plane_init(&dev_priv->drm, &plane->base, + possible_crtcs, plane_funcs, + formats, num_formats, modifiers, + DRM_PLANE_TYPE_OVERLAY, + "sprite %c", sprite_name(pipe, sprite)); if (ret) goto fail; - drm_plane_create_rotation_property(&intel_plane->base, + drm_plane_create_rotation_property(&plane->base, DRM_MODE_ROTATE_0, supported_rotations); - drm_plane_create_color_properties(&intel_plane->base, + drm_plane_create_color_properties(&plane->base, BIT(DRM_COLOR_YCBCR_BT601) | BIT(DRM_COLOR_YCBCR_BT709), BIT(DRM_COLOR_YCBCR_LIMITED_RANGE) | @@ -1904,13 +2250,12 @@ intel_sprite_plane_create(struct drm_i915_private *dev_priv, DRM_COLOR_YCBCR_BT709, DRM_COLOR_YCBCR_LIMITED_RANGE); - drm_plane_helper_add(&intel_plane->base, &intel_plane_helper_funcs); + drm_plane_helper_add(&plane->base, &intel_plane_helper_funcs); - return intel_plane; + return plane; fail: - kfree(state); - kfree(intel_plane); + intel_plane_free(plane); return ERR_PTR(ret); } diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c index b5b04cb892e9..3924c4944e1f 100644 --- a/drivers/gpu/drm/i915/intel_tv.c +++ b/drivers/gpu/drm/i915/intel_tv.c @@ -30,7 +30,6 @@ * Integrated TV-out support for the 915GM and 945GM. */ -#include <drm/drmP.h> #include <drm/drm_atomic_helper.h> #include <drm/drm_crtc.h> #include <drm/drm_edid.h> @@ -307,7 +306,7 @@ struct tv_mode { u32 clock; u16 refresh; /* in millihertz (for precision) */ - u32 oversample; + u8 oversample; u8 hsync_end; u16 hblank_start, hblank_end, htotal; bool progressive : 1, trilevel_sync : 1, component_only : 1; @@ -340,7 +339,6 @@ struct tv_mode { const struct video_levels *composite_levels, *svideo_levels; const struct color_conversion *composite_color, *svideo_color; const u32 *filter_table; - u16 max_srcw; }; @@ -379,8 +377,8 @@ static const struct tv_mode tv_modes[] = { .name = "NTSC-M", .clock = 108000, .refresh = 59940, - .oversample = TV_OVERSAMPLE_8X, - .component_only = 0, + .oversample = 8, + .component_only = false, /* 525 Lines, 60 Fields, 15.734KHz line, Sub-Carrier 3.580MHz */ .hsync_end = 64, .hblank_end = 124, @@ -422,8 +420,8 @@ static const struct tv_mode tv_modes[] = { .name = "NTSC-443", .clock = 108000, .refresh = 59940, - .oversample = TV_OVERSAMPLE_8X, - .component_only = 0, + .oversample = 8, + .component_only = false, /* 525 Lines, 60 Fields, 15.734KHz line, Sub-Carrier 4.43MHz */ .hsync_end = 64, .hblank_end = 124, .hblank_start = 836, .htotal = 857, @@ -464,8 +462,8 @@ static const struct tv_mode tv_modes[] = { .name = "NTSC-J", .clock = 108000, .refresh = 59940, - .oversample = TV_OVERSAMPLE_8X, - .component_only = 0, + .oversample = 8, + .component_only = false, /* 525 Lines, 60 Fields, 15.734KHz line, Sub-Carrier 3.580MHz */ .hsync_end = 64, .hblank_end = 124, @@ -507,8 +505,8 @@ static const struct tv_mode tv_modes[] = { .name = "PAL-M", .clock = 108000, .refresh = 59940, - .oversample = TV_OVERSAMPLE_8X, - .component_only = 0, + .oversample = 8, + .component_only = false, /* 525 Lines, 60 Fields, 15.734KHz line, Sub-Carrier 3.580MHz */ .hsync_end = 64, .hblank_end = 124, @@ -551,8 +549,8 @@ static const struct tv_mode tv_modes[] = { .name = "PAL-N", .clock = 108000, .refresh = 50000, - .oversample = TV_OVERSAMPLE_8X, - .component_only = 0, + .oversample = 8, + .component_only = false, .hsync_end = 64, .hblank_end = 128, .hblank_start = 844, .htotal = 863, @@ -596,8 +594,8 @@ static const struct tv_mode tv_modes[] = { .name = "PAL", .clock = 108000, .refresh = 50000, - .oversample = TV_OVERSAMPLE_8X, - .component_only = 0, + .oversample = 8, + .component_only = false, .hsync_end = 64, .hblank_end = 142, .hblank_start = 844, .htotal = 863, @@ -636,10 +634,10 @@ static const struct tv_mode tv_modes[] = { }, { .name = "480p", - .clock = 107520, + .clock = 108000, .refresh = 59940, - .oversample = TV_OVERSAMPLE_4X, - .component_only = 1, + .oversample = 4, + .component_only = true, .hsync_end = 64, .hblank_end = 122, .hblank_start = 842, .htotal = 857, @@ -660,10 +658,10 @@ static const struct tv_mode tv_modes[] = { }, { .name = "576p", - .clock = 107520, + .clock = 108000, .refresh = 50000, - .oversample = TV_OVERSAMPLE_4X, - .component_only = 1, + .oversample = 4, + .component_only = true, .hsync_end = 64, .hblank_end = 139, .hblank_start = 859, .htotal = 863, @@ -684,10 +682,10 @@ static const struct tv_mode tv_modes[] = { }, { .name = "720p@60Hz", - .clock = 148800, + .clock = 148500, .refresh = 60000, - .oversample = TV_OVERSAMPLE_2X, - .component_only = 1, + .oversample = 2, + .component_only = true, .hsync_end = 80, .hblank_end = 300, .hblank_start = 1580, .htotal = 1649, @@ -708,10 +706,10 @@ static const struct tv_mode tv_modes[] = { }, { .name = "720p@50Hz", - .clock = 148800, + .clock = 148500, .refresh = 50000, - .oversample = TV_OVERSAMPLE_2X, - .component_only = 1, + .oversample = 2, + .component_only = true, .hsync_end = 80, .hblank_end = 300, .hblank_start = 1580, .htotal = 1979, @@ -729,14 +727,13 @@ static const struct tv_mode tv_modes[] = { .burst_ena = false, .filter_table = filter_table, - .max_srcw = 800 }, { .name = "1080i@50Hz", - .clock = 148800, + .clock = 148500, .refresh = 50000, - .oversample = TV_OVERSAMPLE_2X, - .component_only = 1, + .oversample = 2, + .component_only = true, .hsync_end = 88, .hblank_end = 235, .hblank_start = 2155, .htotal = 2639, @@ -759,10 +756,10 @@ static const struct tv_mode tv_modes[] = { }, { .name = "1080i@60Hz", - .clock = 148800, + .clock = 148500, .refresh = 60000, - .oversample = TV_OVERSAMPLE_2X, - .component_only = 1, + .oversample = 2, + .component_only = true, .hsync_end = 88, .hblank_end = 235, .hblank_start = 2155, .htotal = 2199, @@ -783,8 +780,115 @@ static const struct tv_mode tv_modes[] = { .filter_table = filter_table, }, + + { + .name = "1080p@30Hz", + .clock = 148500, + .refresh = 30000, + .oversample = 2, + .component_only = true, + + .hsync_end = 88, .hblank_end = 235, + .hblank_start = 2155, .htotal = 2199, + + .progressive = true, .trilevel_sync = true, + + .vsync_start_f1 = 8, .vsync_start_f2 = 8, + .vsync_len = 10, + + .veq_ena = false, .veq_start_f1 = 0, + .veq_start_f2 = 0, .veq_len = 0, + + .vi_end_f1 = 44, .vi_end_f2 = 44, + .nbr_end = 1079, + + .burst_ena = false, + + .filter_table = filter_table, + }, + + { + .name = "1080p@50Hz", + .clock = 148500, + .refresh = 50000, + .oversample = 1, + .component_only = true, + + .hsync_end = 88, .hblank_end = 235, + .hblank_start = 2155, .htotal = 2639, + + .progressive = true, .trilevel_sync = true, + + .vsync_start_f1 = 8, .vsync_start_f2 = 8, + .vsync_len = 10, + + .veq_ena = false, .veq_start_f1 = 0, + .veq_start_f2 = 0, .veq_len = 0, + + .vi_end_f1 = 44, .vi_end_f2 = 44, + .nbr_end = 1079, + + .burst_ena = false, + + .filter_table = filter_table, + }, + + { + .name = "1080p@60Hz", + .clock = 148500, + .refresh = 60000, + .oversample = 1, + .component_only = true, + + .hsync_end = 88, .hblank_end = 235, + .hblank_start = 2155, .htotal = 2199, + + .progressive = true, .trilevel_sync = true, + + .vsync_start_f1 = 8, .vsync_start_f2 = 8, + .vsync_len = 10, + + .veq_ena = false, .veq_start_f1 = 0, + .veq_start_f2 = 0, .veq_len = 0, + + .vi_end_f1 = 44, .vi_end_f2 = 44, + .nbr_end = 1079, + + .burst_ena = false, + + .filter_table = filter_table, + }, }; +struct intel_tv_connector_state { + struct drm_connector_state base; + + /* + * May need to override the user margins for + * gen3 >1024 wide source vertical centering. + */ + struct { + u16 top, bottom; + } margins; + + bool bypass_vfilter; +}; + +#define to_intel_tv_connector_state(x) container_of(x, struct intel_tv_connector_state, base) + +static struct drm_connector_state * +intel_tv_connector_duplicate_state(struct drm_connector *connector) +{ + struct intel_tv_connector_state *state; + + state = kmemdup(connector->state, sizeof(*state), GFP_KERNEL); + if (!state) + return NULL; + + __drm_atomic_helper_connector_duplicate_state(connector, &state->base); + return &state->base; +} + static struct intel_tv *enc_to_tv(struct intel_encoder *encoder) { return container_of(encoder, struct intel_tv, base); @@ -860,44 +964,370 @@ intel_tv_mode_valid(struct drm_connector *connector, return MODE_CLOCK_RANGE; } +static int +intel_tv_mode_vdisplay(const struct tv_mode *tv_mode) +{ + if (tv_mode->progressive) + return tv_mode->nbr_end + 1; + else + return 2 * (tv_mode->nbr_end + 1); +} + +static void +intel_tv_mode_to_mode(struct drm_display_mode *mode, + const struct tv_mode *tv_mode) +{ + mode->clock = tv_mode->clock / + (tv_mode->oversample >> !tv_mode->progressive); + + /* + * tv_mode horizontal timings: + * + * hsync_end + * | hblank_end + * | | hblank_start + * | | | htotal + * | _______ | + * ____/ \___ + * \__/ \ + */ + mode->hdisplay = + tv_mode->hblank_start - tv_mode->hblank_end; + mode->hsync_start = mode->hdisplay + + tv_mode->htotal - tv_mode->hblank_start; + mode->hsync_end = mode->hsync_start + + tv_mode->hsync_end; + mode->htotal = tv_mode->htotal + 1; + + /* + * tv_mode vertical timings: + * + * vsync_start + * | vsync_end + * | | vi_end nbr_end + * | | | | + * | | _______ + * \__ ____/ \ + * \__/ + */ + mode->vdisplay = intel_tv_mode_vdisplay(tv_mode); + if (tv_mode->progressive) { + mode->vsync_start = mode->vdisplay + + tv_mode->vsync_start_f1 + 1; + mode->vsync_end = mode->vsync_start + + tv_mode->vsync_len; + mode->vtotal = mode->vdisplay + + tv_mode->vi_end_f1 + 1; + } else { + mode->vsync_start = mode->vdisplay + + tv_mode->vsync_start_f1 + 1 + + tv_mode->vsync_start_f2 + 1; + mode->vsync_end = mode->vsync_start + + 2 * tv_mode->vsync_len; + mode->vtotal = mode->vdisplay + + tv_mode->vi_end_f1 + 1 + + tv_mode->vi_end_f2 + 1; + } + + /* TV has it's own notion of sync and other mode flags, so clear them. */ + mode->flags = 0; + + mode->vrefresh = 0; + mode->vrefresh = drm_mode_vrefresh(mode); + + snprintf(mode->name, sizeof(mode->name), + "%dx%d%c (%s)", + mode->hdisplay, mode->vdisplay, + tv_mode->progressive ? 'p' : 'i', + tv_mode->name); +} + +static void intel_tv_scale_mode_horiz(struct drm_display_mode *mode, + int hdisplay, int left_margin, + int right_margin) +{ + int hsync_start = mode->hsync_start - mode->hdisplay + right_margin; + int hsync_end = mode->hsync_end - mode->hdisplay + right_margin; + int new_htotal = mode->htotal * hdisplay / + (mode->hdisplay - left_margin - right_margin); + + mode->clock = mode->clock * new_htotal / mode->htotal; + + mode->hdisplay = hdisplay; + mode->hsync_start = hdisplay + hsync_start * new_htotal / mode->htotal; + mode->hsync_end = hdisplay + hsync_end * new_htotal / mode->htotal; + mode->htotal = new_htotal; +} + +static void intel_tv_scale_mode_vert(struct drm_display_mode *mode, + int vdisplay, int top_margin, + int bottom_margin) +{ + int vsync_start = mode->vsync_start - mode->vdisplay + bottom_margin; + int vsync_end = mode->vsync_end - mode->vdisplay + bottom_margin; + int new_vtotal = mode->vtotal * vdisplay / + (mode->vdisplay - top_margin - bottom_margin); + + mode->clock = mode->clock * new_vtotal / mode->vtotal; + + mode->vdisplay = vdisplay; + mode->vsync_start = vdisplay + vsync_start * new_vtotal / mode->vtotal; + mode->vsync_end = vdisplay + vsync_end * new_vtotal / mode->vtotal; + mode->vtotal = new_vtotal; +} static void intel_tv_get_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) { + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct drm_display_mode *adjusted_mode = + &pipe_config->base.adjusted_mode; + struct drm_display_mode mode = {}; + u32 tv_ctl, hctl1, hctl3, vctl1, vctl2, tmp; + struct tv_mode tv_mode = {}; + int hdisplay = adjusted_mode->crtc_hdisplay; + int vdisplay = adjusted_mode->crtc_vdisplay; + int xsize, ysize, xpos, ypos; + pipe_config->output_types |= BIT(INTEL_OUTPUT_TVOUT); - pipe_config->base.adjusted_mode.crtc_clock = pipe_config->port_clock; + tv_ctl = I915_READ(TV_CTL); + hctl1 = I915_READ(TV_H_CTL_1); + hctl3 = I915_READ(TV_H_CTL_3); + vctl1 = I915_READ(TV_V_CTL_1); + vctl2 = I915_READ(TV_V_CTL_2); + + tv_mode.htotal = (hctl1 & TV_HTOTAL_MASK) >> TV_HTOTAL_SHIFT; + tv_mode.hsync_end = (hctl1 & TV_HSYNC_END_MASK) >> TV_HSYNC_END_SHIFT; + + tv_mode.hblank_start = (hctl3 & TV_HBLANK_START_MASK) >> TV_HBLANK_START_SHIFT; + tv_mode.hblank_end = (hctl3 & TV_HSYNC_END_MASK) >> TV_HBLANK_END_SHIFT; + + tv_mode.nbr_end = (vctl1 & TV_NBR_END_MASK) >> TV_NBR_END_SHIFT; + tv_mode.vi_end_f1 = (vctl1 & TV_VI_END_F1_MASK) >> TV_VI_END_F1_SHIFT; + tv_mode.vi_end_f2 = (vctl1 & TV_VI_END_F2_MASK) >> TV_VI_END_F2_SHIFT; + + tv_mode.vsync_len = (vctl2 & TV_VSYNC_LEN_MASK) >> TV_VSYNC_LEN_SHIFT; + tv_mode.vsync_start_f1 = (vctl2 & TV_VSYNC_START_F1_MASK) >> TV_VSYNC_START_F1_SHIFT; + tv_mode.vsync_start_f2 = (vctl2 & TV_VSYNC_START_F2_MASK) >> TV_VSYNC_START_F2_SHIFT; + + tv_mode.clock = pipe_config->port_clock; + + tv_mode.progressive = tv_ctl & TV_PROGRESSIVE; + + switch (tv_ctl & TV_OVERSAMPLE_MASK) { + case TV_OVERSAMPLE_8X: + tv_mode.oversample = 8; + break; + case TV_OVERSAMPLE_4X: + tv_mode.oversample = 4; + break; + case TV_OVERSAMPLE_2X: + tv_mode.oversample = 2; + break; + default: + tv_mode.oversample = 1; + break; + } + + tmp = I915_READ(TV_WIN_POS); + xpos = tmp >> 16; + ypos = tmp & 0xffff; + + tmp = I915_READ(TV_WIN_SIZE); + xsize = tmp >> 16; + ysize = tmp & 0xffff; + + intel_tv_mode_to_mode(&mode, &tv_mode); + + DRM_DEBUG_KMS("TV mode:\n"); + drm_mode_debug_printmodeline(&mode); + + intel_tv_scale_mode_horiz(&mode, hdisplay, + xpos, mode.hdisplay - xsize - xpos); + intel_tv_scale_mode_vert(&mode, vdisplay, + ypos, mode.vdisplay - ysize - ypos); + + adjusted_mode->crtc_clock = mode.clock; + if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) + adjusted_mode->crtc_clock /= 2; + + /* pixel counter doesn't work on i965gm TV output */ + if (IS_I965GM(dev_priv)) + adjusted_mode->private_flags |= + I915_MODE_FLAG_USE_SCANLINE_COUNTER; } -static bool +static bool intel_tv_source_too_wide(struct drm_i915_private *dev_priv, + int hdisplay) +{ + return IS_GEN(dev_priv, 3) && hdisplay > 1024; +} + +static bool intel_tv_vert_scaling(const struct drm_display_mode *tv_mode, + const struct drm_connector_state *conn_state, + int vdisplay) +{ + return tv_mode->crtc_vdisplay - + conn_state->tv.margins.top - + conn_state->tv.margins.bottom != + vdisplay; +} + +static int intel_tv_compute_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config, struct drm_connector_state *conn_state) { + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_tv_connector_state *tv_conn_state = + to_intel_tv_connector_state(conn_state); const struct tv_mode *tv_mode = intel_tv_mode_find(conn_state); struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; + int hdisplay = adjusted_mode->crtc_hdisplay; + int vdisplay = adjusted_mode->crtc_vdisplay; if (!tv_mode) - return false; + return -EINVAL; if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) - return false; + return -EINVAL; + + pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; - adjusted_mode->crtc_clock = tv_mode->clock; DRM_DEBUG_KMS("forcing bpc to 8 for TV\n"); pipe_config->pipe_bpp = 8*3; - /* TV has it's own notion of sync and other mode flags, so clear them. */ - adjusted_mode->flags = 0; + pipe_config->port_clock = tv_mode->clock; + + intel_tv_mode_to_mode(adjusted_mode, tv_mode); + drm_mode_set_crtcinfo(adjusted_mode, 0); + + if (intel_tv_source_too_wide(dev_priv, hdisplay) || + !intel_tv_vert_scaling(adjusted_mode, conn_state, vdisplay)) { + int extra, top, bottom; + + extra = adjusted_mode->crtc_vdisplay - vdisplay; + + if (extra < 0) { + DRM_DEBUG_KMS("No vertical scaling for >1024 pixel wide modes\n"); + return -EINVAL; + } + + /* Need to turn off the vertical filter and center the image */ + + /* Attempt to maintain the relative sizes of the margins */ + top = conn_state->tv.margins.top; + bottom = conn_state->tv.margins.bottom; + + if (top + bottom) + top = extra * top / (top + bottom); + else + top = extra / 2; + bottom = extra - top; + + tv_conn_state->margins.top = top; + tv_conn_state->margins.bottom = bottom; + + tv_conn_state->bypass_vfilter = true; + + if (!tv_mode->progressive) { + adjusted_mode->clock /= 2; + adjusted_mode->crtc_clock /= 2; + adjusted_mode->flags |= DRM_MODE_FLAG_INTERLACE; + } + } else { + tv_conn_state->margins.top = conn_state->tv.margins.top; + tv_conn_state->margins.bottom = conn_state->tv.margins.bottom; + + tv_conn_state->bypass_vfilter = false; + } + + DRM_DEBUG_KMS("TV mode:\n"); + drm_mode_debug_printmodeline(adjusted_mode); /* - * FIXME: We don't check whether the input mode is actually what we want - * or whether userspace is doing something stupid. + * The pipe scanline counter behaviour looks as follows when + * using the TV encoder: + * + * time -> + * + * dsl=vtotal-1 | | + * || || + * ___| | ___| | + * / | / | + * / | / | + * dsl=0 ___/ |_____/ | + * | | | | | | + * ^ ^ ^ ^ ^ + * | | | | pipe vblank/first part of tv vblank + * | | | bottom margin + * | | active + * | top margin + * remainder of tv vblank + * + * When the TV encoder is used the pipe wants to run faster + * than expected rate. During the active portion the TV + * encoder stalls the pipe every few lines to keep it in + * check. When the TV encoder reaches the bottom margin the + * pipe simply stops. Once we reach the TV vblank the pipe is + * no longer stalled and it runs at the max rate (apparently + * oversample clock on gen3, cdclk on gen4). Once the pipe + * reaches the pipe vtotal the pipe stops for the remainder + * of the TV vblank/top margin. The pipe starts up again when + * the TV encoder exits the top margin. + * + * To avoid huge hassles for vblank timestamping we scale + * the pipe timings as if the pipe always runs at the average + * rate it maintains during the active period. This also + * gives us a reasonable guesstimate as to the pixel rate. + * Due to the variation in the actual pipe speed the scanline + * counter will give us slightly erroneous results during the + * TV vblank/margins. But since vtotal was selected such that + * it matches the average rate of the pipe during the active + * portion the error shouldn't cause any serious grief to + * vblank timestamps. + * + * For posterity here is the empirically derived formula + * that gives us the maximum length of the pipe vblank + * we can use without causing display corruption. Following + * this would allow us to have a ticking scanline counter + * everywhere except during the bottom margin (there the + * pipe always stops). Ie. this would eliminate the second + * flat portion of the above graph. However this would also + * complicate vblank timestamping as the pipe vtotal would + * no longer match the average rate the pipe runs at during + * the active portion. Hence following this formula seems + * more trouble that it's worth. + * + * if (IS_GEN(dev_priv, 4)) { + * num = cdclk * (tv_mode->oversample >> !tv_mode->progressive); + * den = tv_mode->clock; + * } else { + * num = tv_mode->oversample >> !tv_mode->progressive; + * den = 1; + * } + * max_pipe_vblank_len ~= + * (num * tv_htotal * (tv_vblank_len + top_margin)) / + * (den * pipe_htotal); */ + intel_tv_scale_mode_horiz(adjusted_mode, hdisplay, + conn_state->tv.margins.left, + conn_state->tv.margins.right); + intel_tv_scale_mode_vert(adjusted_mode, vdisplay, + tv_conn_state->margins.top, + tv_conn_state->margins.bottom); + drm_mode_set_crtcinfo(adjusted_mode, 0); + adjusted_mode->name[0] = '\0'; + + /* pixel counter doesn't work on i965gm TV output */ + if (IS_I965GM(dev_priv)) + adjusted_mode->private_flags |= + I915_MODE_FLAG_USE_SCANLINE_COUNTER; - return true; + return 0; } static void @@ -986,14 +1416,16 @@ static void intel_tv_pre_enable(struct intel_encoder *encoder, struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_crtc *intel_crtc = to_intel_crtc(pipe_config->base.crtc); struct intel_tv *intel_tv = enc_to_tv(encoder); + const struct intel_tv_connector_state *tv_conn_state = + to_intel_tv_connector_state(conn_state); const struct tv_mode *tv_mode = intel_tv_mode_find(conn_state); - u32 tv_ctl; + u32 tv_ctl, tv_filter_ctl; u32 scctl1, scctl2, scctl3; int i, j; const struct video_levels *video_levels; const struct color_conversion *color_conversion; bool burst_ena; - int xpos = 0x0, ypos = 0x0; + int xpos, ypos; unsigned int xsize, ysize; if (!tv_mode) @@ -1029,7 +1461,21 @@ static void intel_tv_pre_enable(struct intel_encoder *encoder, } tv_ctl |= TV_ENC_PIPE_SEL(intel_crtc->pipe); - tv_ctl |= tv_mode->oversample; + + switch (tv_mode->oversample) { + case 8: + tv_ctl |= TV_OVERSAMPLE_8X; + break; + case 4: + tv_ctl |= TV_OVERSAMPLE_4X; + break; + case 2: + tv_ctl |= TV_OVERSAMPLE_2X; + break; + default: + tv_ctl |= TV_OVERSAMPLE_NONE; + break; + } if (tv_mode->progressive) tv_ctl |= TV_PROGRESSIVE; @@ -1081,19 +1527,20 @@ static void intel_tv_pre_enable(struct intel_encoder *encoder, assert_pipe_disabled(dev_priv, intel_crtc->pipe); /* Filter ctl must be set before TV_WIN_SIZE */ - I915_WRITE(TV_FILTER_CTL_1, TV_AUTO_SCALE); + tv_filter_ctl = TV_AUTO_SCALE; + if (tv_conn_state->bypass_vfilter) + tv_filter_ctl |= TV_V_FILTER_BYPASS; + I915_WRITE(TV_FILTER_CTL_1, tv_filter_ctl); + xsize = tv_mode->hblank_start - tv_mode->hblank_end; - if (tv_mode->progressive) - ysize = tv_mode->nbr_end + 1; - else - ysize = 2*tv_mode->nbr_end + 1; + ysize = intel_tv_mode_vdisplay(tv_mode); - xpos += conn_state->tv.margins.left; - ypos += conn_state->tv.margins.top; + xpos = conn_state->tv.margins.left; + ypos = tv_conn_state->margins.top; xsize -= (conn_state->tv.margins.left + conn_state->tv.margins.right); - ysize -= (conn_state->tv.margins.top + - conn_state->tv.margins.bottom); + ysize -= (tv_conn_state->margins.top + + tv_conn_state->margins.bottom); I915_WRITE(TV_WIN_POS, (xpos<<16)|ypos); I915_WRITE(TV_WIN_SIZE, (xsize<<16)|ysize); @@ -1110,23 +1557,6 @@ static void intel_tv_pre_enable(struct intel_encoder *encoder, I915_WRITE(TV_CTL, tv_ctl); } -static const struct drm_display_mode reported_modes[] = { - { - .name = "NTSC 480i", - .clock = 107520, - .hdisplay = 1280, - .hsync_start = 1368, - .hsync_end = 1496, - .htotal = 1712, - - .vdisplay = 1024, - .vsync_start = 1027, - .vsync_end = 1034, - .vtotal = 1104, - .type = DRM_MODE_TYPE_DRIVER, - }, -}; - static int intel_tv_detect_type(struct intel_tv *intel_tv, struct drm_connector *connector) @@ -1233,16 +1663,18 @@ static void intel_tv_find_better_format(struct drm_connector *connector) const struct tv_mode *tv_mode = intel_tv_mode_find(connector->state); int i; - if ((intel_tv->type == DRM_MODE_CONNECTOR_Component) == - tv_mode->component_only) + /* Component supports everything so we can keep the current mode */ + if (intel_tv->type == DRM_MODE_CONNECTOR_Component) return; + /* If the current mode is fine don't change it */ + if (!tv_mode->component_only) + return; for (i = 0; i < ARRAY_SIZE(tv_modes); i++) { - tv_mode = tv_modes + i; + tv_mode = &tv_modes[i]; - if ((intel_tv->type == DRM_MODE_CONNECTOR_Component) == - tv_mode->component_only) + if (!tv_mode->component_only) break; } @@ -1254,7 +1686,6 @@ intel_tv_detect(struct drm_connector *connector, struct drm_modeset_acquire_ctx *ctx, bool force) { - struct drm_display_mode mode; struct intel_tv *intel_tv = intel_attached_tv(connector); enum drm_connector_status status; int type; @@ -1263,13 +1694,11 @@ intel_tv_detect(struct drm_connector *connector, connector->base.id, connector->name, force); - mode = reported_modes[0]; - if (force) { struct intel_load_detect_pipe tmp; int ret; - ret = intel_get_load_detect_pipe(connector, &mode, &tmp, ctx); + ret = intel_get_load_detect_pipe(connector, NULL, &tmp, ctx); if (ret < 0) return ret; @@ -1293,104 +1722,98 @@ intel_tv_detect(struct drm_connector *connector, } static const struct input_res { - const char *name; - int w, h; + u16 w, h; } input_res_table[] = { - {"640x480", 640, 480}, - {"800x600", 800, 600}, - {"1024x768", 1024, 768}, - {"1280x1024", 1280, 1024}, - {"848x480", 848, 480}, - {"1280x720", 1280, 720}, - {"1920x1080", 1920, 1080}, + { 640, 480 }, + { 800, 600 }, + { 1024, 768 }, + { 1280, 1024 }, + { 848, 480 }, + { 1280, 720 }, + { 1920, 1080 }, }; -/* - * Chose preferred mode according to line number of TV format - */ +/* Choose preferred mode according to line number of TV format */ +static bool +intel_tv_is_preferred_mode(const struct drm_display_mode *mode, + const struct tv_mode *tv_mode) +{ + int vdisplay = intel_tv_mode_vdisplay(tv_mode); + + /* prefer 480 line modes for all SD TV modes */ + if (vdisplay <= 576) + vdisplay = 480; + + return vdisplay == mode->vdisplay; +} + static void -intel_tv_choose_preferred_modes(const struct tv_mode *tv_mode, - struct drm_display_mode *mode_ptr) +intel_tv_set_mode_type(struct drm_display_mode *mode, + const struct tv_mode *tv_mode) { - if (tv_mode->nbr_end < 480 && mode_ptr->vdisplay == 480) - mode_ptr->type |= DRM_MODE_TYPE_PREFERRED; - else if (tv_mode->nbr_end > 480) { - if (tv_mode->progressive == true && tv_mode->nbr_end < 720) { - if (mode_ptr->vdisplay == 720) - mode_ptr->type |= DRM_MODE_TYPE_PREFERRED; - } else if (mode_ptr->vdisplay == 1080) - mode_ptr->type |= DRM_MODE_TYPE_PREFERRED; - } + mode->type = DRM_MODE_TYPE_DRIVER; + + if (intel_tv_is_preferred_mode(mode, tv_mode)) + mode->type |= DRM_MODE_TYPE_PREFERRED; } static int intel_tv_get_modes(struct drm_connector *connector) { - struct drm_display_mode *mode_ptr; + struct drm_i915_private *dev_priv = to_i915(connector->dev); const struct tv_mode *tv_mode = intel_tv_mode_find(connector->state); - int j, count = 0; - u64 tmp; + int i, count = 0; - for (j = 0; j < ARRAY_SIZE(input_res_table); - j++) { - const struct input_res *input = &input_res_table[j]; - unsigned int hactive_s = input->w; - unsigned int vactive_s = input->h; + for (i = 0; i < ARRAY_SIZE(input_res_table); i++) { + const struct input_res *input = &input_res_table[i]; + struct drm_display_mode *mode; - if (tv_mode->max_srcw && input->w > tv_mode->max_srcw) + if (input->w > 1024 && + !tv_mode->progressive && + !tv_mode->component_only) continue; - if (input->w > 1024 && (!tv_mode->progressive - && !tv_mode->component_only)) + /* no vertical scaling with wide sources on gen3 */ + if (IS_GEN(dev_priv, 3) && input->w > 1024 && + input->h > intel_tv_mode_vdisplay(tv_mode)) continue; - mode_ptr = drm_mode_create(connector->dev); - if (!mode_ptr) + mode = drm_mode_create(connector->dev); + if (!mode) continue; - strlcpy(mode_ptr->name, input->name, DRM_DISPLAY_MODE_LEN); - - mode_ptr->hdisplay = hactive_s; - mode_ptr->hsync_start = hactive_s + 1; - mode_ptr->hsync_end = hactive_s + 64; - if (mode_ptr->hsync_end <= mode_ptr->hsync_start) - mode_ptr->hsync_end = mode_ptr->hsync_start + 1; - mode_ptr->htotal = hactive_s + 96; - - mode_ptr->vdisplay = vactive_s; - mode_ptr->vsync_start = vactive_s + 1; - mode_ptr->vsync_end = vactive_s + 32; - if (mode_ptr->vsync_end <= mode_ptr->vsync_start) - mode_ptr->vsync_end = mode_ptr->vsync_start + 1; - mode_ptr->vtotal = vactive_s + 33; - - tmp = mul_u32_u32(tv_mode->refresh, mode_ptr->vtotal); - tmp *= mode_ptr->htotal; - tmp = div_u64(tmp, 1000000); - mode_ptr->clock = (int) tmp; - - mode_ptr->type = DRM_MODE_TYPE_DRIVER; - intel_tv_choose_preferred_modes(tv_mode, mode_ptr); - drm_mode_probed_add(connector, mode_ptr); + + /* + * We take the TV mode and scale it to look + * like it had the expected h/vdisplay. This + * provides the most information to userspace + * about the actual timings of the mode. We + * do ignore the margins though. + */ + intel_tv_mode_to_mode(mode, tv_mode); + if (count == 0) { + DRM_DEBUG_KMS("TV mode:\n"); + drm_mode_debug_printmodeline(mode); + } + intel_tv_scale_mode_horiz(mode, input->w, 0, 0); + intel_tv_scale_mode_vert(mode, input->h, 0, 0); + intel_tv_set_mode_type(mode, tv_mode); + + drm_mode_set_name(mode); + + drm_mode_probed_add(connector, mode); count++; } return count; } -static void -intel_tv_destroy(struct drm_connector *connector) -{ - drm_connector_cleanup(connector); - kfree(connector); -} - static const struct drm_connector_funcs intel_tv_connector_funcs = { .late_register = intel_connector_register, .early_unregister = intel_connector_unregister, - .destroy = intel_tv_destroy, + .destroy = intel_connector_destroy, .fill_modes = drm_helper_probe_single_connector_modes, .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, - .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, + .atomic_duplicate_state = intel_tv_connector_duplicate_state, }; static int intel_tv_atomic_check(struct drm_connector *connector, @@ -1537,11 +1960,15 @@ intel_tv_init(struct drm_i915_private *dev_priv) connector->doublescan_allowed = false; /* Create TV properties then attach current values */ - for (i = 0; i < ARRAY_SIZE(tv_modes); i++) + for (i = 0; i < ARRAY_SIZE(tv_modes); i++) { + /* 1080p50/1080p60 not supported on gen3 */ + if (IS_GEN(dev_priv, 3) && + tv_modes[i].oversample == 1) + break; + tv_format_names[i] = tv_modes[i].name; - drm_mode_create_tv_properties(dev, - ARRAY_SIZE(tv_modes), - tv_format_names); + } + drm_mode_create_tv_properties(dev, i, tv_format_names); drm_object_attach_property(&connector->base, dev->mode_config.tv_mode_property, state->tv.mode); diff --git a/drivers/gpu/drm/i915/intel_uc.c b/drivers/gpu/drm/i915/intel_uc.c index b1b3e81b6e24..e711eb3268bc 100644 --- a/drivers/gpu/drm/i915/intel_uc.c +++ b/drivers/gpu/drm/i915/intel_uc.c @@ -26,6 +26,7 @@ #include "intel_guc_submission.h" #include "intel_guc.h" #include "i915_drv.h" +#include "i915_reset.h" static void guc_free_load_err_log(struct intel_guc *guc); @@ -71,7 +72,7 @@ static int __get_default_guc_log_level(struct drm_i915_private *i915) { int guc_log_level; - if (!HAS_GUC(i915) || !intel_uc_is_using_guc()) + if (!HAS_GUC(i915) || !intel_uc_is_using_guc(i915)) guc_log_level = GUC_LOG_LEVEL_DISABLED; else if (IS_ENABLED(CONFIG_DRM_I915_DEBUG) || IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) @@ -112,11 +113,11 @@ static void sanitize_options_early(struct drm_i915_private *i915) DRM_DEBUG_DRIVER("enable_guc=%d (submission:%s huc:%s)\n", i915_modparams.enable_guc, - yesno(intel_uc_is_using_guc_submission()), - yesno(intel_uc_is_using_huc())); + yesno(intel_uc_is_using_guc_submission(i915)), + yesno(intel_uc_is_using_huc(i915))); /* Verify GuC firmware availability */ - if (intel_uc_is_using_guc() && !intel_uc_fw_is_selected(guc_fw)) { + if (intel_uc_is_using_guc(i915) && !intel_uc_fw_is_selected(guc_fw)) { DRM_WARN("Incompatible option detected: %s=%d, %s!\n", "enable_guc", i915_modparams.enable_guc, !HAS_GUC(i915) ? "no GuC hardware" : @@ -124,7 +125,7 @@ static void sanitize_options_early(struct drm_i915_private *i915) } /* Verify HuC firmware availability */ - if (intel_uc_is_using_huc() && !intel_uc_fw_is_selected(huc_fw)) { + if (intel_uc_is_using_huc(i915) && !intel_uc_fw_is_selected(huc_fw)) { DRM_WARN("Incompatible option detected: %s=%d, %s!\n", "enable_guc", i915_modparams.enable_guc, !HAS_HUC(i915) ? "no HuC hardware" : @@ -136,7 +137,7 @@ static void sanitize_options_early(struct drm_i915_private *i915) i915_modparams.guc_log_level = __get_default_guc_log_level(i915); - if (i915_modparams.guc_log_level > 0 && !intel_uc_is_using_guc()) { + if (i915_modparams.guc_log_level > 0 && !intel_uc_is_using_guc(i915)) { DRM_WARN("Incompatible option detected: %s=%d, %s!\n", "guc_log_level", i915_modparams.guc_log_level, !HAS_GUC(i915) ? "no GuC hardware" : @@ -354,7 +355,7 @@ int intel_uc_init_hw(struct drm_i915_private *i915) /* WaEnableuKernelHeaderValidFix:skl */ /* WaEnableGuCBootHashCheckNotSet:skl,bxt,kbl */ - if (IS_GEN9(i915)) + if (IS_GEN(i915, 9)) attempts = 3; else attempts = 1; @@ -376,7 +377,7 @@ int intel_uc_init_hw(struct drm_i915_private *i915) intel_guc_init_params(guc); ret = intel_guc_fw_upload(guc); - if (ret == 0 || ret != -EAGAIN) + if (ret == 0 || ret != -ETIMEDOUT) break; DRM_DEBUG_DRIVER("GuC fw load failed: %d; will reset and " diff --git a/drivers/gpu/drm/i915/intel_uc.h b/drivers/gpu/drm/i915/intel_uc.h index 25d73ada74ae..870faf9011b9 100644 --- a/drivers/gpu/drm/i915/intel_uc.h +++ b/drivers/gpu/drm/i915/intel_uc.h @@ -41,19 +41,19 @@ void intel_uc_fini(struct drm_i915_private *dev_priv); int intel_uc_suspend(struct drm_i915_private *dev_priv); int intel_uc_resume(struct drm_i915_private *dev_priv); -static inline bool intel_uc_is_using_guc(void) +static inline bool intel_uc_is_using_guc(struct drm_i915_private *i915) { GEM_BUG_ON(i915_modparams.enable_guc < 0); return i915_modparams.enable_guc > 0; } -static inline bool intel_uc_is_using_guc_submission(void) +static inline bool intel_uc_is_using_guc_submission(struct drm_i915_private *i915) { GEM_BUG_ON(i915_modparams.enable_guc < 0); return i915_modparams.enable_guc & ENABLE_GUC_SUBMISSION; } -static inline bool intel_uc_is_using_huc(void) +static inline bool intel_uc_is_using_huc(struct drm_i915_private *i915) { GEM_BUG_ON(i915_modparams.enable_guc < 0); return i915_modparams.enable_guc & ENABLE_GUC_LOAD_HUC; diff --git a/drivers/gpu/drm/i915/intel_uc_fw.c b/drivers/gpu/drm/i915/intel_uc_fw.c index fd496416087c..becf05ebae4d 100644 --- a/drivers/gpu/drm/i915/intel_uc_fw.c +++ b/drivers/gpu/drm/i915/intel_uc_fw.c @@ -46,12 +46,17 @@ void intel_uc_fw_fetch(struct drm_i915_private *dev_priv, size_t size; int err; + if (!uc_fw->path) { + dev_info(dev_priv->drm.dev, + "%s: No firmware was defined for %s!\n", + intel_uc_fw_type_repr(uc_fw->type), + intel_platform_name(INTEL_INFO(dev_priv)->platform)); + return; + } + DRM_DEBUG_DRIVER("%s fw fetch %s\n", intel_uc_fw_type_repr(uc_fw->type), uc_fw->path); - if (!uc_fw->path) - return; - uc_fw->fetch_status = INTEL_UC_FIRMWARE_PENDING; DRM_DEBUG_DRIVER("%s fw fetch %s\n", intel_uc_fw_type_repr(uc_fw->type), diff --git a/drivers/gpu/drm/i915/intel_uc_fw.h b/drivers/gpu/drm/i915/intel_uc_fw.h index 87910aa83267..0e3bd580e267 100644 --- a/drivers/gpu/drm/i915/intel_uc_fw.h +++ b/drivers/gpu/drm/i915/intel_uc_fw.h @@ -115,9 +115,14 @@ static inline bool intel_uc_fw_is_selected(struct intel_uc_fw *uc_fw) return uc_fw->path != NULL; } +static inline bool intel_uc_fw_is_loaded(struct intel_uc_fw *uc_fw) +{ + return uc_fw->load_status == INTEL_UC_FIRMWARE_SUCCESS; +} + static inline void intel_uc_fw_sanitize(struct intel_uc_fw *uc_fw) { - if (uc_fw->load_status == INTEL_UC_FIRMWARE_SUCCESS) + if (intel_uc_fw_is_loaded(uc_fw)) uc_fw->load_status = INTEL_UC_FIRMWARE_PENDING; } diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c index 3ad302c66254..75646a1e0051 100644 --- a/drivers/gpu/drm/i915/intel_uncore.c +++ b/drivers/gpu/drm/i915/intel_uncore.c @@ -528,7 +528,7 @@ check_for_unclaimed_mmio(struct drm_i915_private *dev_priv) if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) ret |= vlv_check_for_unclaimed_mmio(dev_priv); - if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) + if (IS_GEN_RANGE(dev_priv, 6, 7)) ret |= gen6_check_for_fifo_debug(dev_priv); return ret; @@ -556,7 +556,7 @@ static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv, dev_priv->uncore.funcs.force_wake_get(dev_priv, restore_forcewake); - if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) + if (IS_GEN_RANGE(dev_priv, 6, 7)) dev_priv->uncore.fifo_count = fifo_free_entries(dev_priv); spin_unlock_irq(&dev_priv->uncore.lock); @@ -1398,7 +1398,7 @@ static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv) if (INTEL_GEN(dev_priv) <= 5 || intel_vgpu_active(dev_priv)) return; - if (IS_GEN6(dev_priv)) { + if (IS_GEN(dev_priv, 6)) { dev_priv->uncore.fw_reset = 0; dev_priv->uncore.fw_set = FORCEWAKE_KERNEL; dev_priv->uncore.fw_clear = 0; @@ -1437,7 +1437,7 @@ static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv) FORCEWAKE_MEDIA_VEBOX_GEN11(i), FORCEWAKE_ACK_MEDIA_VEBOX_GEN11(i)); } - } else if (IS_GEN9(dev_priv) || IS_GEN10(dev_priv)) { + } else if (IS_GEN_RANGE(dev_priv, 9, 10)) { dev_priv->uncore.funcs.force_wake_get = fw_domains_get_with_fallback; dev_priv->uncore.funcs.force_wake_put = fw_domains_put; @@ -1503,7 +1503,7 @@ static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv) fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, FORCEWAKE, FORCEWAKE_ACK); } - } else if (IS_GEN6(dev_priv)) { + } else if (IS_GEN(dev_priv, 6)) { dev_priv->uncore.funcs.force_wake_get = fw_domains_get_with_thread_status; dev_priv->uncore.funcs.force_wake_put = fw_domains_put; @@ -1567,13 +1567,13 @@ void intel_uncore_init(struct drm_i915_private *dev_priv) dev_priv->uncore.pmic_bus_access_nb.notifier_call = i915_pmic_bus_access_notifier; - if (IS_GEN(dev_priv, 2, 4) || intel_vgpu_active(dev_priv)) { + if (IS_GEN_RANGE(dev_priv, 2, 4) || intel_vgpu_active(dev_priv)) { ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen2); ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen2); - } else if (IS_GEN5(dev_priv)) { + } else if (IS_GEN(dev_priv, 5)) { ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen5); ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen5); - } else if (IS_GEN(dev_priv, 6, 7)) { + } else if (IS_GEN_RANGE(dev_priv, 6, 7)) { ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen6); if (IS_VALLEYVIEW(dev_priv)) { @@ -1582,7 +1582,7 @@ void intel_uncore_init(struct drm_i915_private *dev_priv) } else { ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen6); } - } else if (IS_GEN8(dev_priv)) { + } else if (IS_GEN(dev_priv, 8)) { if (IS_CHERRYVIEW(dev_priv)) { ASSIGN_FW_DOMAINS_TABLE(__chv_fw_ranges); ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, fwtable); @@ -1592,7 +1592,7 @@ void intel_uncore_init(struct drm_i915_private *dev_priv) ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen8); ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen6); } - } else if (IS_GEN(dev_priv, 9, 10)) { + } else if (IS_GEN_RANGE(dev_priv, 9, 10)) { ASSIGN_FW_DOMAINS_TABLE(__gen9_fw_ranges); ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, fwtable); ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable); @@ -1670,6 +1670,7 @@ int i915_reg_read_ioctl(struct drm_device *dev, struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_reg_read *reg = data; struct reg_whitelist const *entry; + intel_wakeref_t wakeref; unsigned int flags; int remain; int ret = 0; @@ -1695,286 +1696,25 @@ int i915_reg_read_ioctl(struct drm_device *dev, flags = reg->offset & (entry->size - 1); - intel_runtime_pm_get(dev_priv); - if (entry->size == 8 && flags == I915_REG_READ_8B_WA) - reg->val = I915_READ64_2x32(entry->offset_ldw, - entry->offset_udw); - else if (entry->size == 8 && flags == 0) - reg->val = I915_READ64(entry->offset_ldw); - else if (entry->size == 4 && flags == 0) - reg->val = I915_READ(entry->offset_ldw); - else if (entry->size == 2 && flags == 0) - reg->val = I915_READ16(entry->offset_ldw); - else if (entry->size == 1 && flags == 0) - reg->val = I915_READ8(entry->offset_ldw); - else - ret = -EINVAL; - intel_runtime_pm_put(dev_priv); - - return ret; -} - -static void gen3_stop_engine(struct intel_engine_cs *engine) -{ - struct drm_i915_private *dev_priv = engine->i915; - const u32 base = engine->mmio_base; - - if (intel_engine_stop_cs(engine)) - DRM_DEBUG_DRIVER("%s: timed out on STOP_RING\n", engine->name); - - I915_WRITE_FW(RING_HEAD(base), I915_READ_FW(RING_TAIL(base))); - POSTING_READ_FW(RING_HEAD(base)); /* paranoia */ - - I915_WRITE_FW(RING_HEAD(base), 0); - I915_WRITE_FW(RING_TAIL(base), 0); - POSTING_READ_FW(RING_TAIL(base)); - - /* The ring must be empty before it is disabled */ - I915_WRITE_FW(RING_CTL(base), 0); - - /* Check acts as a post */ - if (I915_READ_FW(RING_HEAD(base)) != 0) - DRM_DEBUG_DRIVER("%s: ring head not parked\n", - engine->name); -} - -static void i915_stop_engines(struct drm_i915_private *dev_priv, - unsigned int engine_mask) -{ - struct intel_engine_cs *engine; - enum intel_engine_id id; - - if (INTEL_GEN(dev_priv) < 3) - return; - - for_each_engine_masked(engine, dev_priv, engine_mask, id) - gen3_stop_engine(engine); -} - -static bool i915_in_reset(struct pci_dev *pdev) -{ - u8 gdrst; - - pci_read_config_byte(pdev, I915_GDRST, &gdrst); - return gdrst & GRDOM_RESET_STATUS; -} - -static int i915_do_reset(struct drm_i915_private *dev_priv, - unsigned int engine_mask, - unsigned int retry) -{ - struct pci_dev *pdev = dev_priv->drm.pdev; - int err; - - /* Assert reset for at least 20 usec, and wait for acknowledgement. */ - pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE); - usleep_range(50, 200); - err = wait_for(i915_in_reset(pdev), 500); - - /* Clear the reset request. */ - pci_write_config_byte(pdev, I915_GDRST, 0); - usleep_range(50, 200); - if (!err) - err = wait_for(!i915_in_reset(pdev), 500); - - return err; -} - -static bool g4x_reset_complete(struct pci_dev *pdev) -{ - u8 gdrst; - - pci_read_config_byte(pdev, I915_GDRST, &gdrst); - return (gdrst & GRDOM_RESET_ENABLE) == 0; -} - -static int g33_do_reset(struct drm_i915_private *dev_priv, - unsigned int engine_mask, - unsigned int retry) -{ - struct pci_dev *pdev = dev_priv->drm.pdev; - - pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE); - return wait_for(g4x_reset_complete(pdev), 500); -} - -static int g4x_do_reset(struct drm_i915_private *dev_priv, - unsigned int engine_mask, - unsigned int retry) -{ - struct pci_dev *pdev = dev_priv->drm.pdev; - int ret; - - /* WaVcpClkGateDisableForMediaReset:ctg,elk */ - I915_WRITE(VDECCLK_GATE_D, - I915_READ(VDECCLK_GATE_D) | VCP_UNIT_CLOCK_GATE_DISABLE); - POSTING_READ(VDECCLK_GATE_D); - - pci_write_config_byte(pdev, I915_GDRST, - GRDOM_MEDIA | GRDOM_RESET_ENABLE); - ret = wait_for(g4x_reset_complete(pdev), 500); - if (ret) { - DRM_DEBUG_DRIVER("Wait for media reset failed\n"); - goto out; - } - - pci_write_config_byte(pdev, I915_GDRST, - GRDOM_RENDER | GRDOM_RESET_ENABLE); - ret = wait_for(g4x_reset_complete(pdev), 500); - if (ret) { - DRM_DEBUG_DRIVER("Wait for render reset failed\n"); - goto out; - } - -out: - pci_write_config_byte(pdev, I915_GDRST, 0); - - I915_WRITE(VDECCLK_GATE_D, - I915_READ(VDECCLK_GATE_D) & ~VCP_UNIT_CLOCK_GATE_DISABLE); - POSTING_READ(VDECCLK_GATE_D); - - return ret; -} - -static int ironlake_do_reset(struct drm_i915_private *dev_priv, - unsigned int engine_mask, - unsigned int retry) -{ - int ret; - - I915_WRITE(ILK_GDSR, ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE); - ret = intel_wait_for_register(dev_priv, - ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0, - 500); - if (ret) { - DRM_DEBUG_DRIVER("Wait for render reset failed\n"); - goto out; - } - - I915_WRITE(ILK_GDSR, ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE); - ret = intel_wait_for_register(dev_priv, - ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0, - 500); - if (ret) { - DRM_DEBUG_DRIVER("Wait for media reset failed\n"); - goto out; + with_intel_runtime_pm(dev_priv, wakeref) { + if (entry->size == 8 && flags == I915_REG_READ_8B_WA) + reg->val = I915_READ64_2x32(entry->offset_ldw, + entry->offset_udw); + else if (entry->size == 8 && flags == 0) + reg->val = I915_READ64(entry->offset_ldw); + else if (entry->size == 4 && flags == 0) + reg->val = I915_READ(entry->offset_ldw); + else if (entry->size == 2 && flags == 0) + reg->val = I915_READ16(entry->offset_ldw); + else if (entry->size == 1 && flags == 0) + reg->val = I915_READ8(entry->offset_ldw); + else + ret = -EINVAL; } -out: - I915_WRITE(ILK_GDSR, 0); - POSTING_READ(ILK_GDSR); return ret; } -/* Reset the hardware domains (GENX_GRDOM_*) specified by mask */ -static int gen6_hw_domain_reset(struct drm_i915_private *dev_priv, - u32 hw_domain_mask) -{ - int err; - - /* GEN6_GDRST is not in the gt power well, no need to check - * for fifo space for the write or forcewake the chip for - * the read - */ - __raw_i915_write32(dev_priv, GEN6_GDRST, hw_domain_mask); - - /* Wait for the device to ack the reset requests */ - err = __intel_wait_for_register_fw(dev_priv, - GEN6_GDRST, hw_domain_mask, 0, - 500, 0, - NULL); - if (err) - DRM_DEBUG_DRIVER("Wait for 0x%08x engines reset failed\n", - hw_domain_mask); - - return err; -} - -/** - * gen6_reset_engines - reset individual engines - * @dev_priv: i915 device - * @engine_mask: mask of intel_ring_flag() engines or ALL_ENGINES for full reset - * @retry: the count of of previous attempts to reset. - * - * This function will reset the individual engines that are set in engine_mask. - * If you provide ALL_ENGINES as mask, full global domain reset will be issued. - * - * Note: It is responsibility of the caller to handle the difference between - * asking full domain reset versus reset for all available individual engines. - * - * Returns 0 on success, nonzero on error. - */ -static int gen6_reset_engines(struct drm_i915_private *dev_priv, - unsigned int engine_mask, - unsigned int retry) -{ - struct intel_engine_cs *engine; - const u32 hw_engine_mask[I915_NUM_ENGINES] = { - [RCS] = GEN6_GRDOM_RENDER, - [BCS] = GEN6_GRDOM_BLT, - [VCS] = GEN6_GRDOM_MEDIA, - [VCS2] = GEN8_GRDOM_MEDIA2, - [VECS] = GEN6_GRDOM_VECS, - }; - u32 hw_mask; - - if (engine_mask == ALL_ENGINES) { - hw_mask = GEN6_GRDOM_FULL; - } else { - unsigned int tmp; - - hw_mask = 0; - for_each_engine_masked(engine, dev_priv, engine_mask, tmp) - hw_mask |= hw_engine_mask[engine->id]; - } - - return gen6_hw_domain_reset(dev_priv, hw_mask); -} - -/** - * gen11_reset_engines - reset individual engines - * @dev_priv: i915 device - * @engine_mask: mask of intel_ring_flag() engines or ALL_ENGINES for full reset - * - * This function will reset the individual engines that are set in engine_mask. - * If you provide ALL_ENGINES as mask, full global domain reset will be issued. - * - * Note: It is responsibility of the caller to handle the difference between - * asking full domain reset versus reset for all available individual engines. - * - * Returns 0 on success, nonzero on error. - */ -static int gen11_reset_engines(struct drm_i915_private *dev_priv, - unsigned int engine_mask) -{ - struct intel_engine_cs *engine; - const u32 hw_engine_mask[I915_NUM_ENGINES] = { - [RCS] = GEN11_GRDOM_RENDER, - [BCS] = GEN11_GRDOM_BLT, - [VCS] = GEN11_GRDOM_MEDIA, - [VCS2] = GEN11_GRDOM_MEDIA2, - [VCS3] = GEN11_GRDOM_MEDIA3, - [VCS4] = GEN11_GRDOM_MEDIA4, - [VECS] = GEN11_GRDOM_VECS, - [VECS2] = GEN11_GRDOM_VECS2, - }; - u32 hw_mask; - - BUILD_BUG_ON(VECS2 + 1 != I915_NUM_ENGINES); - - if (engine_mask == ALL_ENGINES) { - hw_mask = GEN11_GRDOM_FULL; - } else { - unsigned int tmp; - - hw_mask = 0; - for_each_engine_masked(engine, dev_priv, engine_mask, tmp) - hw_mask |= hw_engine_mask[engine->id]; - } - - return gen6_hw_domain_reset(dev_priv, hw_mask); -} - /** * __intel_wait_for_register_fw - wait until register matches expected state * @dev_priv: the i915 device @@ -2079,202 +1819,15 @@ int __intel_wait_for_register(struct drm_i915_private *dev_priv, (reg_value & mask) == value, slow_timeout_ms * 1000, 10, 1000); + /* just trace the final value */ + trace_i915_reg_rw(false, reg, reg_value, sizeof(reg_value), true); + if (out_value) *out_value = reg_value; return ret; } -static int gen8_engine_reset_prepare(struct intel_engine_cs *engine) -{ - struct drm_i915_private *dev_priv = engine->i915; - int ret; - - I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base), - _MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET)); - - ret = __intel_wait_for_register_fw(dev_priv, - RING_RESET_CTL(engine->mmio_base), - RESET_CTL_READY_TO_RESET, - RESET_CTL_READY_TO_RESET, - 700, 0, - NULL); - if (ret) - DRM_ERROR("%s: reset request timeout\n", engine->name); - - return ret; -} - -static void gen8_engine_reset_cancel(struct intel_engine_cs *engine) -{ - struct drm_i915_private *dev_priv = engine->i915; - - I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base), - _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET)); -} - -static int reset_engines(struct drm_i915_private *i915, - unsigned int engine_mask, - unsigned int retry) -{ - if (INTEL_GEN(i915) >= 11) - return gen11_reset_engines(i915, engine_mask); - else - return gen6_reset_engines(i915, engine_mask, retry); -} - -static int gen8_reset_engines(struct drm_i915_private *dev_priv, - unsigned int engine_mask, - unsigned int retry) -{ - struct intel_engine_cs *engine; - const bool reset_non_ready = retry >= 1; - unsigned int tmp; - int ret; - - for_each_engine_masked(engine, dev_priv, engine_mask, tmp) { - ret = gen8_engine_reset_prepare(engine); - if (ret && !reset_non_ready) - goto skip_reset; - - /* - * If this is not the first failed attempt to prepare, - * we decide to proceed anyway. - * - * By doing so we risk context corruption and with - * some gens (kbl), possible system hang if reset - * happens during active bb execution. - * - * We rather take context corruption instead of - * failed reset with a wedged driver/gpu. And - * active bb execution case should be covered by - * i915_stop_engines we have before the reset. - */ - } - - ret = reset_engines(dev_priv, engine_mask, retry); - -skip_reset: - for_each_engine_masked(engine, dev_priv, engine_mask, tmp) - gen8_engine_reset_cancel(engine); - - return ret; -} - -typedef int (*reset_func)(struct drm_i915_private *, - unsigned int engine_mask, unsigned int retry); - -static reset_func intel_get_gpu_reset(struct drm_i915_private *dev_priv) -{ - if (!i915_modparams.reset) - return NULL; - - if (INTEL_GEN(dev_priv) >= 8) - return gen8_reset_engines; - else if (INTEL_GEN(dev_priv) >= 6) - return gen6_reset_engines; - else if (IS_GEN5(dev_priv)) - return ironlake_do_reset; - else if (IS_G4X(dev_priv)) - return g4x_do_reset; - else if (IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) - return g33_do_reset; - else if (INTEL_GEN(dev_priv) >= 3) - return i915_do_reset; - else - return NULL; -} - -int intel_gpu_reset(struct drm_i915_private *dev_priv, - const unsigned int engine_mask) -{ - reset_func reset = intel_get_gpu_reset(dev_priv); - unsigned int retry; - int ret; - - GEM_BUG_ON(!engine_mask); - - /* - * We want to perform per-engine reset from atomic context (e.g. - * softirq), which imposes the constraint that we cannot sleep. - * However, experience suggests that spending a bit of time waiting - * for a reset helps in various cases, so for a full-device reset - * we apply the opposite rule and wait if we want to. As we should - * always follow up a failed per-engine reset with a full device reset, - * being a little faster, stricter and more error prone for the - * atomic case seems an acceptable compromise. - * - * Unfortunately this leads to a bimodal routine, when the goal was - * to have a single reset function that worked for resetting any - * number of engines simultaneously. - */ - might_sleep_if(engine_mask == ALL_ENGINES); - - /* - * If the power well sleeps during the reset, the reset - * request may be dropped and never completes (causing -EIO). - */ - intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); - for (retry = 0; retry < 3; retry++) { - - /* - * We stop engines, otherwise we might get failed reset and a - * dead gpu (on elk). Also as modern gpu as kbl can suffer - * from system hang if batchbuffer is progressing when - * the reset is issued, regardless of READY_TO_RESET ack. - * Thus assume it is best to stop engines on all gens - * where we have a gpu reset. - * - * WaKBLVECSSemaphoreWaitPoll:kbl (on ALL_ENGINES) - * - * WaMediaResetMainRingCleanup:ctg,elk (presumably) - * - * FIXME: Wa for more modern gens needs to be validated - */ - i915_stop_engines(dev_priv, engine_mask); - - ret = -ENODEV; - if (reset) { - ret = reset(dev_priv, engine_mask, retry); - GEM_TRACE("engine_mask=%x, ret=%d, retry=%d\n", - engine_mask, ret, retry); - } - if (ret != -ETIMEDOUT || engine_mask != ALL_ENGINES) - break; - - cond_resched(); - } - intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); - - return ret; -} - -bool intel_has_gpu_reset(struct drm_i915_private *dev_priv) -{ - return intel_get_gpu_reset(dev_priv) != NULL; -} - -bool intel_has_reset_engine(struct drm_i915_private *dev_priv) -{ - return (dev_priv->info.has_reset_engine && - i915_modparams.reset >= 2); -} - -int intel_reset_guc(struct drm_i915_private *dev_priv) -{ - u32 guc_domain = INTEL_GEN(dev_priv) >= 11 ? GEN11_GRDOM_GUC : - GEN9_GRDOM_GUC; - int ret; - - GEM_BUG_ON(!HAS_GUC(dev_priv)); - - intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); - ret = gen6_hw_domain_reset(dev_priv, guc_domain); - intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); - - return ret; -} - bool intel_uncore_unclaimed_mmio(struct drm_i915_private *dev_priv) { return check_for_unclaimed_mmio(dev_priv); @@ -2321,7 +1874,7 @@ intel_uncore_forcewake_for_read(struct drm_i915_private *dev_priv, } else if (INTEL_GEN(dev_priv) >= 6) { fw_domains = __gen6_reg_read_fw_domains(offset); } else { - WARN_ON(!IS_GEN(dev_priv, 2, 5)); + WARN_ON(!IS_GEN_RANGE(dev_priv, 2, 5)); fw_domains = 0; } @@ -2341,12 +1894,12 @@ intel_uncore_forcewake_for_write(struct drm_i915_private *dev_priv, fw_domains = __gen11_fwtable_reg_write_fw_domains(offset); } else if (HAS_FWTABLE(dev_priv) && !IS_VALLEYVIEW(dev_priv)) { fw_domains = __fwtable_reg_write_fw_domains(offset); - } else if (IS_GEN8(dev_priv)) { + } else if (IS_GEN(dev_priv, 8)) { fw_domains = __gen8_reg_write_fw_domains(offset); - } else if (IS_GEN(dev_priv, 6, 7)) { + } else if (IS_GEN_RANGE(dev_priv, 6, 7)) { fw_domains = FORCEWAKE_RENDER; } else { - WARN_ON(!IS_GEN(dev_priv, 2, 5)); + WARN_ON(!IS_GEN_RANGE(dev_priv, 2, 5)); fw_domains = 0; } diff --git a/drivers/gpu/drm/i915/intel_vbt_defs.h b/drivers/gpu/drm/i915/intel_vbt_defs.h index bba98cf83cbd..bf3662ad5fed 100644 --- a/drivers/gpu/drm/i915/intel_vbt_defs.h +++ b/drivers/gpu/drm/i915/intel_vbt_defs.h @@ -326,6 +326,13 @@ enum vbt_gmbus_ddi { ICL_DDC_BUS_PORT_4, }; +#define DP_AUX_A 0x40 +#define DP_AUX_B 0x10 +#define DP_AUX_C 0x20 +#define DP_AUX_D 0x30 +#define DP_AUX_E 0x50 +#define DP_AUX_F 0x60 + #define VBT_DP_MAX_LINK_RATE_HBR3 0 #define VBT_DP_MAX_LINK_RATE_HBR2 1 #define VBT_DP_MAX_LINK_RATE_HBR 2 diff --git a/drivers/gpu/drm/i915/intel_vdsc.c b/drivers/gpu/drm/i915/intel_vdsc.c new file mode 100644 index 000000000000..23abf03736e7 --- /dev/null +++ b/drivers/gpu/drm/i915/intel_vdsc.c @@ -0,0 +1,1087 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2018 Intel Corporation + * + * Author: Gaurav K Singh <gaurav.k.singh@intel.com> + * Manasi Navare <manasi.d.navare@intel.com> + */ + +#include <drm/i915_drm.h> +#include "i915_drv.h" +#include "intel_drv.h" + +enum ROW_INDEX_BPP { + ROW_INDEX_6BPP = 0, + ROW_INDEX_8BPP, + ROW_INDEX_10BPP, + ROW_INDEX_12BPP, + ROW_INDEX_15BPP, + MAX_ROW_INDEX +}; + +enum COLUMN_INDEX_BPC { + COLUMN_INDEX_8BPC = 0, + COLUMN_INDEX_10BPC, + COLUMN_INDEX_12BPC, + COLUMN_INDEX_14BPC, + COLUMN_INDEX_16BPC, + MAX_COLUMN_INDEX +}; + +#define DSC_SUPPORTED_VERSION_MIN 1 + +/* From DSC_v1.11 spec, rc_parameter_Set syntax element typically constant */ +static u16 rc_buf_thresh[] = { + 896, 1792, 2688, 3584, 4480, 5376, 6272, 6720, 7168, 7616, + 7744, 7872, 8000, 8064 +}; + +struct rc_parameters { + u16 initial_xmit_delay; + u8 first_line_bpg_offset; + u16 initial_offset; + u8 flatness_min_qp; + u8 flatness_max_qp; + u8 rc_quant_incr_limit0; + u8 rc_quant_incr_limit1; + struct drm_dsc_rc_range_parameters rc_range_params[DSC_NUM_BUF_RANGES]; +}; + +/* + * Selected Rate Control Related Parameter Recommended Values + * from DSC_v1.11 spec & C Model release: DSC_model_20161212 + */ +static struct rc_parameters rc_params[][MAX_COLUMN_INDEX] = { +{ + /* 6BPP/8BPC */ + { 768, 15, 6144, 3, 13, 11, 11, { + { 0, 4, 0 }, { 1, 6, -2 }, { 3, 8, -2 }, { 4, 8, -4 }, + { 5, 9, -6 }, { 5, 9, -6 }, { 6, 9, -6 }, { 6, 10, -8 }, + { 7, 11, -8 }, { 8, 12, -10 }, { 9, 12, -10 }, { 10, 12, -12 }, + { 10, 12, -12 }, { 11, 12, -12 }, { 13, 14, -12 } + } + }, + /* 6BPP/10BPC */ + { 768, 15, 6144, 7, 17, 15, 15, { + { 0, 8, 0 }, { 3, 10, -2 }, { 7, 12, -2 }, { 8, 12, -4 }, + { 9, 13, -6 }, { 9, 13, -6 }, { 10, 13, -6 }, { 10, 14, -8 }, + { 11, 15, -8 }, { 12, 16, -10 }, { 13, 16, -10 }, + { 14, 16, -12 }, { 14, 16, -12 }, { 15, 16, -12 }, + { 17, 18, -12 } + } + }, + /* 6BPP/12BPC */ + { 768, 15, 6144, 11, 21, 19, 19, { + { 0, 12, 0 }, { 5, 14, -2 }, { 11, 16, -2 }, { 12, 16, -4 }, + { 13, 17, -6 }, { 13, 17, -6 }, { 14, 17, -6 }, { 14, 18, -8 }, + { 15, 19, -8 }, { 16, 20, -10 }, { 17, 20, -10 }, + { 18, 20, -12 }, { 18, 20, -12 }, { 19, 20, -12 }, + { 21, 22, -12 } + } + }, + /* 6BPP/14BPC */ + { 768, 15, 6144, 15, 25, 23, 27, { + { 0, 16, 0 }, { 7, 18, -2 }, { 15, 20, -2 }, { 16, 20, -4 }, + { 17, 21, -6 }, { 17, 21, -6 }, { 18, 21, -6 }, { 18, 22, -8 }, + { 19, 23, -8 }, { 20, 24, -10 }, { 21, 24, -10 }, + { 22, 24, -12 }, { 22, 24, -12 }, { 23, 24, -12 }, + { 25, 26, -12 } + } + }, + /* 6BPP/16BPC */ + { 768, 15, 6144, 19, 29, 27, 27, { + { 0, 20, 0 }, { 9, 22, -2 }, { 19, 24, -2 }, { 20, 24, -4 }, + { 21, 25, -6 }, { 21, 25, -6 }, { 22, 25, -6 }, { 22, 26, -8 }, + { 23, 27, -8 }, { 24, 28, -10 }, { 25, 28, -10 }, + { 26, 28, -12 }, { 26, 28, -12 }, { 27, 28, -12 }, + { 29, 30, -12 } + } + }, +}, +{ + /* 8BPP/8BPC */ + { 512, 12, 6144, 3, 12, 11, 11, { + { 0, 4, 2 }, { 0, 4, 0 }, { 1, 5, 0 }, { 1, 6, -2 }, + { 3, 7, -4 }, { 3, 7, -6 }, { 3, 7, -8 }, { 3, 8, -8 }, + { 3, 9, -8 }, { 3, 10, -10 }, { 5, 11, -10 }, { 5, 12, -12 }, + { 5, 13, -12 }, { 7, 13, -12 }, { 13, 15, -12 } + } + }, + /* 8BPP/10BPC */ + { 512, 12, 6144, 7, 16, 15, 15, { + { 0, 4, 2 }, { 4, 8, 0 }, { 5, 9, 0 }, { 5, 10, -2 }, + { 7, 11, -4 }, { 7, 11, -6 }, { 7, 11, -8 }, { 7, 12, -8 }, + { 7, 13, -8 }, { 7, 14, -10 }, { 9, 15, -10 }, { 9, 16, -12 }, + { 9, 17, -12 }, { 11, 17, -12 }, { 17, 19, -12 } + } + }, + /* 8BPP/12BPC */ + { 512, 12, 6144, 11, 20, 19, 19, { + { 0, 12, 2 }, { 4, 12, 0 }, { 9, 13, 0 }, { 9, 14, -2 }, + { 11, 15, -4 }, { 11, 15, -6 }, { 11, 15, -8 }, { 11, 16, -8 }, + { 11, 17, -8 }, { 11, 18, -10 }, { 13, 19, -10 }, + { 13, 20, -12 }, { 13, 21, -12 }, { 15, 21, -12 }, + { 21, 23, -12 } + } + }, + /* 8BPP/14BPC */ + { 512, 12, 6144, 15, 24, 23, 23, { + { 0, 12, 0 }, { 5, 13, 0 }, { 11, 15, 0 }, { 12, 17, -2 }, + { 15, 19, -4 }, { 15, 19, -6 }, { 15, 19, -8 }, { 15, 20, -8 }, + { 15, 21, -8 }, { 15, 22, -10 }, { 17, 22, -10 }, + { 17, 23, -12 }, { 17, 23, -12 }, { 21, 24, -12 }, + { 24, 25, -12 } + } + }, + /* 8BPP/16BPC */ + { 512, 12, 6144, 19, 28, 27, 27, { + { 0, 12, 2 }, { 6, 14, 0 }, { 13, 17, 0 }, { 15, 20, -2 }, + { 19, 23, -4 }, { 19, 23, -6 }, { 19, 23, -8 }, { 19, 24, -8 }, + { 19, 25, -8 }, { 19, 26, -10 }, { 21, 26, -10 }, + { 21, 27, -12 }, { 21, 27, -12 }, { 25, 28, -12 }, + { 28, 29, -12 } + } + }, +}, +{ + /* 10BPP/8BPC */ + { 410, 15, 5632, 3, 12, 11, 11, { + { 0, 3, 2 }, { 0, 4, 0 }, { 1, 5, 0 }, { 2, 6, -2 }, + { 3, 7, -4 }, { 3, 7, -6 }, { 3, 7, -8 }, { 3, 8, -8 }, + { 3, 9, -8 }, { 3, 9, -10 }, { 5, 10, -10 }, { 5, 10, -10 }, + { 5, 11, -12 }, { 7, 11, -12 }, { 11, 12, -12 } + } + }, + /* 10BPP/10BPC */ + { 410, 15, 5632, 7, 16, 15, 15, { + { 0, 7, 2 }, { 4, 8, 0 }, { 5, 9, 0 }, { 6, 10, -2 }, + { 7, 11, -4 }, { 7, 11, -6 }, { 7, 11, -8 }, { 7, 12, -8 }, + { 7, 13, -8 }, { 7, 13, -10 }, { 9, 14, -10 }, { 9, 14, -10 }, + { 9, 15, -12 }, { 11, 15, -12 }, { 15, 16, -12 } + } + }, + /* 10BPP/12BPC */ + { 410, 15, 5632, 11, 20, 19, 19, { + { 0, 11, 2 }, { 4, 12, 0 }, { 9, 13, 0 }, { 10, 14, -2 }, + { 11, 15, -4 }, { 11, 15, -6 }, { 11, 15, -8 }, { 11, 16, -8 }, + { 11, 17, -8 }, { 11, 17, -10 }, { 13, 18, -10 }, + { 13, 18, -10 }, { 13, 19, -12 }, { 15, 19, -12 }, + { 19, 20, -12 } + } + }, + /* 10BPP/14BPC */ + { 410, 15, 5632, 15, 24, 23, 23, { + { 0, 11, 2 }, { 5, 13, 0 }, { 11, 15, 0 }, { 13, 18, -2 }, + { 15, 19, -4 }, { 15, 19, -6 }, { 15, 19, -8 }, { 15, 20, -8 }, + { 15, 21, -8 }, { 15, 21, -10 }, { 17, 22, -10 }, + { 17, 22, -10 }, { 17, 23, -12 }, { 19, 23, -12 }, + { 23, 24, -12 } + } + }, + /* 10BPP/16BPC */ + { 410, 15, 5632, 19, 28, 27, 27, { + { 0, 11, 2 }, { 6, 14, 0 }, { 13, 17, 0 }, { 16, 20, -2 }, + { 19, 23, -4 }, { 19, 23, -6 }, { 19, 23, -8 }, { 19, 24, -8 }, + { 19, 25, -8 }, { 19, 25, -10 }, { 21, 26, -10 }, + { 21, 26, -10 }, { 21, 27, -12 }, { 23, 27, -12 }, + { 27, 28, -12 } + } + }, +}, +{ + /* 12BPP/8BPC */ + { 341, 15, 2048, 3, 12, 11, 11, { + { 0, 2, 2 }, { 0, 4, 0 }, { 1, 5, 0 }, { 1, 6, -2 }, + { 3, 7, -4 }, { 3, 7, -6 }, { 3, 7, -8 }, { 3, 8, -8 }, + { 3, 9, -8 }, { 3, 10, -10 }, { 5, 11, -10 }, + { 5, 12, -12 }, { 5, 13, -12 }, { 7, 13, -12 }, { 13, 15, -12 } + } + }, + /* 12BPP/10BPC */ + { 341, 15, 2048, 7, 16, 15, 15, { + { 0, 2, 2 }, { 2, 5, 0 }, { 3, 7, 0 }, { 4, 8, -2 }, + { 6, 9, -4 }, { 7, 10, -6 }, { 7, 11, -8 }, { 7, 12, -8 }, + { 7, 13, -8 }, { 7, 14, -10 }, { 9, 15, -10 }, { 9, 16, -12 }, + { 9, 17, -12 }, { 11, 17, -12 }, { 17, 19, -12 } + } + }, + /* 12BPP/12BPC */ + { 341, 15, 2048, 11, 20, 19, 19, { + { 0, 6, 2 }, { 4, 9, 0 }, { 7, 11, 0 }, { 8, 12, -2 }, + { 10, 13, -4 }, { 11, 14, -6 }, { 11, 15, -8 }, { 11, 16, -8 }, + { 11, 17, -8 }, { 11, 18, -10 }, { 13, 19, -10 }, + { 13, 20, -12 }, { 13, 21, -12 }, { 15, 21, -12 }, + { 21, 23, -12 } + } + }, + /* 12BPP/14BPC */ + { 341, 15, 2048, 15, 24, 23, 23, { + { 0, 6, 2 }, { 7, 10, 0 }, { 9, 13, 0 }, { 11, 16, -2 }, + { 14, 17, -4 }, { 15, 18, -6 }, { 15, 19, -8 }, { 15, 20, -8 }, + { 15, 20, -8 }, { 15, 21, -10 }, { 17, 21, -10 }, + { 17, 21, -12 }, { 17, 21, -12 }, { 19, 22, -12 }, + { 22, 23, -12 } + } + }, + /* 12BPP/16BPC */ + { 341, 15, 2048, 19, 28, 27, 27, { + { 0, 6, 2 }, { 6, 11, 0 }, { 11, 15, 0 }, { 14, 18, -2 }, + { 18, 21, -4 }, { 19, 22, -6 }, { 19, 23, -8 }, { 19, 24, -8 }, + { 19, 24, -8 }, { 19, 25, -10 }, { 21, 25, -10 }, + { 21, 25, -12 }, { 21, 25, -12 }, { 23, 26, -12 }, + { 26, 27, -12 } + } + }, +}, +{ + /* 15BPP/8BPC */ + { 273, 15, 2048, 3, 12, 11, 11, { + { 0, 0, 10 }, { 0, 1, 8 }, { 0, 1, 6 }, { 0, 2, 4 }, + { 1, 2, 2 }, { 1, 3, 0 }, { 1, 3, -2 }, { 2, 4, -4 }, + { 2, 5, -6 }, { 3, 5, -8 }, { 4, 6, -10 }, { 4, 7, -10 }, + { 5, 7, -12 }, { 7, 8, -12 }, { 8, 9, -12 } + } + }, + /* 15BPP/10BPC */ + { 273, 15, 2048, 7, 16, 15, 15, { + { 0, 2, 10 }, { 2, 5, 8 }, { 3, 5, 6 }, { 4, 6, 4 }, + { 5, 6, 2 }, { 5, 7, 0 }, { 5, 7, -2 }, { 6, 8, -4 }, + { 6, 9, -6 }, { 7, 9, -8 }, { 8, 10, -10 }, { 8, 11, -10 }, + { 9, 11, -12 }, { 11, 12, -12 }, { 12, 13, -12 } + } + }, + /* 15BPP/12BPC */ + { 273, 15, 2048, 11, 20, 19, 19, { + { 0, 4, 10 }, { 2, 7, 8 }, { 4, 9, 6 }, { 6, 11, 4 }, + { 9, 11, 2 }, { 9, 11, 0 }, { 9, 12, -2 }, { 10, 12, -4 }, + { 11, 13, -6 }, { 11, 13, -8 }, { 12, 14, -10 }, + { 13, 15, -10 }, { 13, 15, -12 }, { 15, 16, -12 }, + { 16, 17, -12 } + } + }, + /* 15BPP/14BPC */ + { 273, 15, 2048, 15, 24, 23, 23, { + { 0, 4, 10 }, { 3, 8, 8 }, { 6, 11, 6 }, { 9, 14, 4 }, + { 13, 15, 2 }, { 13, 15, 0 }, { 13, 16, -2 }, { 14, 16, -4 }, + { 15, 17, -6 }, { 15, 17, -8 }, { 16, 18, -10 }, + { 17, 19, -10 }, { 17, 19, -12 }, { 19, 20, -12 }, + { 20, 21, -12 } + } + }, + /* 15BPP/16BPC */ + { 273, 15, 2048, 19, 28, 27, 27, { + { 0, 4, 10 }, { 4, 9, 8 }, { 8, 13, 6 }, { 12, 17, 4 }, + { 17, 19, 2 }, { 17, 20, 0 }, { 17, 20, -2 }, { 18, 20, -4 }, + { 19, 21, -6 }, { 19, 21, -8 }, { 20, 22, -10 }, + { 21, 23, -10 }, { 21, 23, -12 }, { 23, 24, -12 }, + { 24, 25, -12 } + } + } +} + +}; + +static int get_row_index_for_rc_params(u16 compressed_bpp) +{ + switch (compressed_bpp) { + case 6: + return ROW_INDEX_6BPP; + case 8: + return ROW_INDEX_8BPP; + case 10: + return ROW_INDEX_10BPP; + case 12: + return ROW_INDEX_12BPP; + case 15: + return ROW_INDEX_15BPP; + default: + return -EINVAL; + } +} + +static int get_column_index_for_rc_params(u8 bits_per_component) +{ + switch (bits_per_component) { + case 8: + return COLUMN_INDEX_8BPC; + case 10: + return COLUMN_INDEX_10BPC; + case 12: + return COLUMN_INDEX_12BPC; + case 14: + return COLUMN_INDEX_14BPC; + case 16: + return COLUMN_INDEX_16BPC; + default: + return -EINVAL; + } +} + +static int intel_compute_rc_parameters(struct drm_dsc_config *vdsc_cfg) +{ + unsigned long groups_per_line = 0; + unsigned long groups_total = 0; + unsigned long num_extra_mux_bits = 0; + unsigned long slice_bits = 0; + unsigned long hrd_delay = 0; + unsigned long final_scale = 0; + unsigned long rbs_min = 0; + + /* Number of groups used to code each line of a slice */ + groups_per_line = DIV_ROUND_UP(vdsc_cfg->slice_width, + DSC_RC_PIXELS_PER_GROUP); + + /* chunksize in Bytes */ + vdsc_cfg->slice_chunk_size = DIV_ROUND_UP(vdsc_cfg->slice_width * + vdsc_cfg->bits_per_pixel, + (8 * 16)); + + if (vdsc_cfg->convert_rgb) + num_extra_mux_bits = 3 * (vdsc_cfg->mux_word_size + + (4 * vdsc_cfg->bits_per_component + 4) + - 2); + else + num_extra_mux_bits = 3 * vdsc_cfg->mux_word_size + + (4 * vdsc_cfg->bits_per_component + 4) + + 2 * (4 * vdsc_cfg->bits_per_component) - 2; + /* Number of bits in one Slice */ + slice_bits = 8 * vdsc_cfg->slice_chunk_size * vdsc_cfg->slice_height; + + while ((num_extra_mux_bits > 0) && + ((slice_bits - num_extra_mux_bits) % vdsc_cfg->mux_word_size)) + num_extra_mux_bits--; + + if (groups_per_line < vdsc_cfg->initial_scale_value - 8) + vdsc_cfg->initial_scale_value = groups_per_line + 8; + + /* scale_decrement_interval calculation according to DSC spec 1.11 */ + if (vdsc_cfg->initial_scale_value > 8) + vdsc_cfg->scale_decrement_interval = groups_per_line / + (vdsc_cfg->initial_scale_value - 8); + else + vdsc_cfg->scale_decrement_interval = DSC_SCALE_DECREMENT_INTERVAL_MAX; + + vdsc_cfg->final_offset = vdsc_cfg->rc_model_size - + (vdsc_cfg->initial_xmit_delay * + vdsc_cfg->bits_per_pixel + 8) / 16 + num_extra_mux_bits; + + if (vdsc_cfg->final_offset >= vdsc_cfg->rc_model_size) { + DRM_DEBUG_KMS("FinalOfs < RcModelSze for this InitialXmitDelay\n"); + return -ERANGE; + } + + final_scale = (vdsc_cfg->rc_model_size * 8) / + (vdsc_cfg->rc_model_size - vdsc_cfg->final_offset); + if (vdsc_cfg->slice_height > 1) + /* + * NflBpgOffset is 16 bit value with 11 fractional bits + * hence we multiply by 2^11 for preserving the + * fractional part + */ + vdsc_cfg->nfl_bpg_offset = DIV_ROUND_UP((vdsc_cfg->first_line_bpg_offset << 11), + (vdsc_cfg->slice_height - 1)); + else + vdsc_cfg->nfl_bpg_offset = 0; + + /* 2^16 - 1 */ + if (vdsc_cfg->nfl_bpg_offset > 65535) { + DRM_DEBUG_KMS("NflBpgOffset is too large for this slice height\n"); + return -ERANGE; + } + + /* Number of groups used to code the entire slice */ + groups_total = groups_per_line * vdsc_cfg->slice_height; + + /* slice_bpg_offset is 16 bit value with 11 fractional bits */ + vdsc_cfg->slice_bpg_offset = DIV_ROUND_UP(((vdsc_cfg->rc_model_size - + vdsc_cfg->initial_offset + + num_extra_mux_bits) << 11), + groups_total); + + if (final_scale > 9) { + /* + * ScaleIncrementInterval = + * finaloffset/((NflBpgOffset + SliceBpgOffset)*8(finalscale - 1.125)) + * as (NflBpgOffset + SliceBpgOffset) has 11 bit fractional value, + * we need divide by 2^11 from pstDscCfg values + */ + vdsc_cfg->scale_increment_interval = + (vdsc_cfg->final_offset * (1 << 11)) / + ((vdsc_cfg->nfl_bpg_offset + + vdsc_cfg->slice_bpg_offset) * + (final_scale - 9)); + } else { + /* + * If finalScaleValue is less than or equal to 9, a value of 0 should + * be used to disable the scale increment at the end of the slice + */ + vdsc_cfg->scale_increment_interval = 0; + } + + if (vdsc_cfg->scale_increment_interval > 65535) { + DRM_DEBUG_KMS("ScaleIncrementInterval is large for slice height\n"); + return -ERANGE; + } + + /* + * DSC spec mentions that bits_per_pixel specifies the target + * bits/pixel (bpp) rate that is used by the encoder, + * in steps of 1/16 of a bit per pixel + */ + rbs_min = vdsc_cfg->rc_model_size - vdsc_cfg->initial_offset + + DIV_ROUND_UP(vdsc_cfg->initial_xmit_delay * + vdsc_cfg->bits_per_pixel, 16) + + groups_per_line * vdsc_cfg->first_line_bpg_offset; + + hrd_delay = DIV_ROUND_UP((rbs_min * 16), vdsc_cfg->bits_per_pixel); + vdsc_cfg->rc_bits = (hrd_delay * vdsc_cfg->bits_per_pixel) / 16; + vdsc_cfg->initial_dec_delay = hrd_delay - vdsc_cfg->initial_xmit_delay; + + return 0; +} + +int intel_dp_compute_dsc_params(struct intel_dp *intel_dp, + struct intel_crtc_state *pipe_config) +{ + struct drm_dsc_config *vdsc_cfg = &pipe_config->dp_dsc_cfg; + u16 compressed_bpp = pipe_config->dsc_params.compressed_bpp; + u8 i = 0; + int row_index = 0; + int column_index = 0; + u8 line_buf_depth = 0; + + vdsc_cfg->pic_width = pipe_config->base.adjusted_mode.crtc_hdisplay; + vdsc_cfg->pic_height = pipe_config->base.adjusted_mode.crtc_vdisplay; + vdsc_cfg->slice_width = DIV_ROUND_UP(vdsc_cfg->pic_width, + pipe_config->dsc_params.slice_count); + /* + * Slice Height of 8 works for all currently available panels. So start + * with that if pic_height is an integral multiple of 8. + * Eventually add logic to try multiple slice heights. + */ + if (vdsc_cfg->pic_height % 8 == 0) + vdsc_cfg->slice_height = 8; + else if (vdsc_cfg->pic_height % 4 == 0) + vdsc_cfg->slice_height = 4; + else + vdsc_cfg->slice_height = 2; + + /* Values filled from DSC Sink DPCD */ + vdsc_cfg->dsc_version_major = + (intel_dp->dsc_dpcd[DP_DSC_REV - DP_DSC_SUPPORT] & + DP_DSC_MAJOR_MASK) >> DP_DSC_MAJOR_SHIFT; + vdsc_cfg->dsc_version_minor = + min(DSC_SUPPORTED_VERSION_MIN, + (intel_dp->dsc_dpcd[DP_DSC_REV - DP_DSC_SUPPORT] & + DP_DSC_MINOR_MASK) >> DP_DSC_MINOR_SHIFT); + + vdsc_cfg->convert_rgb = intel_dp->dsc_dpcd[DP_DSC_DEC_COLOR_FORMAT_CAP - DP_DSC_SUPPORT] & + DP_DSC_RGB; + + line_buf_depth = drm_dp_dsc_sink_line_buf_depth(intel_dp->dsc_dpcd); + if (!line_buf_depth) { + DRM_DEBUG_KMS("DSC Sink Line Buffer Depth invalid\n"); + return -EINVAL; + } + if (vdsc_cfg->dsc_version_minor == 2) + vdsc_cfg->line_buf_depth = (line_buf_depth == DSC_1_2_MAX_LINEBUF_DEPTH_BITS) ? + DSC_1_2_MAX_LINEBUF_DEPTH_VAL : line_buf_depth; + else + vdsc_cfg->line_buf_depth = (line_buf_depth > DSC_1_1_MAX_LINEBUF_DEPTH_BITS) ? + DSC_1_1_MAX_LINEBUF_DEPTH_BITS : line_buf_depth; + + /* Gen 11 does not support YCbCr */ + vdsc_cfg->enable422 = false; + /* Gen 11 does not support VBR */ + vdsc_cfg->vbr_enable = false; + vdsc_cfg->block_pred_enable = + intel_dp->dsc_dpcd[DP_DSC_BLK_PREDICTION_SUPPORT - DP_DSC_SUPPORT] & + DP_DSC_BLK_PREDICTION_IS_SUPPORTED; + + /* Gen 11 only supports integral values of bpp */ + vdsc_cfg->bits_per_pixel = compressed_bpp << 4; + vdsc_cfg->bits_per_component = pipe_config->pipe_bpp / 3; + + for (i = 0; i < DSC_NUM_BUF_RANGES - 1; i++) { + /* + * six 0s are appended to the lsb of each threshold value + * internally in h/w. + * Only 8 bits are allowed for programming RcBufThreshold + */ + vdsc_cfg->rc_buf_thresh[i] = rc_buf_thresh[i] >> 6; + } + + /* + * For 6bpp, RC Buffer threshold 12 and 13 need a different value + * as per C Model + */ + if (compressed_bpp == 6) { + vdsc_cfg->rc_buf_thresh[12] = 0x7C; + vdsc_cfg->rc_buf_thresh[13] = 0x7D; + } + + row_index = get_row_index_for_rc_params(compressed_bpp); + column_index = + get_column_index_for_rc_params(vdsc_cfg->bits_per_component); + + if (row_index < 0 || column_index < 0) + return -EINVAL; + + vdsc_cfg->first_line_bpg_offset = + rc_params[row_index][column_index].first_line_bpg_offset; + vdsc_cfg->initial_xmit_delay = + rc_params[row_index][column_index].initial_xmit_delay; + vdsc_cfg->initial_offset = + rc_params[row_index][column_index].initial_offset; + vdsc_cfg->flatness_min_qp = + rc_params[row_index][column_index].flatness_min_qp; + vdsc_cfg->flatness_max_qp = + rc_params[row_index][column_index].flatness_max_qp; + vdsc_cfg->rc_quant_incr_limit0 = + rc_params[row_index][column_index].rc_quant_incr_limit0; + vdsc_cfg->rc_quant_incr_limit1 = + rc_params[row_index][column_index].rc_quant_incr_limit1; + + for (i = 0; i < DSC_NUM_BUF_RANGES; i++) { + vdsc_cfg->rc_range_params[i].range_min_qp = + rc_params[row_index][column_index].rc_range_params[i].range_min_qp; + vdsc_cfg->rc_range_params[i].range_max_qp = + rc_params[row_index][column_index].rc_range_params[i].range_max_qp; + /* + * Range BPG Offset uses 2's complement and is only a 6 bits. So + * mask it to get only 6 bits. + */ + vdsc_cfg->rc_range_params[i].range_bpg_offset = + rc_params[row_index][column_index].rc_range_params[i].range_bpg_offset & + DSC_RANGE_BPG_OFFSET_MASK; + } + + /* + * BitsPerComponent value determines mux_word_size: + * When BitsPerComponent is 12bpc, muxWordSize will be equal to 64 bits + * When BitsPerComponent is 8 or 10bpc, muxWordSize will be equal to + * 48 bits + */ + if (vdsc_cfg->bits_per_component == 8 || + vdsc_cfg->bits_per_component == 10) + vdsc_cfg->mux_word_size = DSC_MUX_WORD_SIZE_8_10_BPC; + else if (vdsc_cfg->bits_per_component == 12) + vdsc_cfg->mux_word_size = DSC_MUX_WORD_SIZE_12_BPC; + + /* RC_MODEL_SIZE is a constant across all configurations */ + vdsc_cfg->rc_model_size = DSC_RC_MODEL_SIZE_CONST; + /* InitialScaleValue is a 6 bit value with 3 fractional bits (U3.3) */ + vdsc_cfg->initial_scale_value = (vdsc_cfg->rc_model_size << 3) / + (vdsc_cfg->rc_model_size - vdsc_cfg->initial_offset); + + return intel_compute_rc_parameters(vdsc_cfg); +} + +enum intel_display_power_domain +intel_dsc_power_domain(const struct intel_crtc_state *crtc_state) +{ + enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; + + /* + * On ICL VDSC/joining for eDP transcoder uses a separate power well PW2 + * This requires POWER_DOMAIN_TRANSCODER_EDP_VDSC power domain. + * For any other transcoder, VDSC/joining uses the power well associated + * with the pipe/transcoder in use. Hence another reference on the + * transcoder power domain will suffice. + */ + if (cpu_transcoder == TRANSCODER_EDP) + return POWER_DOMAIN_TRANSCODER_EDP_VDSC; + else + return POWER_DOMAIN_TRANSCODER(cpu_transcoder); +} + +static void intel_configure_pps_for_dsc_encoder(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + const struct drm_dsc_config *vdsc_cfg = &crtc_state->dp_dsc_cfg; + enum pipe pipe = crtc->pipe; + enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; + u32 pps_val = 0; + u32 rc_buf_thresh_dword[4]; + u32 rc_range_params_dword[8]; + u8 num_vdsc_instances = (crtc_state->dsc_params.dsc_split) ? 2 : 1; + int i = 0; + + /* Populate PICTURE_PARAMETER_SET_0 registers */ + pps_val = DSC_VER_MAJ | vdsc_cfg->dsc_version_minor << + DSC_VER_MIN_SHIFT | + vdsc_cfg->bits_per_component << DSC_BPC_SHIFT | + vdsc_cfg->line_buf_depth << DSC_LINE_BUF_DEPTH_SHIFT; + if (vdsc_cfg->block_pred_enable) + pps_val |= DSC_BLOCK_PREDICTION; + if (vdsc_cfg->convert_rgb) + pps_val |= DSC_COLOR_SPACE_CONVERSION; + if (vdsc_cfg->enable422) + pps_val |= DSC_422_ENABLE; + if (vdsc_cfg->vbr_enable) + pps_val |= DSC_VBR_ENABLE; + DRM_INFO("PPS0 = 0x%08x\n", pps_val); + if (cpu_transcoder == TRANSCODER_EDP) { + I915_WRITE(DSCA_PICTURE_PARAMETER_SET_0, pps_val); + /* + * If 2 VDSC instances are needed, configure PPS for second + * VDSC + */ + if (crtc_state->dsc_params.dsc_split) + I915_WRITE(DSCC_PICTURE_PARAMETER_SET_0, pps_val); + } else { + I915_WRITE(ICL_DSC0_PICTURE_PARAMETER_SET_0(pipe), pps_val); + if (crtc_state->dsc_params.dsc_split) + I915_WRITE(ICL_DSC1_PICTURE_PARAMETER_SET_0(pipe), + pps_val); + } + + /* Populate PICTURE_PARAMETER_SET_1 registers */ + pps_val = 0; + pps_val |= DSC_BPP(vdsc_cfg->bits_per_pixel); + DRM_INFO("PPS1 = 0x%08x\n", pps_val); + if (cpu_transcoder == TRANSCODER_EDP) { + I915_WRITE(DSCA_PICTURE_PARAMETER_SET_1, pps_val); + /* + * If 2 VDSC instances are needed, configure PPS for second + * VDSC + */ + if (crtc_state->dsc_params.dsc_split) + I915_WRITE(DSCC_PICTURE_PARAMETER_SET_1, pps_val); + } else { + I915_WRITE(ICL_DSC0_PICTURE_PARAMETER_SET_1(pipe), pps_val); + if (crtc_state->dsc_params.dsc_split) + I915_WRITE(ICL_DSC1_PICTURE_PARAMETER_SET_1(pipe), + pps_val); + } + + /* Populate PICTURE_PARAMETER_SET_2 registers */ + pps_val = 0; + pps_val |= DSC_PIC_HEIGHT(vdsc_cfg->pic_height) | + DSC_PIC_WIDTH(vdsc_cfg->pic_width / num_vdsc_instances); + DRM_INFO("PPS2 = 0x%08x\n", pps_val); + if (encoder->type == INTEL_OUTPUT_EDP) { + I915_WRITE(DSCA_PICTURE_PARAMETER_SET_2, pps_val); + /* + * If 2 VDSC instances are needed, configure PPS for second + * VDSC + */ + if (crtc_state->dsc_params.dsc_split) + I915_WRITE(DSCC_PICTURE_PARAMETER_SET_2, pps_val); + } else { + I915_WRITE(ICL_DSC0_PICTURE_PARAMETER_SET_2(pipe), pps_val); + if (crtc_state->dsc_params.dsc_split) + I915_WRITE(ICL_DSC1_PICTURE_PARAMETER_SET_2(pipe), + pps_val); + } + + /* Populate PICTURE_PARAMETER_SET_3 registers */ + pps_val = 0; + pps_val |= DSC_SLICE_HEIGHT(vdsc_cfg->slice_height) | + DSC_SLICE_WIDTH(vdsc_cfg->slice_width); + DRM_INFO("PPS3 = 0x%08x\n", pps_val); + if (cpu_transcoder == TRANSCODER_EDP) { + I915_WRITE(DSCA_PICTURE_PARAMETER_SET_3, pps_val); + /* + * If 2 VDSC instances are needed, configure PPS for second + * VDSC + */ + if (crtc_state->dsc_params.dsc_split) + I915_WRITE(DSCC_PICTURE_PARAMETER_SET_3, pps_val); + } else { + I915_WRITE(ICL_DSC0_PICTURE_PARAMETER_SET_3(pipe), pps_val); + if (crtc_state->dsc_params.dsc_split) + I915_WRITE(ICL_DSC1_PICTURE_PARAMETER_SET_3(pipe), + pps_val); + } + + /* Populate PICTURE_PARAMETER_SET_4 registers */ + pps_val = 0; + pps_val |= DSC_INITIAL_XMIT_DELAY(vdsc_cfg->initial_xmit_delay) | + DSC_INITIAL_DEC_DELAY(vdsc_cfg->initial_dec_delay); + DRM_INFO("PPS4 = 0x%08x\n", pps_val); + if (cpu_transcoder == TRANSCODER_EDP) { + I915_WRITE(DSCA_PICTURE_PARAMETER_SET_4, pps_val); + /* + * If 2 VDSC instances are needed, configure PPS for second + * VDSC + */ + if (crtc_state->dsc_params.dsc_split) + I915_WRITE(DSCC_PICTURE_PARAMETER_SET_4, pps_val); + } else { + I915_WRITE(ICL_DSC0_PICTURE_PARAMETER_SET_4(pipe), pps_val); + if (crtc_state->dsc_params.dsc_split) + I915_WRITE(ICL_DSC1_PICTURE_PARAMETER_SET_4(pipe), + pps_val); + } + + /* Populate PICTURE_PARAMETER_SET_5 registers */ + pps_val = 0; + pps_val |= DSC_SCALE_INC_INT(vdsc_cfg->scale_increment_interval) | + DSC_SCALE_DEC_INT(vdsc_cfg->scale_decrement_interval); + DRM_INFO("PPS5 = 0x%08x\n", pps_val); + if (cpu_transcoder == TRANSCODER_EDP) { + I915_WRITE(DSCA_PICTURE_PARAMETER_SET_5, pps_val); + /* + * If 2 VDSC instances are needed, configure PPS for second + * VDSC + */ + if (crtc_state->dsc_params.dsc_split) + I915_WRITE(DSCC_PICTURE_PARAMETER_SET_5, pps_val); + } else { + I915_WRITE(ICL_DSC0_PICTURE_PARAMETER_SET_5(pipe), pps_val); + if (crtc_state->dsc_params.dsc_split) + I915_WRITE(ICL_DSC1_PICTURE_PARAMETER_SET_5(pipe), + pps_val); + } + + /* Populate PICTURE_PARAMETER_SET_6 registers */ + pps_val = 0; + pps_val |= DSC_INITIAL_SCALE_VALUE(vdsc_cfg->initial_scale_value) | + DSC_FIRST_LINE_BPG_OFFSET(vdsc_cfg->first_line_bpg_offset) | + DSC_FLATNESS_MIN_QP(vdsc_cfg->flatness_min_qp) | + DSC_FLATNESS_MAX_QP(vdsc_cfg->flatness_max_qp); + DRM_INFO("PPS6 = 0x%08x\n", pps_val); + if (cpu_transcoder == TRANSCODER_EDP) { + I915_WRITE(DSCA_PICTURE_PARAMETER_SET_6, pps_val); + /* + * If 2 VDSC instances are needed, configure PPS for second + * VDSC + */ + if (crtc_state->dsc_params.dsc_split) + I915_WRITE(DSCC_PICTURE_PARAMETER_SET_6, pps_val); + } else { + I915_WRITE(ICL_DSC0_PICTURE_PARAMETER_SET_6(pipe), pps_val); + if (crtc_state->dsc_params.dsc_split) + I915_WRITE(ICL_DSC1_PICTURE_PARAMETER_SET_6(pipe), + pps_val); + } + + /* Populate PICTURE_PARAMETER_SET_7 registers */ + pps_val = 0; + pps_val |= DSC_SLICE_BPG_OFFSET(vdsc_cfg->slice_bpg_offset) | + DSC_NFL_BPG_OFFSET(vdsc_cfg->nfl_bpg_offset); + DRM_INFO("PPS7 = 0x%08x\n", pps_val); + if (cpu_transcoder == TRANSCODER_EDP) { + I915_WRITE(DSCA_PICTURE_PARAMETER_SET_7, pps_val); + /* + * If 2 VDSC instances are needed, configure PPS for second + * VDSC + */ + if (crtc_state->dsc_params.dsc_split) + I915_WRITE(DSCC_PICTURE_PARAMETER_SET_7, pps_val); + } else { + I915_WRITE(ICL_DSC0_PICTURE_PARAMETER_SET_7(pipe), pps_val); + if (crtc_state->dsc_params.dsc_split) + I915_WRITE(ICL_DSC1_PICTURE_PARAMETER_SET_7(pipe), + pps_val); + } + + /* Populate PICTURE_PARAMETER_SET_8 registers */ + pps_val = 0; + pps_val |= DSC_FINAL_OFFSET(vdsc_cfg->final_offset) | + DSC_INITIAL_OFFSET(vdsc_cfg->initial_offset); + DRM_INFO("PPS8 = 0x%08x\n", pps_val); + if (cpu_transcoder == TRANSCODER_EDP) { + I915_WRITE(DSCA_PICTURE_PARAMETER_SET_8, pps_val); + /* + * If 2 VDSC instances are needed, configure PPS for second + * VDSC + */ + if (crtc_state->dsc_params.dsc_split) + I915_WRITE(DSCC_PICTURE_PARAMETER_SET_8, pps_val); + } else { + I915_WRITE(ICL_DSC0_PICTURE_PARAMETER_SET_8(pipe), pps_val); + if (crtc_state->dsc_params.dsc_split) + I915_WRITE(ICL_DSC1_PICTURE_PARAMETER_SET_8(pipe), + pps_val); + } + + /* Populate PICTURE_PARAMETER_SET_9 registers */ + pps_val = 0; + pps_val |= DSC_RC_MODEL_SIZE(DSC_RC_MODEL_SIZE_CONST) | + DSC_RC_EDGE_FACTOR(DSC_RC_EDGE_FACTOR_CONST); + DRM_INFO("PPS9 = 0x%08x\n", pps_val); + if (cpu_transcoder == TRANSCODER_EDP) { + I915_WRITE(DSCA_PICTURE_PARAMETER_SET_9, pps_val); + /* + * If 2 VDSC instances are needed, configure PPS for second + * VDSC + */ + if (crtc_state->dsc_params.dsc_split) + I915_WRITE(DSCC_PICTURE_PARAMETER_SET_9, pps_val); + } else { + I915_WRITE(ICL_DSC0_PICTURE_PARAMETER_SET_9(pipe), pps_val); + if (crtc_state->dsc_params.dsc_split) + I915_WRITE(ICL_DSC1_PICTURE_PARAMETER_SET_9(pipe), + pps_val); + } + + /* Populate PICTURE_PARAMETER_SET_10 registers */ + pps_val = 0; + pps_val |= DSC_RC_QUANT_INC_LIMIT0(vdsc_cfg->rc_quant_incr_limit0) | + DSC_RC_QUANT_INC_LIMIT1(vdsc_cfg->rc_quant_incr_limit1) | + DSC_RC_TARGET_OFF_HIGH(DSC_RC_TGT_OFFSET_HI_CONST) | + DSC_RC_TARGET_OFF_LOW(DSC_RC_TGT_OFFSET_LO_CONST); + DRM_INFO("PPS10 = 0x%08x\n", pps_val); + if (cpu_transcoder == TRANSCODER_EDP) { + I915_WRITE(DSCA_PICTURE_PARAMETER_SET_10, pps_val); + /* + * If 2 VDSC instances are needed, configure PPS for second + * VDSC + */ + if (crtc_state->dsc_params.dsc_split) + I915_WRITE(DSCC_PICTURE_PARAMETER_SET_10, pps_val); + } else { + I915_WRITE(ICL_DSC0_PICTURE_PARAMETER_SET_10(pipe), pps_val); + if (crtc_state->dsc_params.dsc_split) + I915_WRITE(ICL_DSC1_PICTURE_PARAMETER_SET_10(pipe), + pps_val); + } + + /* Populate Picture parameter set 16 */ + pps_val = 0; + pps_val |= DSC_SLICE_CHUNK_SIZE(vdsc_cfg->slice_chunk_size) | + DSC_SLICE_PER_LINE((vdsc_cfg->pic_width / num_vdsc_instances) / + vdsc_cfg->slice_width) | + DSC_SLICE_ROW_PER_FRAME(vdsc_cfg->pic_height / + vdsc_cfg->slice_height); + DRM_INFO("PPS16 = 0x%08x\n", pps_val); + if (cpu_transcoder == TRANSCODER_EDP) { + I915_WRITE(DSCA_PICTURE_PARAMETER_SET_16, pps_val); + /* + * If 2 VDSC instances are needed, configure PPS for second + * VDSC + */ + if (crtc_state->dsc_params.dsc_split) + I915_WRITE(DSCC_PICTURE_PARAMETER_SET_16, pps_val); + } else { + I915_WRITE(ICL_DSC0_PICTURE_PARAMETER_SET_16(pipe), pps_val); + if (crtc_state->dsc_params.dsc_split) + I915_WRITE(ICL_DSC1_PICTURE_PARAMETER_SET_16(pipe), + pps_val); + } + + /* Populate the RC_BUF_THRESH registers */ + memset(rc_buf_thresh_dword, 0, sizeof(rc_buf_thresh_dword)); + for (i = 0; i < DSC_NUM_BUF_RANGES - 1; i++) { + rc_buf_thresh_dword[i / 4] |= + (u32)(vdsc_cfg->rc_buf_thresh[i] << + BITS_PER_BYTE * (i % 4)); + DRM_INFO(" RC_BUF_THRESH%d = 0x%08x\n", i, + rc_buf_thresh_dword[i / 4]); + } + if (cpu_transcoder == TRANSCODER_EDP) { + I915_WRITE(DSCA_RC_BUF_THRESH_0, rc_buf_thresh_dword[0]); + I915_WRITE(DSCA_RC_BUF_THRESH_0_UDW, rc_buf_thresh_dword[1]); + I915_WRITE(DSCA_RC_BUF_THRESH_1, rc_buf_thresh_dword[2]); + I915_WRITE(DSCA_RC_BUF_THRESH_1_UDW, rc_buf_thresh_dword[3]); + if (crtc_state->dsc_params.dsc_split) { + I915_WRITE(DSCC_RC_BUF_THRESH_0, + rc_buf_thresh_dword[0]); + I915_WRITE(DSCC_RC_BUF_THRESH_0_UDW, + rc_buf_thresh_dword[1]); + I915_WRITE(DSCC_RC_BUF_THRESH_1, + rc_buf_thresh_dword[2]); + I915_WRITE(DSCC_RC_BUF_THRESH_1_UDW, + rc_buf_thresh_dword[3]); + } + } else { + I915_WRITE(ICL_DSC0_RC_BUF_THRESH_0(pipe), + rc_buf_thresh_dword[0]); + I915_WRITE(ICL_DSC0_RC_BUF_THRESH_0_UDW(pipe), + rc_buf_thresh_dword[1]); + I915_WRITE(ICL_DSC0_RC_BUF_THRESH_1(pipe), + rc_buf_thresh_dword[2]); + I915_WRITE(ICL_DSC0_RC_BUF_THRESH_1_UDW(pipe), + rc_buf_thresh_dword[3]); + if (crtc_state->dsc_params.dsc_split) { + I915_WRITE(ICL_DSC1_RC_BUF_THRESH_0(pipe), + rc_buf_thresh_dword[0]); + I915_WRITE(ICL_DSC1_RC_BUF_THRESH_0_UDW(pipe), + rc_buf_thresh_dword[1]); + I915_WRITE(ICL_DSC1_RC_BUF_THRESH_1(pipe), + rc_buf_thresh_dword[2]); + I915_WRITE(ICL_DSC1_RC_BUF_THRESH_1_UDW(pipe), + rc_buf_thresh_dword[3]); + } + } + + /* Populate the RC_RANGE_PARAMETERS registers */ + memset(rc_range_params_dword, 0, sizeof(rc_range_params_dword)); + for (i = 0; i < DSC_NUM_BUF_RANGES; i++) { + rc_range_params_dword[i / 2] |= + (u32)(((vdsc_cfg->rc_range_params[i].range_bpg_offset << + RC_BPG_OFFSET_SHIFT) | + (vdsc_cfg->rc_range_params[i].range_max_qp << + RC_MAX_QP_SHIFT) | + (vdsc_cfg->rc_range_params[i].range_min_qp << + RC_MIN_QP_SHIFT)) << 16 * (i % 2)); + DRM_INFO(" RC_RANGE_PARAM_%d = 0x%08x\n", i, + rc_range_params_dword[i / 2]); + } + if (cpu_transcoder == TRANSCODER_EDP) { + I915_WRITE(DSCA_RC_RANGE_PARAMETERS_0, + rc_range_params_dword[0]); + I915_WRITE(DSCA_RC_RANGE_PARAMETERS_0_UDW, + rc_range_params_dword[1]); + I915_WRITE(DSCA_RC_RANGE_PARAMETERS_1, + rc_range_params_dword[2]); + I915_WRITE(DSCA_RC_RANGE_PARAMETERS_1_UDW, + rc_range_params_dword[3]); + I915_WRITE(DSCA_RC_RANGE_PARAMETERS_2, + rc_range_params_dword[4]); + I915_WRITE(DSCA_RC_RANGE_PARAMETERS_2_UDW, + rc_range_params_dword[5]); + I915_WRITE(DSCA_RC_RANGE_PARAMETERS_3, + rc_range_params_dword[6]); + I915_WRITE(DSCA_RC_RANGE_PARAMETERS_3_UDW, + rc_range_params_dword[7]); + if (crtc_state->dsc_params.dsc_split) { + I915_WRITE(DSCC_RC_RANGE_PARAMETERS_0, + rc_range_params_dword[0]); + I915_WRITE(DSCC_RC_RANGE_PARAMETERS_0_UDW, + rc_range_params_dword[1]); + I915_WRITE(DSCC_RC_RANGE_PARAMETERS_1, + rc_range_params_dword[2]); + I915_WRITE(DSCC_RC_RANGE_PARAMETERS_1_UDW, + rc_range_params_dword[3]); + I915_WRITE(DSCC_RC_RANGE_PARAMETERS_2, + rc_range_params_dword[4]); + I915_WRITE(DSCC_RC_RANGE_PARAMETERS_2_UDW, + rc_range_params_dword[5]); + I915_WRITE(DSCC_RC_RANGE_PARAMETERS_3, + rc_range_params_dword[6]); + I915_WRITE(DSCC_RC_RANGE_PARAMETERS_3_UDW, + rc_range_params_dword[7]); + } + } else { + I915_WRITE(ICL_DSC0_RC_RANGE_PARAMETERS_0(pipe), + rc_range_params_dword[0]); + I915_WRITE(ICL_DSC0_RC_RANGE_PARAMETERS_0_UDW(pipe), + rc_range_params_dword[1]); + I915_WRITE(ICL_DSC0_RC_RANGE_PARAMETERS_1(pipe), + rc_range_params_dword[2]); + I915_WRITE(ICL_DSC0_RC_RANGE_PARAMETERS_1_UDW(pipe), + rc_range_params_dword[3]); + I915_WRITE(ICL_DSC0_RC_RANGE_PARAMETERS_2(pipe), + rc_range_params_dword[4]); + I915_WRITE(ICL_DSC0_RC_RANGE_PARAMETERS_2_UDW(pipe), + rc_range_params_dword[5]); + I915_WRITE(ICL_DSC0_RC_RANGE_PARAMETERS_3(pipe), + rc_range_params_dword[6]); + I915_WRITE(ICL_DSC0_RC_RANGE_PARAMETERS_3_UDW(pipe), + rc_range_params_dword[7]); + if (crtc_state->dsc_params.dsc_split) { + I915_WRITE(ICL_DSC1_RC_RANGE_PARAMETERS_0(pipe), + rc_range_params_dword[0]); + I915_WRITE(ICL_DSC1_RC_RANGE_PARAMETERS_0_UDW(pipe), + rc_range_params_dword[1]); + I915_WRITE(ICL_DSC1_RC_RANGE_PARAMETERS_1(pipe), + rc_range_params_dword[2]); + I915_WRITE(ICL_DSC1_RC_RANGE_PARAMETERS_1_UDW(pipe), + rc_range_params_dword[3]); + I915_WRITE(ICL_DSC1_RC_RANGE_PARAMETERS_2(pipe), + rc_range_params_dword[4]); + I915_WRITE(ICL_DSC1_RC_RANGE_PARAMETERS_2_UDW(pipe), + rc_range_params_dword[5]); + I915_WRITE(ICL_DSC1_RC_RANGE_PARAMETERS_3(pipe), + rc_range_params_dword[6]); + I915_WRITE(ICL_DSC1_RC_RANGE_PARAMETERS_3_UDW(pipe), + rc_range_params_dword[7]); + } + } +} + +static void intel_dp_write_dsc_pps_sdp(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state) +{ + struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + const struct drm_dsc_config *vdsc_cfg = &crtc_state->dp_dsc_cfg; + struct drm_dsc_pps_infoframe dp_dsc_pps_sdp; + + /* Prepare DP SDP PPS header as per DP 1.4 spec, Table 2-123 */ + drm_dsc_dp_pps_header_init(&dp_dsc_pps_sdp); + + /* Fill the PPS payload bytes as per DSC spec 1.2 Table 4-1 */ + drm_dsc_pps_infoframe_pack(&dp_dsc_pps_sdp, vdsc_cfg); + + intel_dig_port->write_infoframe(encoder, crtc_state, + DP_SDP_PPS, &dp_dsc_pps_sdp, + sizeof(dp_dsc_pps_sdp)); +} + +void intel_dsc_enable(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + enum pipe pipe = crtc->pipe; + i915_reg_t dss_ctl1_reg, dss_ctl2_reg; + u32 dss_ctl1_val = 0; + u32 dss_ctl2_val = 0; + + if (!crtc_state->dsc_params.compression_enable) + return; + + /* Enable Power wells for VDSC/joining */ + intel_display_power_get(dev_priv, + intel_dsc_power_domain(crtc_state)); + + intel_configure_pps_for_dsc_encoder(encoder, crtc_state); + + intel_dp_write_dsc_pps_sdp(encoder, crtc_state); + + if (crtc_state->cpu_transcoder == TRANSCODER_EDP) { + dss_ctl1_reg = DSS_CTL1; + dss_ctl2_reg = DSS_CTL2; + } else { + dss_ctl1_reg = ICL_PIPE_DSS_CTL1(pipe); + dss_ctl2_reg = ICL_PIPE_DSS_CTL2(pipe); + } + dss_ctl2_val |= LEFT_BRANCH_VDSC_ENABLE; + if (crtc_state->dsc_params.dsc_split) { + dss_ctl2_val |= RIGHT_BRANCH_VDSC_ENABLE; + dss_ctl1_val |= JOINER_ENABLE; + } + I915_WRITE(dss_ctl1_reg, dss_ctl1_val); + I915_WRITE(dss_ctl2_reg, dss_ctl2_val); +} + +void intel_dsc_disable(const struct intel_crtc_state *old_crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; + i915_reg_t dss_ctl1_reg, dss_ctl2_reg; + u32 dss_ctl1_val = 0, dss_ctl2_val = 0; + + if (!old_crtc_state->dsc_params.compression_enable) + return; + + if (old_crtc_state->cpu_transcoder == TRANSCODER_EDP) { + dss_ctl1_reg = DSS_CTL1; + dss_ctl2_reg = DSS_CTL2; + } else { + dss_ctl1_reg = ICL_PIPE_DSS_CTL1(pipe); + dss_ctl2_reg = ICL_PIPE_DSS_CTL2(pipe); + } + dss_ctl1_val = I915_READ(dss_ctl1_reg); + if (dss_ctl1_val & JOINER_ENABLE) + dss_ctl1_val &= ~JOINER_ENABLE; + I915_WRITE(dss_ctl1_reg, dss_ctl1_val); + + dss_ctl2_val = I915_READ(dss_ctl2_reg); + if (dss_ctl2_val & LEFT_BRANCH_VDSC_ENABLE || + dss_ctl2_val & RIGHT_BRANCH_VDSC_ENABLE) + dss_ctl2_val &= ~(LEFT_BRANCH_VDSC_ENABLE | + RIGHT_BRANCH_VDSC_ENABLE); + I915_WRITE(dss_ctl2_reg, dss_ctl2_val); + + /* Disable Power wells for VDSC/joining */ + intel_display_power_put_unchecked(dev_priv, + intel_dsc_power_domain(old_crtc_state)); +} diff --git a/drivers/gpu/drm/i915/intel_wopcm.c b/drivers/gpu/drm/i915/intel_wopcm.c index 92cb82dd0c07..f82a415ea2ba 100644 --- a/drivers/gpu/drm/i915/intel_wopcm.c +++ b/drivers/gpu/drm/i915/intel_wopcm.c @@ -130,11 +130,11 @@ static inline int check_hw_restriction(struct drm_i915_private *i915, { int err = 0; - if (IS_GEN9(i915)) + if (IS_GEN(i915, 9)) err = gen9_check_dword_gap(guc_wopcm_base, guc_wopcm_size); if (!err && - (IS_GEN9(i915) || IS_CNL_REVID(i915, CNL_REVID_A0, CNL_REVID_A0))) + (IS_GEN(i915, 9) || IS_CNL_REVID(i915, CNL_REVID_A0, CNL_REVID_A0))) err = gen9_check_huc_fw_fits(guc_wopcm_size, huc_fw_size); return err; @@ -163,7 +163,7 @@ int intel_wopcm_init(struct intel_wopcm *wopcm) u32 guc_wopcm_rsvd; int err; - if (!USES_GUC(dev_priv)) + if (!USES_GUC(i915)) return 0; GEM_BUG_ON(!wopcm->size); diff --git a/drivers/gpu/drm/i915/intel_workarounds.c b/drivers/gpu/drm/i915/intel_workarounds.c index 6e580891db96..15f4a6dee5aa 100644 --- a/drivers/gpu/drm/i915/intel_workarounds.c +++ b/drivers/gpu/drm/i915/intel_workarounds.c @@ -53,79 +53,138 @@ static void wa_init_start(struct i915_wa_list *wal, const char *name) wal->name = name; } +#define WA_LIST_CHUNK (1 << 4) + static void wa_init_finish(struct i915_wa_list *wal) { + /* Trim unused entries. */ + if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) { + struct i915_wa *list = kmemdup(wal->list, + wal->count * sizeof(*list), + GFP_KERNEL); + + if (list) { + kfree(wal->list); + wal->list = list; + } + } + if (!wal->count) return; DRM_DEBUG_DRIVER("Initialized %u %s workarounds\n", - wal->count, wal->name); + wal->wa_count, wal->name); } -static void wa_add(struct drm_i915_private *i915, - i915_reg_t reg, const u32 mask, const u32 val) +static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa) { - struct i915_workarounds *wa = &i915->workarounds; - unsigned int start = 0, end = wa->count; - unsigned int addr = i915_mmio_reg_offset(reg); - struct i915_wa_reg *r; + unsigned int addr = i915_mmio_reg_offset(wa->reg); + unsigned int start = 0, end = wal->count; + const unsigned int grow = WA_LIST_CHUNK; + struct i915_wa *wa_; + + GEM_BUG_ON(!is_power_of_2(grow)); + + if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */ + struct i915_wa *list; + + list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa), + GFP_KERNEL); + if (!list) { + DRM_ERROR("No space for workaround init!\n"); + return; + } + + if (wal->list) + memcpy(list, wal->list, sizeof(*wa) * wal->count); + + wal->list = list; + } while (start < end) { unsigned int mid = start + (end - start) / 2; - if (wa->reg[mid].addr < addr) { + if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) { start = mid + 1; - } else if (wa->reg[mid].addr > addr) { + } else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) { end = mid; } else { - r = &wa->reg[mid]; + wa_ = &wal->list[mid]; - if ((mask & ~r->mask) == 0) { + if ((wa->mask & ~wa_->mask) == 0) { DRM_ERROR("Discarding overwritten w/a for reg %04x (mask: %08x, value: %08x)\n", - addr, r->mask, r->value); + i915_mmio_reg_offset(wa_->reg), + wa_->mask, wa_->val); - r->value &= ~mask; + wa_->val &= ~wa->mask; } - r->value |= val; - r->mask |= mask; + wal->wa_count++; + wa_->val |= wa->val; + wa_->mask |= wa->mask; return; } } - if (WARN_ON_ONCE(wa->count >= I915_MAX_WA_REGS)) { - DRM_ERROR("Dropping w/a for reg %04x (mask: %08x, value: %08x)\n", - addr, mask, val); - return; - } - - r = &wa->reg[wa->count++]; - r->addr = addr; - r->value = val; - r->mask = mask; + wal->wa_count++; + wa_ = &wal->list[wal->count++]; + *wa_ = *wa; - while (r-- > wa->reg) { - GEM_BUG_ON(r[0].addr == r[1].addr); - if (r[1].addr > r[0].addr) + while (wa_-- > wal->list) { + GEM_BUG_ON(i915_mmio_reg_offset(wa_[0].reg) == + i915_mmio_reg_offset(wa_[1].reg)); + if (i915_mmio_reg_offset(wa_[1].reg) > + i915_mmio_reg_offset(wa_[0].reg)) break; - swap(r[1], r[0]); + swap(wa_[1], wa_[0]); } } -#define WA_REG(addr, mask, val) wa_add(dev_priv, (addr), (mask), (val)) +static void +wa_write_masked_or(struct i915_wa_list *wal, i915_reg_t reg, u32 mask, + u32 val) +{ + struct i915_wa wa = { + .reg = reg, + .mask = mask, + .val = val + }; + + _wa_add(wal, &wa); +} + +static void +wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val) +{ + wa_write_masked_or(wal, reg, val, _MASKED_BIT_ENABLE(val)); +} + +static void +wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 val) +{ + wa_write_masked_or(wal, reg, ~0, val); +} + +static void +wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 val) +{ + wa_write_masked_or(wal, reg, val, val); +} #define WA_SET_BIT_MASKED(addr, mask) \ - WA_REG(addr, (mask), _MASKED_BIT_ENABLE(mask)) + wa_write_masked_or(wal, (addr), (mask), _MASKED_BIT_ENABLE(mask)) #define WA_CLR_BIT_MASKED(addr, mask) \ - WA_REG(addr, (mask), _MASKED_BIT_DISABLE(mask)) + wa_write_masked_or(wal, (addr), (mask), _MASKED_BIT_DISABLE(mask)) #define WA_SET_FIELD_MASKED(addr, mask, value) \ - WA_REG(addr, (mask), _MASKED_FIELD(mask, value)) + wa_write_masked_or(wal, (addr), (mask), _MASKED_FIELD((mask), (value))) -static int gen8_ctx_workarounds_init(struct drm_i915_private *dev_priv) +static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine) { + struct i915_wa_list *wal = &engine->ctx_wa_list; + WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING); /* WaDisableAsyncFlipPerfMode:bdw,chv */ @@ -169,17 +228,14 @@ static int gen8_ctx_workarounds_init(struct drm_i915_private *dev_priv) WA_SET_FIELD_MASKED(GEN7_GT_MODE, GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4); - - return 0; } -static int bdw_ctx_workarounds_init(struct drm_i915_private *dev_priv) +static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine) { - int ret; + struct drm_i915_private *i915 = engine->i915; + struct i915_wa_list *wal = &engine->ctx_wa_list; - ret = gen8_ctx_workarounds_init(dev_priv); - if (ret) - return ret; + gen8_ctx_workarounds_init(engine); /* WaDisableThreadStallDopClockGating:bdw (pre-production) */ WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE); @@ -199,31 +255,28 @@ static int bdw_ctx_workarounds_init(struct drm_i915_private *dev_priv) /* WaForceContextSaveRestoreNonCoherent:bdw */ HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT | /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */ - (IS_BDW_GT3(dev_priv) ? HDC_FENCE_DEST_SLM_DISABLE : 0)); - - return 0; + (IS_BDW_GT3(i915) ? HDC_FENCE_DEST_SLM_DISABLE : 0)); } -static int chv_ctx_workarounds_init(struct drm_i915_private *dev_priv) +static void chv_ctx_workarounds_init(struct intel_engine_cs *engine) { - int ret; + struct i915_wa_list *wal = &engine->ctx_wa_list; - ret = gen8_ctx_workarounds_init(dev_priv); - if (ret) - return ret; + gen8_ctx_workarounds_init(engine); /* WaDisableThreadStallDopClockGating:chv */ WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE); /* Improve HiZ throughput on CHV. */ WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X); - - return 0; } -static int gen9_ctx_workarounds_init(struct drm_i915_private *dev_priv) +static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine) { - if (HAS_LLC(dev_priv)) { + struct drm_i915_private *i915 = engine->i915; + struct i915_wa_list *wal = &engine->ctx_wa_list; + + if (HAS_LLC(i915)) { /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl * * Must match Display Engine. See @@ -242,7 +295,7 @@ static int gen9_ctx_workarounds_init(struct drm_i915_private *dev_priv) PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE); /* Syncing dependencies between camera and graphics:skl,bxt,kbl */ - if (!IS_COFFEELAKE(dev_priv)) + if (!IS_COFFEELAKE(i915)) WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC); @@ -285,9 +338,7 @@ static int gen9_ctx_workarounds_init(struct drm_i915_private *dev_priv) HDC_FORCE_NON_COHERENT); /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */ - if (IS_SKYLAKE(dev_priv) || - IS_KABYLAKE(dev_priv) || - IS_COFFEELAKE(dev_priv)) + if (IS_SKYLAKE(i915) || IS_KABYLAKE(i915) || IS_COFFEELAKE(i915)) WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, GEN8_SAMPLER_POWER_BYPASS_DIS); @@ -314,14 +365,14 @@ static int gen9_ctx_workarounds_init(struct drm_i915_private *dev_priv) GEN9_PREEMPT_GPGPU_COMMAND_LEVEL); /* WaClearHIZ_WM_CHICKEN3:bxt,glk */ - if (IS_GEN9_LP(dev_priv)) + if (IS_GEN9_LP(i915)) WA_SET_BIT_MASKED(GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ); - - return 0; } -static int skl_tune_iz_hashing(struct drm_i915_private *dev_priv) +static void skl_tune_iz_hashing(struct intel_engine_cs *engine) { + struct drm_i915_private *i915 = engine->i915; + struct i915_wa_list *wal = &engine->ctx_wa_list; u8 vals[3] = { 0, 0, 0 }; unsigned int i; @@ -332,7 +383,7 @@ static int skl_tune_iz_hashing(struct drm_i915_private *dev_priv) * Only consider slices where one, and only one, subslice has 7 * EUs */ - if (!is_power_of_2(INTEL_INFO(dev_priv)->sseu.subslice_7eu[i])) + if (!is_power_of_2(RUNTIME_INFO(i915)->sseu.subslice_7eu[i])) continue; /* @@ -341,12 +392,12 @@ static int skl_tune_iz_hashing(struct drm_i915_private *dev_priv) * * -> 0 <= ss <= 3; */ - ss = ffs(INTEL_INFO(dev_priv)->sseu.subslice_7eu[i]) - 1; + ss = ffs(RUNTIME_INFO(i915)->sseu.subslice_7eu[i]) - 1; vals[i] = 3 - ss; } if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0) - return 0; + return; /* Tune IZ hashing. See intel_device_info_runtime_init() */ WA_SET_FIELD_MASKED(GEN7_GT_MODE, @@ -356,28 +407,19 @@ static int skl_tune_iz_hashing(struct drm_i915_private *dev_priv) GEN9_IZ_HASHING(2, vals[2]) | GEN9_IZ_HASHING(1, vals[1]) | GEN9_IZ_HASHING(0, vals[0])); - - return 0; } -static int skl_ctx_workarounds_init(struct drm_i915_private *dev_priv) +static void skl_ctx_workarounds_init(struct intel_engine_cs *engine) { - int ret; - - ret = gen9_ctx_workarounds_init(dev_priv); - if (ret) - return ret; - - return skl_tune_iz_hashing(dev_priv); + gen9_ctx_workarounds_init(engine); + skl_tune_iz_hashing(engine); } -static int bxt_ctx_workarounds_init(struct drm_i915_private *dev_priv) +static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine) { - int ret; + struct i915_wa_list *wal = &engine->ctx_wa_list; - ret = gen9_ctx_workarounds_init(dev_priv); - if (ret) - return ret; + gen9_ctx_workarounds_init(engine); /* WaDisableThreadStallDopClockGating:bxt */ WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, @@ -386,57 +428,41 @@ static int bxt_ctx_workarounds_init(struct drm_i915_private *dev_priv) /* WaToEnableHwFixForPushConstHWBug:bxt */ WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2, GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION); - - return 0; } -static int kbl_ctx_workarounds_init(struct drm_i915_private *dev_priv) +static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine) { - int ret; - - ret = gen9_ctx_workarounds_init(dev_priv); - if (ret) - return ret; + struct drm_i915_private *i915 = engine->i915; + struct i915_wa_list *wal = &engine->ctx_wa_list; - /* WaDisableFenceDestinationToSLM:kbl (pre-prod) */ - if (IS_KBL_REVID(dev_priv, KBL_REVID_A0, KBL_REVID_A0)) - WA_SET_BIT_MASKED(HDC_CHICKEN0, - HDC_FENCE_DEST_SLM_DISABLE); + gen9_ctx_workarounds_init(engine); /* WaToEnableHwFixForPushConstHWBug:kbl */ - if (IS_KBL_REVID(dev_priv, KBL_REVID_C0, REVID_FOREVER)) + if (IS_KBL_REVID(i915, KBL_REVID_C0, REVID_FOREVER)) WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2, GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION); /* WaDisableSbeCacheDispatchPortSharing:kbl */ WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1, GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE); - - return 0; } -static int glk_ctx_workarounds_init(struct drm_i915_private *dev_priv) +static void glk_ctx_workarounds_init(struct intel_engine_cs *engine) { - int ret; + struct i915_wa_list *wal = &engine->ctx_wa_list; - ret = gen9_ctx_workarounds_init(dev_priv); - if (ret) - return ret; + gen9_ctx_workarounds_init(engine); /* WaToEnableHwFixForPushConstHWBug:glk */ WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2, GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION); - - return 0; } -static int cfl_ctx_workarounds_init(struct drm_i915_private *dev_priv) +static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine) { - int ret; + struct i915_wa_list *wal = &engine->ctx_wa_list; - ret = gen9_ctx_workarounds_init(dev_priv); - if (ret) - return ret; + gen9_ctx_workarounds_init(engine); /* WaToEnableHwFixForPushConstHWBug:cfl */ WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2, @@ -445,18 +471,19 @@ static int cfl_ctx_workarounds_init(struct drm_i915_private *dev_priv) /* WaDisableSbeCacheDispatchPortSharing:cfl */ WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1, GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE); - - return 0; } -static int cnl_ctx_workarounds_init(struct drm_i915_private *dev_priv) +static void cnl_ctx_workarounds_init(struct intel_engine_cs *engine) { + struct drm_i915_private *i915 = engine->i915; + struct i915_wa_list *wal = &engine->ctx_wa_list; + /* WaForceContextSaveRestoreNonCoherent:cnl */ WA_SET_BIT_MASKED(CNL_HDC_CHICKEN0, HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT); /* WaThrottleEUPerfToAvoidTDBackPressure:cnl(pre-prod) */ - if (IS_CNL_REVID(dev_priv, CNL_REVID_B0, CNL_REVID_B0)) + if (IS_CNL_REVID(i915, CNL_REVID_B0, CNL_REVID_B0)) WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, THROTTLE_12_5); /* WaDisableReplayBufferBankArbitrationOptimization:cnl */ @@ -464,7 +491,7 @@ static int cnl_ctx_workarounds_init(struct drm_i915_private *dev_priv) GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION); /* WaDisableEnhancedSBEVertexCaching:cnl (pre-prod) */ - if (IS_CNL_REVID(dev_priv, 0, CNL_REVID_B0)) + if (IS_CNL_REVID(i915, 0, CNL_REVID_B0)) WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2, GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE); @@ -484,16 +511,17 @@ static int cnl_ctx_workarounds_init(struct drm_i915_private *dev_priv) /* WaDisableEarlyEOT:cnl */ WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, DISABLE_EARLY_EOT); - - return 0; } -static int icl_ctx_workarounds_init(struct drm_i915_private *dev_priv) +static void icl_ctx_workarounds_init(struct intel_engine_cs *engine) { + struct drm_i915_private *i915 = engine->i915; + struct i915_wa_list *wal = &engine->ctx_wa_list; + /* Wa_1604370585:icl (pre-prod) * Formerly known as WaPushConstantDereferenceHoldDisable */ - if (IS_ICL_REVID(dev_priv, ICL_REVID_A0, ICL_REVID_B0)) + if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0)) WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE); @@ -509,7 +537,7 @@ static int icl_ctx_workarounds_init(struct drm_i915_private *dev_priv) /* Wa_2006611047:icl (pre-prod) * Formerly known as WaDisableImprovedTdlClkGating */ - if (IS_ICL_REVID(dev_priv, ICL_REVID_A0, ICL_REVID_A0)) + if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0)) WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, GEN11_TDL_CLOCK_GATING_FIX_DISABLE); @@ -518,70 +546,73 @@ static int icl_ctx_workarounds_init(struct drm_i915_private *dev_priv) GEN11_STATE_CACHE_REDIRECT_TO_CS); /* Wa_2006665173:icl (pre-prod) */ - if (IS_ICL_REVID(dev_priv, ICL_REVID_A0, ICL_REVID_A0)) + if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0)) WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3, GEN11_BLEND_EMB_FIX_DISABLE_IN_RCC); - return 0; + /* WaEnableFloatBlendOptimization:icl */ + wa_write_masked_or(wal, + GEN10_CACHE_MODE_SS, + 0, /* write-only, so skip validation */ + _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE)); } -int intel_ctx_workarounds_init(struct drm_i915_private *dev_priv) +void intel_engine_init_ctx_wa(struct intel_engine_cs *engine) { - int err = 0; - - dev_priv->workarounds.count = 0; - - if (INTEL_GEN(dev_priv) < 8) - err = 0; - else if (IS_BROADWELL(dev_priv)) - err = bdw_ctx_workarounds_init(dev_priv); - else if (IS_CHERRYVIEW(dev_priv)) - err = chv_ctx_workarounds_init(dev_priv); - else if (IS_SKYLAKE(dev_priv)) - err = skl_ctx_workarounds_init(dev_priv); - else if (IS_BROXTON(dev_priv)) - err = bxt_ctx_workarounds_init(dev_priv); - else if (IS_KABYLAKE(dev_priv)) - err = kbl_ctx_workarounds_init(dev_priv); - else if (IS_GEMINILAKE(dev_priv)) - err = glk_ctx_workarounds_init(dev_priv); - else if (IS_COFFEELAKE(dev_priv)) - err = cfl_ctx_workarounds_init(dev_priv); - else if (IS_CANNONLAKE(dev_priv)) - err = cnl_ctx_workarounds_init(dev_priv); - else if (IS_ICELAKE(dev_priv)) - err = icl_ctx_workarounds_init(dev_priv); + struct drm_i915_private *i915 = engine->i915; + struct i915_wa_list *wal = &engine->ctx_wa_list; + + wa_init_start(wal, "context"); + + if (INTEL_GEN(i915) < 8) + return; + else if (IS_BROADWELL(i915)) + bdw_ctx_workarounds_init(engine); + else if (IS_CHERRYVIEW(i915)) + chv_ctx_workarounds_init(engine); + else if (IS_SKYLAKE(i915)) + skl_ctx_workarounds_init(engine); + else if (IS_BROXTON(i915)) + bxt_ctx_workarounds_init(engine); + else if (IS_KABYLAKE(i915)) + kbl_ctx_workarounds_init(engine); + else if (IS_GEMINILAKE(i915)) + glk_ctx_workarounds_init(engine); + else if (IS_COFFEELAKE(i915)) + cfl_ctx_workarounds_init(engine); + else if (IS_CANNONLAKE(i915)) + cnl_ctx_workarounds_init(engine); + else if (IS_ICELAKE(i915)) + icl_ctx_workarounds_init(engine); else - MISSING_CASE(INTEL_GEN(dev_priv)); - if (err) - return err; + MISSING_CASE(INTEL_GEN(i915)); - DRM_DEBUG_DRIVER("Number of context specific w/a: %d\n", - dev_priv->workarounds.count); - return 0; + wa_init_finish(wal); } -int intel_ctx_workarounds_emit(struct i915_request *rq) +int intel_engine_emit_ctx_wa(struct i915_request *rq) { - struct i915_workarounds *w = &rq->i915->workarounds; + struct i915_wa_list *wal = &rq->engine->ctx_wa_list; + struct i915_wa *wa; + unsigned int i; u32 *cs; - int ret, i; + int ret; - if (w->count == 0) + if (wal->count == 0) return 0; ret = rq->engine->emit_flush(rq, EMIT_BARRIER); if (ret) return ret; - cs = intel_ring_begin(rq, (w->count * 2 + 2)); + cs = intel_ring_begin(rq, (wal->count * 2 + 2)); if (IS_ERR(cs)) return PTR_ERR(cs); - *cs++ = MI_LOAD_REGISTER_IMM(w->count); - for (i = 0; i < w->count; i++) { - *cs++ = w->reg[i].addr; - *cs++ = w->reg[i].value; + *cs++ = MI_LOAD_REGISTER_IMM(wal->count); + for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { + *cs++ = i915_mmio_reg_offset(wa->reg); + *cs++ = wa->val; } *cs++ = MI_NOOP; @@ -595,72 +626,8 @@ int intel_ctx_workarounds_emit(struct i915_request *rq) } static void -wal_add(struct i915_wa_list *wal, const struct i915_wa *wa) +gen9_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) { - const unsigned int grow = 1 << 4; - - GEM_BUG_ON(!is_power_of_2(grow)); - - if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */ - struct i915_wa *list; - - list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa), - GFP_KERNEL); - if (!list) { - DRM_ERROR("No space for workaround init!\n"); - return; - } - - if (wal->list) - memcpy(list, wal->list, sizeof(*wa) * wal->count); - - wal->list = list; - } - - wal->list[wal->count++] = *wa; -} - -static void -wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val) -{ - struct i915_wa wa = { - .reg = reg, - .mask = val, - .val = _MASKED_BIT_ENABLE(val) - }; - - wal_add(wal, &wa); -} - -static void -wa_write_masked_or(struct i915_wa_list *wal, i915_reg_t reg, u32 mask, - u32 val) -{ - struct i915_wa wa = { - .reg = reg, - .mask = mask, - .val = val - }; - - wal_add(wal, &wa); -} - -static void -wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 val) -{ - wa_write_masked_or(wal, reg, ~0, val); -} - -static void -wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 val) -{ - wa_write_masked_or(wal, reg, val, val); -} - -static void gen9_gt_workarounds_init(struct drm_i915_private *i915) -{ - struct i915_wa_list *wal = &i915->gt_wa_list; - /* WaDisableKillLogic:bxt,skl,kbl */ if (!IS_COFFEELAKE(i915)) wa_write_or(wal, @@ -684,11 +651,10 @@ static void gen9_gt_workarounds_init(struct drm_i915_private *i915) BDW_DISABLE_HDC_INVALIDATION); } -static void skl_gt_workarounds_init(struct drm_i915_private *i915) +static void +skl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) { - struct i915_wa_list *wal = &i915->gt_wa_list; - - gen9_gt_workarounds_init(i915); + gen9_gt_workarounds_init(i915, wal); /* WaDisableGafsUnitClkGating:skl */ wa_write_or(wal, @@ -702,11 +668,10 @@ static void skl_gt_workarounds_init(struct drm_i915_private *i915) GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); } -static void bxt_gt_workarounds_init(struct drm_i915_private *i915) +static void +bxt_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) { - struct i915_wa_list *wal = &i915->gt_wa_list; - - gen9_gt_workarounds_init(i915); + gen9_gt_workarounds_init(i915, wal); /* WaInPlaceDecompressionHang:bxt */ wa_write_or(wal, @@ -714,11 +679,10 @@ static void bxt_gt_workarounds_init(struct drm_i915_private *i915) GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); } -static void kbl_gt_workarounds_init(struct drm_i915_private *i915) +static void +kbl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) { - struct i915_wa_list *wal = &i915->gt_wa_list; - - gen9_gt_workarounds_init(i915); + gen9_gt_workarounds_init(i915, wal); /* WaDisableDynamicCreditSharing:kbl */ if (IS_KBL_REVID(i915, 0, KBL_REVID_B0)) @@ -737,16 +701,16 @@ static void kbl_gt_workarounds_init(struct drm_i915_private *i915) GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); } -static void glk_gt_workarounds_init(struct drm_i915_private *i915) +static void +glk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) { - gen9_gt_workarounds_init(i915); + gen9_gt_workarounds_init(i915, wal); } -static void cfl_gt_workarounds_init(struct drm_i915_private *i915) +static void +cfl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) { - struct i915_wa_list *wal = &i915->gt_wa_list; - - gen9_gt_workarounds_init(i915); + gen9_gt_workarounds_init(i915, wal); /* WaDisableGafsUnitClkGating:cfl */ wa_write_or(wal, @@ -759,10 +723,10 @@ static void cfl_gt_workarounds_init(struct drm_i915_private *i915) GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); } -static void wa_init_mcr(struct drm_i915_private *dev_priv) +static void +wa_init_mcr(struct drm_i915_private *dev_priv, struct i915_wa_list *wal) { - const struct sseu_dev_info *sseu = &(INTEL_INFO(dev_priv)->sseu); - struct i915_wa_list *wal = &dev_priv->gt_wa_list; + const struct sseu_dev_info *sseu = &RUNTIME_INFO(dev_priv)->sseu; u32 mcr_slice_subslice_mask; /* @@ -822,11 +786,10 @@ static void wa_init_mcr(struct drm_i915_private *dev_priv) intel_calculate_mcr_s_ss_select(dev_priv)); } -static void cnl_gt_workarounds_init(struct drm_i915_private *i915) +static void +cnl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) { - struct i915_wa_list *wal = &i915->gt_wa_list; - - wa_init_mcr(i915); + wa_init_mcr(i915, wal); /* WaDisableI2mCycleOnWRPort:cnl (pre-prod) */ if (IS_CNL_REVID(i915, CNL_REVID_B0, CNL_REVID_B0)) @@ -840,11 +803,10 @@ static void cnl_gt_workarounds_init(struct drm_i915_private *i915) GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); } -static void icl_gt_workarounds_init(struct drm_i915_private *i915) +static void +icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) { - struct i915_wa_list *wal = &i915->gt_wa_list; - - wa_init_mcr(i915); + wa_init_mcr(i915, wal); /* WaInPlaceDecompressionHang:icl */ wa_write_or(wal, @@ -897,12 +859,9 @@ static void icl_gt_workarounds_init(struct drm_i915_private *i915) GAMT_CHKN_DISABLE_L3_COH_PIPE); } -void intel_gt_init_workarounds(struct drm_i915_private *i915) +static void +gt_init_workarounds(struct drm_i915_private *i915, struct i915_wa_list *wal) { - struct i915_wa_list *wal = &i915->gt_wa_list; - - wa_init_start(wal, "GT"); - if (INTEL_GEN(i915) < 8) return; else if (IS_BROADWELL(i915)) @@ -910,22 +869,29 @@ void intel_gt_init_workarounds(struct drm_i915_private *i915) else if (IS_CHERRYVIEW(i915)) return; else if (IS_SKYLAKE(i915)) - skl_gt_workarounds_init(i915); + skl_gt_workarounds_init(i915, wal); else if (IS_BROXTON(i915)) - bxt_gt_workarounds_init(i915); + bxt_gt_workarounds_init(i915, wal); else if (IS_KABYLAKE(i915)) - kbl_gt_workarounds_init(i915); + kbl_gt_workarounds_init(i915, wal); else if (IS_GEMINILAKE(i915)) - glk_gt_workarounds_init(i915); + glk_gt_workarounds_init(i915, wal); else if (IS_COFFEELAKE(i915)) - cfl_gt_workarounds_init(i915); + cfl_gt_workarounds_init(i915, wal); else if (IS_CANNONLAKE(i915)) - cnl_gt_workarounds_init(i915); + cnl_gt_workarounds_init(i915, wal); else if (IS_ICELAKE(i915)) - icl_gt_workarounds_init(i915); + icl_gt_workarounds_init(i915, wal); else MISSING_CASE(INTEL_GEN(i915)); +} +void intel_gt_init_workarounds(struct drm_i915_private *i915) +{ + struct i915_wa_list *wal = &i915->gt_wa_list; + + wa_init_start(wal, "GT"); + gt_init_workarounds(i915, wal); wa_init_finish(wal); } @@ -973,8 +939,6 @@ wa_list_apply(struct drm_i915_private *dev_priv, const struct i915_wa_list *wal) intel_uncore_forcewake_put__locked(dev_priv, fw); spin_unlock_irqrestore(&dev_priv->uncore.lock, flags); - - DRM_DEBUG_DRIVER("Applied %u %s workarounds\n", wal->count, wal->name); } void intel_gt_apply_workarounds(struct drm_i915_private *dev_priv) @@ -982,29 +946,54 @@ void intel_gt_apply_workarounds(struct drm_i915_private *dev_priv) wa_list_apply(dev_priv, &dev_priv->gt_wa_list); } -struct whitelist { - i915_reg_t reg[RING_MAX_NONPRIV_SLOTS]; - unsigned int count; - u32 nopid; -}; +static bool +wa_verify(const struct i915_wa *wa, u32 cur, const char *name, const char *from) +{ + if ((cur ^ wa->val) & wa->mask) { + DRM_ERROR("%s workaround lost on %s! (%x=%x/%x, expected %x, mask=%x)\n", + name, from, i915_mmio_reg_offset(wa->reg), cur, + cur & wa->mask, wa->val, wa->mask); -static void whitelist_reg(struct whitelist *w, i915_reg_t reg) + return false; + } + + return true; +} + +static bool wa_list_verify(struct drm_i915_private *dev_priv, + const struct i915_wa_list *wal, + const char *from) { - if (GEM_WARN_ON(w->count >= RING_MAX_NONPRIV_SLOTS)) - return; + struct i915_wa *wa; + unsigned int i; + bool ok = true; + + for (i = 0, wa = wal->list; i < wal->count; i++, wa++) + ok &= wa_verify(wa, I915_READ(wa->reg), wal->name, from); - w->reg[w->count++] = reg; + return ok; } -static void bdw_whitelist_build(struct whitelist *w) +bool intel_gt_verify_workarounds(struct drm_i915_private *dev_priv, + const char *from) { + return wa_list_verify(dev_priv, &dev_priv->gt_wa_list, from); } -static void chv_whitelist_build(struct whitelist *w) +static void +whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg) { + struct i915_wa wa = { + .reg = reg + }; + + if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS)) + return; + + _wa_add(wal, &wa); } -static void gen9_whitelist_build(struct whitelist *w) +static void gen9_whitelist_build(struct i915_wa_list *w) { /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */ whitelist_reg(w, GEN9_CTX_PREEMPT_REG); @@ -1016,7 +1005,7 @@ static void gen9_whitelist_build(struct whitelist *w) whitelist_reg(w, GEN8_HDC_CHICKEN1); } -static void skl_whitelist_build(struct whitelist *w) +static void skl_whitelist_build(struct i915_wa_list *w) { gen9_whitelist_build(w); @@ -1024,12 +1013,12 @@ static void skl_whitelist_build(struct whitelist *w) whitelist_reg(w, GEN8_L3SQCREG4); } -static void bxt_whitelist_build(struct whitelist *w) +static void bxt_whitelist_build(struct i915_wa_list *w) { gen9_whitelist_build(w); } -static void kbl_whitelist_build(struct whitelist *w) +static void kbl_whitelist_build(struct i915_wa_list *w) { gen9_whitelist_build(w); @@ -1037,7 +1026,7 @@ static void kbl_whitelist_build(struct whitelist *w) whitelist_reg(w, GEN8_L3SQCREG4); } -static void glk_whitelist_build(struct whitelist *w) +static void glk_whitelist_build(struct i915_wa_list *w) { gen9_whitelist_build(w); @@ -1045,37 +1034,41 @@ static void glk_whitelist_build(struct whitelist *w) whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1); } -static void cfl_whitelist_build(struct whitelist *w) +static void cfl_whitelist_build(struct i915_wa_list *w) { gen9_whitelist_build(w); } -static void cnl_whitelist_build(struct whitelist *w) +static void cnl_whitelist_build(struct i915_wa_list *w) { /* WaEnablePreemptionGranularityControlByUMD:cnl */ whitelist_reg(w, GEN8_CS_CHICKEN1); } -static void icl_whitelist_build(struct whitelist *w) +static void icl_whitelist_build(struct i915_wa_list *w) { + /* WaAllowUMDToModifyHalfSliceChicken7:icl */ + whitelist_reg(w, GEN9_HALF_SLICE_CHICKEN7); + + /* WaAllowUMDToModifySamplerMode:icl */ + whitelist_reg(w, GEN10_SAMPLER_MODE); } -static struct whitelist *whitelist_build(struct intel_engine_cs *engine, - struct whitelist *w) +void intel_engine_init_whitelist(struct intel_engine_cs *engine) { struct drm_i915_private *i915 = engine->i915; + struct i915_wa_list *w = &engine->whitelist; GEM_BUG_ON(engine->id != RCS); - w->count = 0; - w->nopid = i915_mmio_reg_offset(RING_NOPID(engine->mmio_base)); + wa_init_start(w, "whitelist"); if (INTEL_GEN(i915) < 8) - return NULL; + return; else if (IS_BROADWELL(i915)) - bdw_whitelist_build(w); + return; else if (IS_CHERRYVIEW(i915)) - chv_whitelist_build(w); + return; else if (IS_SKYLAKE(i915)) skl_whitelist_build(w); else if (IS_BROXTON(i915)) @@ -1093,43 +1086,34 @@ static struct whitelist *whitelist_build(struct intel_engine_cs *engine, else MISSING_CASE(INTEL_GEN(i915)); - return w; + wa_init_finish(w); } -static void whitelist_apply(struct intel_engine_cs *engine, - const struct whitelist *w) +void intel_engine_apply_whitelist(struct intel_engine_cs *engine) { struct drm_i915_private *dev_priv = engine->i915; + const struct i915_wa_list *wal = &engine->whitelist; const u32 base = engine->mmio_base; + struct i915_wa *wa; unsigned int i; - if (!w) + if (!wal->count) return; - intel_uncore_forcewake_get(engine->i915, FORCEWAKE_ALL); - - for (i = 0; i < w->count; i++) - I915_WRITE_FW(RING_FORCE_TO_NONPRIV(base, i), - i915_mmio_reg_offset(w->reg[i])); + for (i = 0, wa = wal->list; i < wal->count; i++, wa++) + I915_WRITE(RING_FORCE_TO_NONPRIV(base, i), + i915_mmio_reg_offset(wa->reg)); /* And clear the rest just in case of garbage */ for (; i < RING_MAX_NONPRIV_SLOTS; i++) - I915_WRITE_FW(RING_FORCE_TO_NONPRIV(base, i), w->nopid); - - intel_uncore_forcewake_put(engine->i915, FORCEWAKE_ALL); -} - -void intel_whitelist_workarounds_apply(struct intel_engine_cs *engine) -{ - struct whitelist w; - - whitelist_apply(engine, whitelist_build(engine, &w)); + I915_WRITE(RING_FORCE_TO_NONPRIV(base, i), + i915_mmio_reg_offset(RING_NOPID(base))); } -static void rcs_engine_wa_init(struct intel_engine_cs *engine) +static void +rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { struct drm_i915_private *i915 = engine->i915; - struct i915_wa_list *wal = &engine->wa_list; if (IS_ICELAKE(i915)) { /* This is not an Wa. Enable for better image quality */ @@ -1171,20 +1155,22 @@ static void rcs_engine_wa_init(struct intel_engine_cs *engine) GEN8_L3SQCREG4, GEN11_LQSC_CLEAN_EVICT_DISABLE); - /* Wa_1604302699:icl */ - wa_write_or(wal, - GEN10_L3_CHICKEN_MODE_REGISTER, - GEN11_I2M_WRITE_DISABLE); - /* WaForwardProgressSoftReset:icl */ wa_write_or(wal, GEN10_SCRATCH_LNCF2, PMFLUSHDONE_LNICRSDROP | PMFLUSH_GAPL3UNBLOCK | PMFLUSHDONE_LNEBLK); + + /* Wa_1406609255:icl (pre-prod) */ + if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0)) + wa_write_or(wal, + GEN7_SARCHKMD, + GEN7_DISABLE_DEMAND_PREFETCH | + GEN7_DISABLE_SAMPLER_PREFETCH); } - if (IS_GEN9(i915) || IS_CANNONLAKE(i915)) { + if (IS_GEN(i915, 9) || IS_CANNONLAKE(i915)) { /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,cnl */ wa_masked_en(wal, GEN7_FF_SLICE_CS_CHICKEN1, @@ -1205,7 +1191,7 @@ static void rcs_engine_wa_init(struct intel_engine_cs *engine) GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE); } - if (IS_GEN9(i915)) { + if (IS_GEN(i915, 9)) { /* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */ wa_masked_en(wal, GEN9_CSFE_CHICKEN1_RCS, @@ -1231,10 +1217,10 @@ static void rcs_engine_wa_init(struct intel_engine_cs *engine) } } -static void xcs_engine_wa_init(struct intel_engine_cs *engine) +static void +xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { struct drm_i915_private *i915 = engine->i915; - struct i915_wa_list *wal = &engine->wa_list; /* WaKBLVECSSemaphoreWaitPoll:kbl */ if (IS_KBL_REVID(i915, KBL_REVID_A0, KBL_REVID_E0)) { @@ -1244,6 +1230,18 @@ static void xcs_engine_wa_init(struct intel_engine_cs *engine) } } +static void +engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal) +{ + if (I915_SELFTEST_ONLY(INTEL_GEN(engine->i915) < 8)) + return; + + if (engine->id == RCS) + rcs_engine_wa_init(engine, wal); + else + xcs_engine_wa_init(engine, wal); +} + void intel_engine_init_workarounds(struct intel_engine_cs *engine) { struct i915_wa_list *wal = &engine->wa_list; @@ -1252,12 +1250,7 @@ void intel_engine_init_workarounds(struct intel_engine_cs *engine) return; wa_init_start(wal, engine->name); - - if (engine->id == RCS) - rcs_engine_wa_init(engine); - else - xcs_engine_wa_init(engine); - + engine_init_workarounds(engine, wal); wa_init_finish(wal); } diff --git a/drivers/gpu/drm/i915/intel_workarounds.h b/drivers/gpu/drm/i915/intel_workarounds.h index 979695a53964..7c734714b05e 100644 --- a/drivers/gpu/drm/i915/intel_workarounds.h +++ b/drivers/gpu/drm/i915/intel_workarounds.h @@ -19,6 +19,7 @@ struct i915_wa_list { const char *name; struct i915_wa *list; unsigned int count; + unsigned int wa_count; }; static inline void intel_wa_list_free(struct i915_wa_list *wal) @@ -27,13 +28,16 @@ static inline void intel_wa_list_free(struct i915_wa_list *wal) memset(wal, 0, sizeof(*wal)); } -int intel_ctx_workarounds_init(struct drm_i915_private *dev_priv); -int intel_ctx_workarounds_emit(struct i915_request *rq); +void intel_engine_init_ctx_wa(struct intel_engine_cs *engine); +int intel_engine_emit_ctx_wa(struct i915_request *rq); void intel_gt_init_workarounds(struct drm_i915_private *dev_priv); void intel_gt_apply_workarounds(struct drm_i915_private *dev_priv); +bool intel_gt_verify_workarounds(struct drm_i915_private *dev_priv, + const char *from); -void intel_whitelist_workarounds_apply(struct intel_engine_cs *engine); +void intel_engine_init_whitelist(struct intel_engine_cs *engine); +void intel_engine_apply_whitelist(struct intel_engine_cs *engine); void intel_engine_init_workarounds(struct intel_engine_cs *engine); void intel_engine_apply_workarounds(struct intel_engine_cs *engine); diff --git a/drivers/gpu/drm/i915/selftests/huge_pages.c b/drivers/gpu/drm/i915/selftests/huge_pages.c index 5c22f2c8d4cf..a9a2fa35876f 100644 --- a/drivers/gpu/drm/i915/selftests/huge_pages.c +++ b/drivers/gpu/drm/i915/selftests/huge_pages.c @@ -972,7 +972,6 @@ static int gpu_write(struct i915_vma *vma, { struct i915_request *rq; struct i915_vma *batch; - int flags = 0; int err; GEM_BUG_ON(!intel_engine_can_store_dword(engine)); @@ -981,14 +980,14 @@ static int gpu_write(struct i915_vma *vma, if (err) return err; - rq = i915_request_alloc(engine, ctx); - if (IS_ERR(rq)) - return PTR_ERR(rq); - batch = gpu_write_dw(vma, dword * sizeof(u32), value); - if (IS_ERR(batch)) { - err = PTR_ERR(batch); - goto err_request; + if (IS_ERR(batch)) + return PTR_ERR(batch); + + rq = i915_request_alloc(engine, ctx); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto err_batch; } err = i915_vma_move_to_active(batch, rq, 0); @@ -996,21 +995,21 @@ static int gpu_write(struct i915_vma *vma, goto err_request; i915_gem_object_set_active_reference(batch->obj); - i915_vma_unpin(batch); - i915_vma_close(batch); - err = engine->emit_bb_start(rq, - batch->node.start, batch->node.size, - flags); + err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); if (err) goto err_request; - err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); + err = engine->emit_bb_start(rq, + batch->node.start, batch->node.size, + 0); +err_request: if (err) i915_request_skip(rq, err); - -err_request: i915_request_add(rq); +err_batch: + i915_vma_unpin(batch); + i915_vma_close(batch); return err; } @@ -1135,7 +1134,8 @@ static int igt_write_huge(struct i915_gem_context *ctx, n = 0; for_each_engine(engine, i915, id) { if (!intel_engine_can_store_dword(engine)) { - pr_info("store-dword-imm not supported on engine=%u\n", id); + pr_info("store-dword-imm not supported on engine=%u\n", + id); continue; } engines[n++] = engine; @@ -1167,17 +1167,30 @@ static int igt_write_huge(struct i915_gem_context *ctx, engine = engines[order[i] % n]; i = (i + 1) % (n * I915_NUM_ENGINES); - err = __igt_write_huge(ctx, engine, obj, size, offset_low, dword, num + 1); + /* + * In order to utilize 64K pages we need to both pad the vma + * size and ensure the vma offset is at the start of the pt + * boundary, however to improve coverage we opt for testing both + * aligned and unaligned offsets. + */ + if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K) + offset_low = round_down(offset_low, + I915_GTT_PAGE_SIZE_2M); + + err = __igt_write_huge(ctx, engine, obj, size, offset_low, + dword, num + 1); if (err) break; - err = __igt_write_huge(ctx, engine, obj, size, offset_high, dword, num + 1); + err = __igt_write_huge(ctx, engine, obj, size, offset_high, + dword, num + 1); if (err) break; if (igt_timeout(end_time, "%s timed out on engine=%u, offset_low=%llx offset_high=%llx, max_page_size=%x\n", - __func__, engine->id, offset_low, offset_high, max_page_size)) + __func__, engine->id, offset_low, offset_high, + max_page_size)) break; } @@ -1436,7 +1449,7 @@ static int igt_ppgtt_pin_update(void *arg) * huge-gtt-pages. */ - if (!USES_FULL_48BIT_PPGTT(dev_priv)) { + if (!ppgtt || !i915_vm_is_48bit(&ppgtt->vm)) { pr_info("48b PPGTT not supported, skipping\n"); return 0; } @@ -1687,9 +1700,7 @@ int i915_gem_huge_page_mock_selftests(void) SUBTEST(igt_mock_ppgtt_huge_fill), SUBTEST(igt_mock_ppgtt_64K), }; - int saved_ppgtt = i915_modparams.enable_ppgtt; struct drm_i915_private *dev_priv; - struct pci_dev *pdev; struct i915_hw_ppgtt *ppgtt; int err; @@ -1698,10 +1709,7 @@ int i915_gem_huge_page_mock_selftests(void) return -ENOMEM; /* Pretend to be a device which supports the 48b PPGTT */ - i915_modparams.enable_ppgtt = 3; - - pdev = dev_priv->drm.pdev; - dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(39)); + mkwrite_device_info(dev_priv)->ppgtt = INTEL_PPGTT_FULL_4LVL; mutex_lock(&dev_priv->drm.struct_mutex); ppgtt = i915_ppgtt_create(dev_priv, ERR_PTR(-ENODEV)); @@ -1731,9 +1739,6 @@ out_close: out_unlock: mutex_unlock(&dev_priv->drm.struct_mutex); - - i915_modparams.enable_ppgtt = saved_ppgtt; - drm_dev_put(&dev_priv->drm); return err; @@ -1751,9 +1756,10 @@ int i915_gem_huge_page_live_selftests(struct drm_i915_private *dev_priv) }; struct drm_file *file; struct i915_gem_context *ctx; + intel_wakeref_t wakeref; int err; - if (!USES_PPGTT(dev_priv)) { + if (!HAS_PPGTT(dev_priv)) { pr_info("PPGTT not supported, skipping live-selftests\n"); return 0; } @@ -1766,7 +1772,7 @@ int i915_gem_huge_page_live_selftests(struct drm_i915_private *dev_priv) return PTR_ERR(file); mutex_lock(&dev_priv->drm.struct_mutex); - intel_runtime_pm_get(dev_priv); + wakeref = intel_runtime_pm_get(dev_priv); ctx = live_context(dev_priv, file); if (IS_ERR(ctx)) { @@ -1780,7 +1786,7 @@ int i915_gem_huge_page_live_selftests(struct drm_i915_private *dev_priv) err = i915_subtests(tests, ctx); out_unlock: - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, wakeref); mutex_unlock(&dev_priv->drm.struct_mutex); mock_file_free(dev_priv, file); diff --git a/drivers/gpu/drm/i915/selftests/i915_active.c b/drivers/gpu/drm/i915/selftests/i915_active.c new file mode 100644 index 000000000000..337b1f98b923 --- /dev/null +++ b/drivers/gpu/drm/i915/selftests/i915_active.c @@ -0,0 +1,157 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright © 2018 Intel Corporation + */ + +#include "../i915_selftest.h" + +#include "igt_flush_test.h" +#include "lib_sw_fence.h" + +struct live_active { + struct i915_active base; + bool retired; +}; + +static void __live_active_retire(struct i915_active *base) +{ + struct live_active *active = container_of(base, typeof(*active), base); + + active->retired = true; +} + +static int __live_active_setup(struct drm_i915_private *i915, + struct live_active *active) +{ + struct intel_engine_cs *engine; + struct i915_sw_fence *submit; + enum intel_engine_id id; + unsigned int count = 0; + int err = 0; + + submit = heap_fence_create(GFP_KERNEL); + if (!submit) + return -ENOMEM; + + i915_active_init(i915, &active->base, __live_active_retire); + active->retired = false; + + if (!i915_active_acquire(&active->base)) { + pr_err("First i915_active_acquire should report being idle\n"); + err = -EINVAL; + goto out; + } + + for_each_engine(engine, i915, id) { + struct i915_request *rq; + + rq = i915_request_alloc(engine, i915->kernel_context); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + break; + } + + err = i915_sw_fence_await_sw_fence_gfp(&rq->submit, + submit, + GFP_KERNEL); + if (err >= 0) + err = i915_active_ref(&active->base, + rq->fence.context, rq); + i915_request_add(rq); + if (err) { + pr_err("Failed to track active ref!\n"); + break; + } + + count++; + } + + i915_active_release(&active->base); + if (active->retired && count) { + pr_err("i915_active retired before submission!\n"); + err = -EINVAL; + } + if (active->base.count != count) { + pr_err("i915_active not tracking all requests, found %d, expected %d\n", + active->base.count, count); + err = -EINVAL; + } + +out: + i915_sw_fence_commit(submit); + heap_fence_put(submit); + + return err; +} + +static int live_active_wait(void *arg) +{ + struct drm_i915_private *i915 = arg; + struct live_active active; + intel_wakeref_t wakeref; + int err; + + /* Check that we get a callback when requests retire upon waiting */ + + mutex_lock(&i915->drm.struct_mutex); + wakeref = intel_runtime_pm_get(i915); + + err = __live_active_setup(i915, &active); + + i915_active_wait(&active.base); + if (!active.retired) { + pr_err("i915_active not retired after waiting!\n"); + err = -EINVAL; + } + + i915_active_fini(&active.base); + if (igt_flush_test(i915, I915_WAIT_LOCKED)) + err = -EIO; + + intel_runtime_pm_put(i915, wakeref); + mutex_unlock(&i915->drm.struct_mutex); + return err; +} + +static int live_active_retire(void *arg) +{ + struct drm_i915_private *i915 = arg; + struct live_active active; + intel_wakeref_t wakeref; + int err; + + /* Check that we get a callback when requests are indirectly retired */ + + mutex_lock(&i915->drm.struct_mutex); + wakeref = intel_runtime_pm_get(i915); + + err = __live_active_setup(i915, &active); + + /* waits for & retires all requests */ + if (igt_flush_test(i915, I915_WAIT_LOCKED)) + err = -EIO; + + if (!active.retired) { + pr_err("i915_active not retired after flushing!\n"); + err = -EINVAL; + } + + i915_active_fini(&active.base); + intel_runtime_pm_put(i915, wakeref); + mutex_unlock(&i915->drm.struct_mutex); + return err; +} + +int i915_active_live_selftests(struct drm_i915_private *i915) +{ + static const struct i915_subtest tests[] = { + SUBTEST(live_active_wait), + SUBTEST(live_active_retire), + }; + + if (i915_terminally_wedged(&i915->gpu_error)) + return 0; + + return i915_subtests(tests, i915); +} diff --git a/drivers/gpu/drm/i915/selftests/i915_gem.c b/drivers/gpu/drm/i915/selftests/i915_gem.c index d0aa19d17653..e77b7ed449ae 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem.c @@ -16,9 +16,10 @@ static int switch_to_context(struct drm_i915_private *i915, { struct intel_engine_cs *engine; enum intel_engine_id id; + intel_wakeref_t wakeref; int err = 0; - intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(i915); for_each_engine(engine, i915, id) { struct i915_request *rq; @@ -32,7 +33,7 @@ static int switch_to_context(struct drm_i915_private *i915, i915_request_add(rq); } - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); return err; } @@ -65,7 +66,9 @@ static void trash_stolen(struct drm_i915_private *i915) static void simulate_hibernate(struct drm_i915_private *i915) { - intel_runtime_pm_get(i915); + intel_wakeref_t wakeref; + + wakeref = intel_runtime_pm_get(i915); /* * As a final sting in the tail, invalidate stolen. Under a real S4, @@ -76,7 +79,7 @@ static void simulate_hibernate(struct drm_i915_private *i915) */ trash_stolen(i915); - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); } static int pm_prepare(struct drm_i915_private *i915) @@ -93,39 +96,39 @@ static int pm_prepare(struct drm_i915_private *i915) static void pm_suspend(struct drm_i915_private *i915) { - intel_runtime_pm_get(i915); - - i915_gem_suspend_gtt_mappings(i915); - i915_gem_suspend_late(i915); + intel_wakeref_t wakeref; - intel_runtime_pm_put(i915); + with_intel_runtime_pm(i915, wakeref) { + i915_gem_suspend_gtt_mappings(i915); + i915_gem_suspend_late(i915); + } } static void pm_hibernate(struct drm_i915_private *i915) { - intel_runtime_pm_get(i915); + intel_wakeref_t wakeref; - i915_gem_suspend_gtt_mappings(i915); + with_intel_runtime_pm(i915, wakeref) { + i915_gem_suspend_gtt_mappings(i915); - i915_gem_freeze(i915); - i915_gem_freeze_late(i915); - - intel_runtime_pm_put(i915); + i915_gem_freeze(i915); + i915_gem_freeze_late(i915); + } } static void pm_resume(struct drm_i915_private *i915) { + intel_wakeref_t wakeref; + /* * Both suspend and hibernate follow the same wakeup path and assume * that runtime-pm just works. */ - intel_runtime_pm_get(i915); - - intel_engines_sanitize(i915); - i915_gem_sanitize(i915); - i915_gem_resume(i915); - - intel_runtime_pm_put(i915); + with_intel_runtime_pm(i915, wakeref) { + intel_engines_sanitize(i915, false); + i915_gem_sanitize(i915); + i915_gem_resume(i915); + } } static int igt_gem_suspend(void *arg) diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c b/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c index f7392c1ffe75..fd89a5a33c1a 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c @@ -279,6 +279,7 @@ static int igt_gem_coherency(void *arg) struct drm_i915_private *i915 = arg; const struct igt_coherency_mode *read, *write, *over; struct drm_i915_gem_object *obj; + intel_wakeref_t wakeref; unsigned long count, n; u32 *offsets, *values; int err = 0; @@ -298,7 +299,7 @@ static int igt_gem_coherency(void *arg) values = offsets + ncachelines; mutex_lock(&i915->drm.struct_mutex); - intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(i915); for (over = igt_coherency_mode; over->name; over++) { if (!over->set) continue; @@ -376,7 +377,7 @@ static int igt_gem_coherency(void *arg) } } unlock: - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); mutex_unlock(&i915->drm.struct_mutex); kfree(offsets); return err; diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/selftests/i915_gem_context.c index 76df25aa90c9..7eb58a9d1319 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem_context.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem_context.c @@ -24,9 +24,13 @@ #include <linux/prime_numbers.h> +#include "../i915_reset.h" #include "../i915_selftest.h" #include "i915_random.h" #include "igt_flush_test.h" +#include "igt_live_test.h" +#include "igt_reset.h" +#include "igt_spinner.h" #include "mock_drm.h" #include "mock_gem_device.h" @@ -34,63 +38,6 @@ #define DW_PER_PAGE (PAGE_SIZE / sizeof(u32)) -struct live_test { - struct drm_i915_private *i915; - const char *func; - const char *name; - - unsigned int reset_count; -}; - -static int begin_live_test(struct live_test *t, - struct drm_i915_private *i915, - const char *func, - const char *name) -{ - int err; - - t->i915 = i915; - t->func = func; - t->name = name; - - err = i915_gem_wait_for_idle(i915, - I915_WAIT_LOCKED, - MAX_SCHEDULE_TIMEOUT); - if (err) { - pr_err("%s(%s): failed to idle before, with err=%d!", - func, name, err); - return err; - } - - i915->gpu_error.missed_irq_rings = 0; - t->reset_count = i915_reset_count(&i915->gpu_error); - - return 0; -} - -static int end_live_test(struct live_test *t) -{ - struct drm_i915_private *i915 = t->i915; - - if (igt_flush_test(i915, I915_WAIT_LOCKED)) - return -EIO; - - if (t->reset_count != i915_reset_count(&i915->gpu_error)) { - pr_err("%s(%s): GPU was reset %d times!\n", - t->func, t->name, - i915_reset_count(&i915->gpu_error) - t->reset_count); - return -EIO; - } - - if (i915->gpu_error.missed_irq_rings) { - pr_err("%s(%s): Missed interrupts on engines %lx\n", - t->func, t->name, i915->gpu_error.missed_irq_rings); - return -EIO; - } - - return 0; -} - static int live_nop_switch(void *arg) { const unsigned int nctx = 1024; @@ -98,8 +45,9 @@ static int live_nop_switch(void *arg) struct intel_engine_cs *engine; struct i915_gem_context **ctx; enum intel_engine_id id; + intel_wakeref_t wakeref; + struct igt_live_test t; struct drm_file *file; - struct live_test t; unsigned long n; int err = -ENODEV; @@ -119,7 +67,7 @@ static int live_nop_switch(void *arg) return PTR_ERR(file); mutex_lock(&i915->drm.struct_mutex); - intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(i915); ctx = kcalloc(nctx, sizeof(*ctx), GFP_KERNEL); if (!ctx) { @@ -163,7 +111,7 @@ static int live_nop_switch(void *arg) pr_info("Populated %d contexts on %s in %lluns\n", nctx, engine->name, ktime_to_ns(times[1] - times[0])); - err = begin_live_test(&t, i915, __func__, engine->name); + err = igt_live_test_begin(&t, i915, __func__, engine->name); if (err) goto out_unlock; @@ -211,7 +159,7 @@ static int live_nop_switch(void *arg) break; } - err = end_live_test(&t); + err = igt_live_test_end(&t); if (err) goto out_unlock; @@ -222,7 +170,7 @@ static int live_nop_switch(void *arg) } out_unlock: - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); mutex_unlock(&i915->drm.struct_mutex); mock_file_free(i915, file); return err; @@ -531,11 +479,11 @@ static int igt_ctx_exec(void *arg) { struct drm_i915_private *i915 = arg; struct drm_i915_gem_object *obj = NULL; + unsigned long ncontexts, ndwords, dw; + struct igt_live_test t; struct drm_file *file; IGT_TIMEOUT(end_time); LIST_HEAD(objects); - unsigned long ncontexts, ndwords, dw; - bool first_shared_gtt = true; int err = -ENODEV; /* @@ -553,6 +501,10 @@ static int igt_ctx_exec(void *arg) mutex_lock(&i915->drm.struct_mutex); + err = igt_live_test_begin(&t, i915, __func__, ""); + if (err) + goto out_unlock; + ncontexts = 0; ndwords = 0; dw = 0; @@ -561,18 +513,15 @@ static int igt_ctx_exec(void *arg) struct i915_gem_context *ctx; unsigned int id; - if (first_shared_gtt) { - ctx = __create_hw_context(i915, file->driver_priv); - first_shared_gtt = false; - } else { - ctx = i915_gem_create_context(i915, file->driver_priv); - } + ctx = i915_gem_create_context(i915, file->driver_priv); if (IS_ERR(ctx)) { err = PTR_ERR(ctx); goto out_unlock; } for_each_engine(engine, i915, id) { + intel_wakeref_t wakeref; + if (!engine->context_size) continue; /* No logical context support in HW */ @@ -587,9 +536,9 @@ static int igt_ctx_exec(void *arg) } } - intel_runtime_pm_get(i915); - err = gpu_fill(obj, ctx, engine, dw); - intel_runtime_pm_put(i915); + err = 0; + with_intel_runtime_pm(i915, wakeref) + err = gpu_fill(obj, ctx, engine, dw); if (err) { pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) in ctx %u [full-ppgtt? %s], err=%d\n", ndwords, dw, max_dwords(obj), @@ -607,7 +556,7 @@ static int igt_ctx_exec(void *arg) ncontexts++; } pr_info("Submitted %lu contexts (across %u engines), filling %lu dwords\n", - ncontexts, INTEL_INFO(i915)->num_rings, ndwords); + ncontexts, RUNTIME_INFO(i915)->num_rings, ndwords); dw = 0; list_for_each_entry(obj, &objects, st_link) { @@ -622,7 +571,7 @@ static int igt_ctx_exec(void *arg) } out_unlock: - if (igt_flush_test(i915, I915_WAIT_LOCKED)) + if (igt_live_test_end(&t)) err = -EIO; mutex_unlock(&i915->drm.struct_mutex); @@ -630,17 +579,480 @@ out_unlock: return err; } +static struct i915_vma *rpcs_query_batch(struct i915_vma *vma) +{ + struct drm_i915_gem_object *obj; + u32 *cmd; + int err; + + if (INTEL_GEN(vma->vm->i915) < 8) + return ERR_PTR(-EINVAL); + + obj = i915_gem_object_create_internal(vma->vm->i915, PAGE_SIZE); + if (IS_ERR(obj)) + return ERR_CAST(obj); + + cmd = i915_gem_object_pin_map(obj, I915_MAP_WB); + if (IS_ERR(cmd)) { + err = PTR_ERR(cmd); + goto err; + } + + *cmd++ = MI_STORE_REGISTER_MEM_GEN8; + *cmd++ = i915_mmio_reg_offset(GEN8_R_PWR_CLK_STATE); + *cmd++ = lower_32_bits(vma->node.start); + *cmd++ = upper_32_bits(vma->node.start); + *cmd = MI_BATCH_BUFFER_END; + + i915_gem_object_unpin_map(obj); + + err = i915_gem_object_set_to_gtt_domain(obj, false); + if (err) + goto err; + + vma = i915_vma_instance(obj, vma->vm, NULL); + if (IS_ERR(vma)) { + err = PTR_ERR(vma); + goto err; + } + + err = i915_vma_pin(vma, 0, 0, PIN_USER); + if (err) + goto err; + + return vma; + +err: + i915_gem_object_put(obj); + return ERR_PTR(err); +} + +static int +emit_rpcs_query(struct drm_i915_gem_object *obj, + struct i915_gem_context *ctx, + struct intel_engine_cs *engine, + struct i915_request **rq_out) +{ + struct i915_request *rq; + struct i915_vma *batch; + struct i915_vma *vma; + int err; + + GEM_BUG_ON(!intel_engine_can_store_dword(engine)); + + vma = i915_vma_instance(obj, &ctx->ppgtt->vm, NULL); + if (IS_ERR(vma)) + return PTR_ERR(vma); + + err = i915_gem_object_set_to_gtt_domain(obj, false); + if (err) + return err; + + err = i915_vma_pin(vma, 0, 0, PIN_USER); + if (err) + return err; + + batch = rpcs_query_batch(vma); + if (IS_ERR(batch)) { + err = PTR_ERR(batch); + goto err_vma; + } + + rq = i915_request_alloc(engine, ctx); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto err_batch; + } + + err = engine->emit_bb_start(rq, batch->node.start, batch->node.size, 0); + if (err) + goto err_request; + + err = i915_vma_move_to_active(batch, rq, 0); + if (err) + goto skip_request; + + err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); + if (err) + goto skip_request; + + i915_gem_object_set_active_reference(batch->obj); + i915_vma_unpin(batch); + i915_vma_close(batch); + + i915_vma_unpin(vma); + + *rq_out = i915_request_get(rq); + + i915_request_add(rq); + + return 0; + +skip_request: + i915_request_skip(rq, err); +err_request: + i915_request_add(rq); +err_batch: + i915_vma_unpin(batch); +err_vma: + i915_vma_unpin(vma); + + return err; +} + +#define TEST_IDLE BIT(0) +#define TEST_BUSY BIT(1) +#define TEST_RESET BIT(2) + +static int +__sseu_prepare(struct drm_i915_private *i915, + const char *name, + unsigned int flags, + struct i915_gem_context *ctx, + struct intel_engine_cs *engine, + struct igt_spinner **spin) +{ + struct i915_request *rq; + int ret; + + *spin = NULL; + if (!(flags & (TEST_BUSY | TEST_RESET))) + return 0; + + *spin = kzalloc(sizeof(**spin), GFP_KERNEL); + if (!*spin) + return -ENOMEM; + + ret = igt_spinner_init(*spin, i915); + if (ret) + goto err_free; + + rq = igt_spinner_create_request(*spin, ctx, engine, MI_NOOP); + if (IS_ERR(rq)) { + ret = PTR_ERR(rq); + goto err_fini; + } + + i915_request_add(rq); + + if (!igt_wait_for_spinner(*spin, rq)) { + pr_err("%s: Spinner failed to start!\n", name); + ret = -ETIMEDOUT; + goto err_end; + } + + return 0; + +err_end: + igt_spinner_end(*spin); +err_fini: + igt_spinner_fini(*spin); +err_free: + kfree(fetch_and_zero(spin)); + return ret; +} + +static int +__read_slice_count(struct drm_i915_private *i915, + struct i915_gem_context *ctx, + struct intel_engine_cs *engine, + struct drm_i915_gem_object *obj, + struct igt_spinner *spin, + u32 *rpcs) +{ + struct i915_request *rq = NULL; + u32 s_mask, s_shift; + unsigned int cnt; + u32 *buf, val; + long ret; + + ret = emit_rpcs_query(obj, ctx, engine, &rq); + if (ret) + return ret; + + if (spin) + igt_spinner_end(spin); + + ret = i915_request_wait(rq, I915_WAIT_LOCKED, MAX_SCHEDULE_TIMEOUT); + i915_request_put(rq); + if (ret < 0) + return ret; + + buf = i915_gem_object_pin_map(obj, I915_MAP_WB); + if (IS_ERR(buf)) { + ret = PTR_ERR(buf); + return ret; + } + + if (INTEL_GEN(i915) >= 11) { + s_mask = GEN11_RPCS_S_CNT_MASK; + s_shift = GEN11_RPCS_S_CNT_SHIFT; + } else { + s_mask = GEN8_RPCS_S_CNT_MASK; + s_shift = GEN8_RPCS_S_CNT_SHIFT; + } + + val = *buf; + cnt = (val & s_mask) >> s_shift; + *rpcs = val; + + i915_gem_object_unpin_map(obj); + + return cnt; +} + +static int +__check_rpcs(const char *name, u32 rpcs, int slices, unsigned int expected, + const char *prefix, const char *suffix) +{ + if (slices == expected) + return 0; + + if (slices < 0) { + pr_err("%s: %s read slice count failed with %d%s\n", + name, prefix, slices, suffix); + return slices; + } + + pr_err("%s: %s slice count %d is not %u%s\n", + name, prefix, slices, expected, suffix); + + pr_info("RPCS=0x%x; %u%sx%u%s\n", + rpcs, slices, + (rpcs & GEN8_RPCS_S_CNT_ENABLE) ? "*" : "", + (rpcs & GEN8_RPCS_SS_CNT_MASK) >> GEN8_RPCS_SS_CNT_SHIFT, + (rpcs & GEN8_RPCS_SS_CNT_ENABLE) ? "*" : ""); + + return -EINVAL; +} + +static int +__sseu_finish(struct drm_i915_private *i915, + const char *name, + unsigned int flags, + struct i915_gem_context *ctx, + struct i915_gem_context *kctx, + struct intel_engine_cs *engine, + struct drm_i915_gem_object *obj, + unsigned int expected, + struct igt_spinner *spin) +{ + unsigned int slices = + hweight32(intel_device_default_sseu(i915).slice_mask); + u32 rpcs = 0; + int ret = 0; + + if (flags & TEST_RESET) { + ret = i915_reset_engine(engine, "sseu"); + if (ret) + goto out; + } + + ret = __read_slice_count(i915, ctx, engine, obj, + flags & TEST_RESET ? NULL : spin, &rpcs); + ret = __check_rpcs(name, rpcs, ret, expected, "Context", "!"); + if (ret) + goto out; + + ret = __read_slice_count(i915, kctx, engine, obj, NULL, &rpcs); + ret = __check_rpcs(name, rpcs, ret, slices, "Kernel context", "!"); + +out: + if (spin) + igt_spinner_end(spin); + + if ((flags & TEST_IDLE) && ret == 0) { + ret = i915_gem_wait_for_idle(i915, + I915_WAIT_LOCKED, + MAX_SCHEDULE_TIMEOUT); + if (ret) + return ret; + + ret = __read_slice_count(i915, ctx, engine, obj, NULL, &rpcs); + ret = __check_rpcs(name, rpcs, ret, expected, + "Context", " after idle!"); + } + + return ret; +} + +static int +__sseu_test(struct drm_i915_private *i915, + const char *name, + unsigned int flags, + struct i915_gem_context *ctx, + struct intel_engine_cs *engine, + struct drm_i915_gem_object *obj, + struct intel_sseu sseu) +{ + struct igt_spinner *spin = NULL; + struct i915_gem_context *kctx; + int ret; + + kctx = kernel_context(i915); + if (IS_ERR(kctx)) + return PTR_ERR(kctx); + + ret = __sseu_prepare(i915, name, flags, ctx, engine, &spin); + if (ret) + goto out_context; + + ret = __i915_gem_context_reconfigure_sseu(ctx, engine, sseu); + if (ret) + goto out_spin; + + ret = __sseu_finish(i915, name, flags, ctx, kctx, engine, obj, + hweight32(sseu.slice_mask), spin); + +out_spin: + if (spin) { + igt_spinner_end(spin); + igt_spinner_fini(spin); + kfree(spin); + } + +out_context: + kernel_context_close(kctx); + + return ret; +} + +static int +__igt_ctx_sseu(struct drm_i915_private *i915, + const char *name, + unsigned int flags) +{ + struct intel_sseu default_sseu = intel_device_default_sseu(i915); + struct intel_engine_cs *engine = i915->engine[RCS]; + struct drm_i915_gem_object *obj; + struct i915_gem_context *ctx; + struct intel_sseu pg_sseu; + intel_wakeref_t wakeref; + struct drm_file *file; + int ret; + + if (INTEL_GEN(i915) < 9) + return 0; + + if (!RUNTIME_INFO(i915)->sseu.has_slice_pg) + return 0; + + if (hweight32(default_sseu.slice_mask) < 2) + return 0; + + /* + * Gen11 VME friendly power-gated configuration with half enabled + * sub-slices. + */ + pg_sseu = default_sseu; + pg_sseu.slice_mask = 1; + pg_sseu.subslice_mask = + ~(~0 << (hweight32(default_sseu.subslice_mask) / 2)); + + pr_info("SSEU subtest '%s', flags=%x, def_slices=%u, pg_slices=%u\n", + name, flags, hweight32(default_sseu.slice_mask), + hweight32(pg_sseu.slice_mask)); + + file = mock_file(i915); + if (IS_ERR(file)) + return PTR_ERR(file); + + if (flags & TEST_RESET) + igt_global_reset_lock(i915); + + mutex_lock(&i915->drm.struct_mutex); + + ctx = i915_gem_create_context(i915, file->driver_priv); + if (IS_ERR(ctx)) { + ret = PTR_ERR(ctx); + goto out_unlock; + } + + obj = i915_gem_object_create_internal(i915, PAGE_SIZE); + if (IS_ERR(obj)) { + ret = PTR_ERR(obj); + goto out_unlock; + } + + wakeref = intel_runtime_pm_get(i915); + + /* First set the default mask. */ + ret = __sseu_test(i915, name, flags, ctx, engine, obj, default_sseu); + if (ret) + goto out_fail; + + /* Then set a power-gated configuration. */ + ret = __sseu_test(i915, name, flags, ctx, engine, obj, pg_sseu); + if (ret) + goto out_fail; + + /* Back to defaults. */ + ret = __sseu_test(i915, name, flags, ctx, engine, obj, default_sseu); + if (ret) + goto out_fail; + + /* One last power-gated configuration for the road. */ + ret = __sseu_test(i915, name, flags, ctx, engine, obj, pg_sseu); + if (ret) + goto out_fail; + +out_fail: + if (igt_flush_test(i915, I915_WAIT_LOCKED)) + ret = -EIO; + + i915_gem_object_put(obj); + + intel_runtime_pm_put(i915, wakeref); + +out_unlock: + mutex_unlock(&i915->drm.struct_mutex); + + if (flags & TEST_RESET) + igt_global_reset_unlock(i915); + + mock_file_free(i915, file); + + if (ret) + pr_err("%s: Failed with %d!\n", name, ret); + + return ret; +} + +static int igt_ctx_sseu(void *arg) +{ + struct { + const char *name; + unsigned int flags; + } *phase, phases[] = { + { .name = "basic", .flags = 0 }, + { .name = "idle", .flags = TEST_IDLE }, + { .name = "busy", .flags = TEST_BUSY }, + { .name = "busy-reset", .flags = TEST_BUSY | TEST_RESET }, + { .name = "busy-idle", .flags = TEST_BUSY | TEST_IDLE }, + { .name = "reset-idle", .flags = TEST_RESET | TEST_IDLE }, + }; + unsigned int i; + int ret = 0; + + for (i = 0, phase = phases; ret == 0 && i < ARRAY_SIZE(phases); + i++, phase++) + ret = __igt_ctx_sseu(arg, phase->name, phase->flags); + + return ret; +} + static int igt_ctx_readonly(void *arg) { struct drm_i915_private *i915 = arg; struct drm_i915_gem_object *obj = NULL; + struct i915_gem_context *ctx; + struct i915_hw_ppgtt *ppgtt; + unsigned long ndwords, dw; + struct igt_live_test t; struct drm_file *file; I915_RND_STATE(prng); IGT_TIMEOUT(end_time); LIST_HEAD(objects); - struct i915_gem_context *ctx; - struct i915_hw_ppgtt *ppgtt; - unsigned long ndwords, dw; int err = -ENODEV; /* @@ -655,6 +1067,10 @@ static int igt_ctx_readonly(void *arg) mutex_lock(&i915->drm.struct_mutex); + err = igt_live_test_begin(&t, i915, __func__, ""); + if (err) + goto out_unlock; + ctx = i915_gem_create_context(i915, file->driver_priv); if (IS_ERR(ctx)) { err = PTR_ERR(ctx); @@ -674,6 +1090,8 @@ static int igt_ctx_readonly(void *arg) unsigned int id; for_each_engine(engine, i915, id) { + intel_wakeref_t wakeref; + if (!intel_engine_can_store_dword(engine)) continue; @@ -688,9 +1106,9 @@ static int igt_ctx_readonly(void *arg) i915_gem_object_set_readonly(obj); } - intel_runtime_pm_get(i915); - err = gpu_fill(obj, ctx, engine, dw); - intel_runtime_pm_put(i915); + err = 0; + with_intel_runtime_pm(i915, wakeref) + err = gpu_fill(obj, ctx, engine, dw); if (err) { pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) in ctx %u [full-ppgtt? %s], err=%d\n", ndwords, dw, max_dwords(obj), @@ -707,7 +1125,7 @@ static int igt_ctx_readonly(void *arg) } } pr_info("Submitted %lu dwords (across %u engines)\n", - ndwords, INTEL_INFO(i915)->num_rings); + ndwords, RUNTIME_INFO(i915)->num_rings); dw = 0; list_for_each_entry(obj, &objects, st_link) { @@ -727,7 +1145,325 @@ static int igt_ctx_readonly(void *arg) } out_unlock: - if (igt_flush_test(i915, I915_WAIT_LOCKED)) + if (igt_live_test_end(&t)) + err = -EIO; + mutex_unlock(&i915->drm.struct_mutex); + + mock_file_free(i915, file); + return err; +} + +static int check_scratch(struct i915_gem_context *ctx, u64 offset) +{ + struct drm_mm_node *node = + __drm_mm_interval_first(&ctx->ppgtt->vm.mm, + offset, offset + sizeof(u32) - 1); + if (!node || node->start > offset) + return 0; + + GEM_BUG_ON(offset >= node->start + node->size); + + pr_err("Target offset 0x%08x_%08x overlaps with a node in the mm!\n", + upper_32_bits(offset), lower_32_bits(offset)); + return -EINVAL; +} + +static int write_to_scratch(struct i915_gem_context *ctx, + struct intel_engine_cs *engine, + u64 offset, u32 value) +{ + struct drm_i915_private *i915 = ctx->i915; + struct drm_i915_gem_object *obj; + struct i915_request *rq; + struct i915_vma *vma; + u32 *cmd; + int err; + + GEM_BUG_ON(offset < I915_GTT_PAGE_SIZE); + + obj = i915_gem_object_create_internal(i915, PAGE_SIZE); + if (IS_ERR(obj)) + return PTR_ERR(obj); + + cmd = i915_gem_object_pin_map(obj, I915_MAP_WB); + if (IS_ERR(cmd)) { + err = PTR_ERR(cmd); + goto err; + } + + *cmd++ = MI_STORE_DWORD_IMM_GEN4; + if (INTEL_GEN(i915) >= 8) { + *cmd++ = lower_32_bits(offset); + *cmd++ = upper_32_bits(offset); + } else { + *cmd++ = 0; + *cmd++ = offset; + } + *cmd++ = value; + *cmd = MI_BATCH_BUFFER_END; + i915_gem_object_unpin_map(obj); + + err = i915_gem_object_set_to_gtt_domain(obj, false); + if (err) + goto err; + + vma = i915_vma_instance(obj, &ctx->ppgtt->vm, NULL); + if (IS_ERR(vma)) { + err = PTR_ERR(vma); + goto err; + } + + err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_OFFSET_FIXED); + if (err) + goto err; + + err = check_scratch(ctx, offset); + if (err) + goto err_unpin; + + rq = i915_request_alloc(engine, ctx); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto err_unpin; + } + + err = engine->emit_bb_start(rq, vma->node.start, vma->node.size, 0); + if (err) + goto err_request; + + err = i915_vma_move_to_active(vma, rq, 0); + if (err) + goto skip_request; + + i915_gem_object_set_active_reference(obj); + i915_vma_unpin(vma); + i915_vma_close(vma); + + i915_request_add(rq); + + return 0; + +skip_request: + i915_request_skip(rq, err); +err_request: + i915_request_add(rq); +err_unpin: + i915_vma_unpin(vma); +err: + i915_gem_object_put(obj); + return err; +} + +static int read_from_scratch(struct i915_gem_context *ctx, + struct intel_engine_cs *engine, + u64 offset, u32 *value) +{ + struct drm_i915_private *i915 = ctx->i915; + struct drm_i915_gem_object *obj; + const u32 RCS_GPR0 = 0x2600; /* not all engines have their own GPR! */ + const u32 result = 0x100; + struct i915_request *rq; + struct i915_vma *vma; + u32 *cmd; + int err; + + GEM_BUG_ON(offset < I915_GTT_PAGE_SIZE); + + obj = i915_gem_object_create_internal(i915, PAGE_SIZE); + if (IS_ERR(obj)) + return PTR_ERR(obj); + + cmd = i915_gem_object_pin_map(obj, I915_MAP_WB); + if (IS_ERR(cmd)) { + err = PTR_ERR(cmd); + goto err; + } + + memset(cmd, POISON_INUSE, PAGE_SIZE); + if (INTEL_GEN(i915) >= 8) { + *cmd++ = MI_LOAD_REGISTER_MEM_GEN8; + *cmd++ = RCS_GPR0; + *cmd++ = lower_32_bits(offset); + *cmd++ = upper_32_bits(offset); + *cmd++ = MI_STORE_REGISTER_MEM_GEN8; + *cmd++ = RCS_GPR0; + *cmd++ = result; + *cmd++ = 0; + } else { + *cmd++ = MI_LOAD_REGISTER_MEM; + *cmd++ = RCS_GPR0; + *cmd++ = offset; + *cmd++ = MI_STORE_REGISTER_MEM; + *cmd++ = RCS_GPR0; + *cmd++ = result; + } + *cmd = MI_BATCH_BUFFER_END; + i915_gem_object_unpin_map(obj); + + err = i915_gem_object_set_to_gtt_domain(obj, false); + if (err) + goto err; + + vma = i915_vma_instance(obj, &ctx->ppgtt->vm, NULL); + if (IS_ERR(vma)) { + err = PTR_ERR(vma); + goto err; + } + + err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_OFFSET_FIXED); + if (err) + goto err; + + err = check_scratch(ctx, offset); + if (err) + goto err_unpin; + + rq = i915_request_alloc(engine, ctx); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto err_unpin; + } + + err = engine->emit_bb_start(rq, vma->node.start, vma->node.size, 0); + if (err) + goto err_request; + + err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); + if (err) + goto skip_request; + + i915_vma_unpin(vma); + i915_vma_close(vma); + + i915_request_add(rq); + + err = i915_gem_object_set_to_cpu_domain(obj, false); + if (err) + goto err; + + cmd = i915_gem_object_pin_map(obj, I915_MAP_WB); + if (IS_ERR(cmd)) { + err = PTR_ERR(cmd); + goto err; + } + + *value = cmd[result / sizeof(*cmd)]; + i915_gem_object_unpin_map(obj); + i915_gem_object_put(obj); + + return 0; + +skip_request: + i915_request_skip(rq, err); +err_request: + i915_request_add(rq); +err_unpin: + i915_vma_unpin(vma); +err: + i915_gem_object_put(obj); + return err; +} + +static int igt_vm_isolation(void *arg) +{ + struct drm_i915_private *i915 = arg; + struct i915_gem_context *ctx_a, *ctx_b; + struct intel_engine_cs *engine; + intel_wakeref_t wakeref; + struct igt_live_test t; + struct drm_file *file; + I915_RND_STATE(prng); + unsigned long count; + unsigned int id; + u64 vm_total; + int err; + + if (INTEL_GEN(i915) < 7) + return 0; + + /* + * The simple goal here is that a write into one context is not + * observed in a second (separate page tables and scratch). + */ + + file = mock_file(i915); + if (IS_ERR(file)) + return PTR_ERR(file); + + mutex_lock(&i915->drm.struct_mutex); + + err = igt_live_test_begin(&t, i915, __func__, ""); + if (err) + goto out_unlock; + + ctx_a = i915_gem_create_context(i915, file->driver_priv); + if (IS_ERR(ctx_a)) { + err = PTR_ERR(ctx_a); + goto out_unlock; + } + + ctx_b = i915_gem_create_context(i915, file->driver_priv); + if (IS_ERR(ctx_b)) { + err = PTR_ERR(ctx_b); + goto out_unlock; + } + + /* We can only test vm isolation, if the vm are distinct */ + if (ctx_a->ppgtt == ctx_b->ppgtt) + goto out_unlock; + + vm_total = ctx_a->ppgtt->vm.total; + GEM_BUG_ON(ctx_b->ppgtt->vm.total != vm_total); + vm_total -= I915_GTT_PAGE_SIZE; + + wakeref = intel_runtime_pm_get(i915); + + count = 0; + for_each_engine(engine, i915, id) { + IGT_TIMEOUT(end_time); + unsigned long this = 0; + + if (!intel_engine_can_store_dword(engine)) + continue; + + while (!__igt_timeout(end_time, NULL)) { + u32 value = 0xc5c5c5c5; + u64 offset; + + div64_u64_rem(i915_prandom_u64_state(&prng), + vm_total, &offset); + offset &= ~sizeof(u32); + offset += I915_GTT_PAGE_SIZE; + + err = write_to_scratch(ctx_a, engine, + offset, 0xdeadbeef); + if (err == 0) + err = read_from_scratch(ctx_b, engine, + offset, &value); + if (err) + goto out_rpm; + + if (value) { + pr_err("%s: Read %08x from scratch (offset 0x%08x_%08x), after %lu reads!\n", + engine->name, value, + upper_32_bits(offset), + lower_32_bits(offset), + this); + err = -EINVAL; + goto out_rpm; + } + + this++; + } + count += this; + } + pr_info("Checked %lu scratch offsets across %d engines\n", + count, RUNTIME_INFO(i915)->num_rings); + +out_rpm: + intel_runtime_pm_put(i915, wakeref); +out_unlock: + if (igt_live_test_end(&t)) err = -EIO; mutex_unlock(&i915->drm.struct_mutex); @@ -823,6 +1559,7 @@ static int igt_switch_to_kernel_context(void *arg) struct intel_engine_cs *engine; struct i915_gem_context *ctx; enum intel_engine_id id; + intel_wakeref_t wakeref; int err; /* @@ -833,7 +1570,7 @@ static int igt_switch_to_kernel_context(void *arg) */ mutex_lock(&i915->drm.struct_mutex); - intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(i915); ctx = kernel_context(i915); if (IS_ERR(ctx)) { @@ -858,40 +1595,13 @@ out_unlock: if (igt_flush_test(i915, I915_WAIT_LOCKED)) err = -EIO; - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); mutex_unlock(&i915->drm.struct_mutex); kernel_context_close(ctx); return err; } -static int fake_aliasing_ppgtt_enable(struct drm_i915_private *i915) -{ - struct drm_i915_gem_object *obj; - int err; - - err = i915_gem_init_aliasing_ppgtt(i915); - if (err) - return err; - - list_for_each_entry(obj, &i915->mm.bound_list, mm.link) { - struct i915_vma *vma; - - vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); - if (IS_ERR(vma)) - continue; - - vma->flags &= ~I915_VMA_LOCAL_BIND; - } - - return 0; -} - -static void fake_aliasing_ppgtt_disable(struct drm_i915_private *i915) -{ - i915_gem_fini_aliasing_ppgtt(i915); -} - int i915_gem_context_mock_selftests(void) { static const struct i915_subtest tests[] = { @@ -917,32 +1627,12 @@ int i915_gem_context_live_selftests(struct drm_i915_private *dev_priv) SUBTEST(live_nop_switch), SUBTEST(igt_ctx_exec), SUBTEST(igt_ctx_readonly), + SUBTEST(igt_ctx_sseu), + SUBTEST(igt_vm_isolation), }; - bool fake_alias = false; - int err; if (i915_terminally_wedged(&dev_priv->gpu_error)) return 0; - /* Install a fake aliasing gtt for exercise */ - if (USES_PPGTT(dev_priv) && !dev_priv->mm.aliasing_ppgtt) { - mutex_lock(&dev_priv->drm.struct_mutex); - err = fake_aliasing_ppgtt_enable(dev_priv); - mutex_unlock(&dev_priv->drm.struct_mutex); - if (err) - return err; - - GEM_BUG_ON(!dev_priv->mm.aliasing_ppgtt); - fake_alias = true; - } - - err = i915_subtests(tests, dev_priv); - - if (fake_alias) { - mutex_lock(&dev_priv->drm.struct_mutex); - fake_aliasing_ppgtt_disable(dev_priv); - mutex_unlock(&dev_priv->drm.struct_mutex); - } - - return err; + return i915_subtests(tests, dev_priv); } diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c index 128ad1cf0647..b9b0ea4e2404 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c @@ -29,11 +29,23 @@ #include "mock_drm.h" #include "mock_gem_device.h" -static int populate_ggtt(struct drm_i915_private *i915) +static void quirk_add(struct drm_i915_gem_object *obj, + struct list_head *objects) { + /* quirk is only for live tiled objects, use it to declare ownership */ + GEM_BUG_ON(obj->mm.quirked); + obj->mm.quirked = true; + list_add(&obj->st_link, objects); +} + +static int populate_ggtt(struct drm_i915_private *i915, + struct list_head *objects) +{ + unsigned long unbound, bound, count; struct drm_i915_gem_object *obj; u64 size; + count = 0; for (size = 0; size + I915_GTT_PAGE_SIZE <= i915->ggtt.vm.total; size += I915_GTT_PAGE_SIZE) { @@ -43,21 +55,36 @@ static int populate_ggtt(struct drm_i915_private *i915) if (IS_ERR(obj)) return PTR_ERR(obj); + quirk_add(obj, objects); + vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0); if (IS_ERR(vma)) return PTR_ERR(vma); + + count++; } - if (!list_empty(&i915->mm.unbound_list)) { - size = 0; - list_for_each_entry(obj, &i915->mm.unbound_list, mm.link) - size++; + unbound = 0; + list_for_each_entry(obj, &i915->mm.unbound_list, mm.link) + if (obj->mm.quirked) + unbound++; + if (unbound) { + pr_err("%s: Found %lu objects unbound, expected %u!\n", + __func__, unbound, 0); + return -EINVAL; + } - pr_err("Found %lld objects unbound!\n", size); + bound = 0; + list_for_each_entry(obj, &i915->mm.bound_list, mm.link) + if (obj->mm.quirked) + bound++; + if (bound != count) { + pr_err("%s: Found %lu objects bound, expected %lu!\n", + __func__, bound, count); return -EINVAL; } - if (list_empty(&i915->ggtt.vm.inactive_list)) { + if (list_empty(&i915->ggtt.vm.bound_list)) { pr_err("No objects on the GGTT inactive list!\n"); return -EINVAL; } @@ -67,21 +94,26 @@ static int populate_ggtt(struct drm_i915_private *i915) static void unpin_ggtt(struct drm_i915_private *i915) { + struct i915_ggtt *ggtt = &i915->ggtt; struct i915_vma *vma; - list_for_each_entry(vma, &i915->ggtt.vm.inactive_list, vm_link) - i915_vma_unpin(vma); + mutex_lock(&ggtt->vm.mutex); + list_for_each_entry(vma, &i915->ggtt.vm.bound_list, vm_link) + if (vma->obj->mm.quirked) + i915_vma_unpin(vma); + mutex_unlock(&ggtt->vm.mutex); } -static void cleanup_objects(struct drm_i915_private *i915) +static void cleanup_objects(struct drm_i915_private *i915, + struct list_head *list) { struct drm_i915_gem_object *obj, *on; - list_for_each_entry_safe(obj, on, &i915->mm.unbound_list, mm.link) - i915_gem_object_put(obj); - - list_for_each_entry_safe(obj, on, &i915->mm.bound_list, mm.link) + list_for_each_entry_safe(obj, on, list, st_link) { + GEM_BUG_ON(!obj->mm.quirked); + obj->mm.quirked = false; i915_gem_object_put(obj); + } mutex_unlock(&i915->drm.struct_mutex); @@ -94,11 +126,12 @@ static int igt_evict_something(void *arg) { struct drm_i915_private *i915 = arg; struct i915_ggtt *ggtt = &i915->ggtt; + LIST_HEAD(objects); int err; /* Fill the GGTT with pinned objects and try to evict one. */ - err = populate_ggtt(i915); + err = populate_ggtt(i915, &objects); if (err) goto cleanup; @@ -127,7 +160,7 @@ static int igt_evict_something(void *arg) } cleanup: - cleanup_objects(i915); + cleanup_objects(i915, &objects); return err; } @@ -136,13 +169,14 @@ static int igt_overcommit(void *arg) struct drm_i915_private *i915 = arg; struct drm_i915_gem_object *obj; struct i915_vma *vma; + LIST_HEAD(objects); int err; /* Fill the GGTT with pinned objects and then try to pin one more. * We expect it to fail. */ - err = populate_ggtt(i915); + err = populate_ggtt(i915, &objects); if (err) goto cleanup; @@ -152,6 +186,8 @@ static int igt_overcommit(void *arg) goto cleanup; } + quirk_add(obj, &objects); + vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0); if (!IS_ERR(vma) || PTR_ERR(vma) != -ENOSPC) { pr_err("Failed to evict+insert, i915_gem_object_ggtt_pin returned err=%d\n", (int)PTR_ERR(vma)); @@ -160,7 +196,7 @@ static int igt_overcommit(void *arg) } cleanup: - cleanup_objects(i915); + cleanup_objects(i915, &objects); return err; } @@ -172,11 +208,12 @@ static int igt_evict_for_vma(void *arg) .start = 0, .size = 4096, }; + LIST_HEAD(objects); int err; /* Fill the GGTT with pinned objects and try to evict a range. */ - err = populate_ggtt(i915); + err = populate_ggtt(i915, &objects); if (err) goto cleanup; @@ -199,7 +236,7 @@ static int igt_evict_for_vma(void *arg) } cleanup: - cleanup_objects(i915); + cleanup_objects(i915, &objects); return err; } @@ -222,6 +259,7 @@ static int igt_evict_for_cache_color(void *arg) }; struct drm_i915_gem_object *obj; struct i915_vma *vma; + LIST_HEAD(objects); int err; /* Currently the use of color_adjust is limited to cache domains within @@ -237,6 +275,7 @@ static int igt_evict_for_cache_color(void *arg) goto cleanup; } i915_gem_object_set_cache_level(obj, I915_CACHE_LLC); + quirk_add(obj, &objects); vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, I915_GTT_PAGE_SIZE | flags); @@ -252,6 +291,7 @@ static int igt_evict_for_cache_color(void *arg) goto cleanup; } i915_gem_object_set_cache_level(obj, I915_CACHE_LLC); + quirk_add(obj, &objects); /* Neighbouring; same colour - should fit */ vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, @@ -287,7 +327,7 @@ static int igt_evict_for_cache_color(void *arg) cleanup: unpin_ggtt(i915); - cleanup_objects(i915); + cleanup_objects(i915, &objects); ggtt->vm.mm.color_adjust = NULL; return err; } @@ -296,11 +336,12 @@ static int igt_evict_vm(void *arg) { struct drm_i915_private *i915 = arg; struct i915_ggtt *ggtt = &i915->ggtt; + LIST_HEAD(objects); int err; /* Fill the GGTT with pinned objects and try to evict everything. */ - err = populate_ggtt(i915); + err = populate_ggtt(i915, &objects); if (err) goto cleanup; @@ -322,7 +363,7 @@ static int igt_evict_vm(void *arg) } cleanup: - cleanup_objects(i915); + cleanup_objects(i915, &objects); return err; } @@ -336,6 +377,7 @@ static int igt_evict_contexts(void *arg) struct drm_mm_node node; struct reserved *next; } *reserved = NULL; + intel_wakeref_t wakeref; struct drm_mm_node hole; unsigned long count; int err; @@ -351,11 +393,11 @@ static int igt_evict_contexts(void *arg) * where the GTT space of the request is separate from the GGTT * allocation required to build the request. */ - if (!USES_FULL_PPGTT(i915)) + if (!HAS_FULL_PPGTT(i915)) return 0; mutex_lock(&i915->drm.struct_mutex); - intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(i915); /* Reserve a block so that we know we have enough to fit a few rq */ memset(&hole, 0, sizeof(hole)); @@ -400,8 +442,10 @@ static int igt_evict_contexts(void *arg) struct drm_file *file; file = mock_file(i915); - if (IS_ERR(file)) - return PTR_ERR(file); + if (IS_ERR(file)) { + err = PTR_ERR(file); + break; + } count = 0; mutex_lock(&i915->drm.struct_mutex); @@ -411,7 +455,7 @@ static int igt_evict_contexts(void *arg) struct i915_gem_context *ctx; ctx = live_context(i915, file); - if (!ctx) + if (IS_ERR(ctx)) break; /* We will need some GGTT space for the rq's context */ @@ -464,7 +508,7 @@ out_locked: } if (drm_mm_node_allocated(&hole)) drm_mm_remove_node(&hole); - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); mutex_unlock(&i915->drm.struct_mutex); return err; @@ -480,14 +524,17 @@ int i915_gem_evict_mock_selftests(void) SUBTEST(igt_overcommit), }; struct drm_i915_private *i915; - int err; + intel_wakeref_t wakeref; + int err = 0; i915 = mock_gem_device(); if (!i915) return -ENOMEM; mutex_lock(&i915->drm.struct_mutex); - err = i915_subtests(tests, i915); + with_intel_runtime_pm(i915, wakeref) + err = i915_subtests(tests, i915); + mutex_unlock(&i915->drm.struct_mutex); drm_dev_put(&i915->drm); diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c index 127d81513671..3850ef4a5ec8 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c @@ -153,7 +153,7 @@ static int igt_ppgtt_alloc(void *arg) /* Allocate a ppggt and try to fill the entire range */ - if (!USES_PPGTT(dev_priv)) + if (!HAS_PPGTT(dev_priv)) return 0; ppgtt = __hw_ppgtt_create(dev_priv); @@ -170,7 +170,7 @@ static int igt_ppgtt_alloc(void *arg) * This should ensure that we do not run into the oomkiller during * the test and take down the machine wilfully. */ - limit = totalram_pages << PAGE_SHIFT; + limit = totalram_pages() << PAGE_SHIFT; limit = min(ppgtt->vm.total, limit); /* Check we can allocate the entire range */ @@ -275,6 +275,7 @@ static int lowlevel_hole(struct drm_i915_private *i915, for (n = 0; n < count; n++) { u64 addr = hole_start + order[n] * BIT_ULL(size); + intel_wakeref_t wakeref; GEM_BUG_ON(addr + BIT_ULL(size) > vm->total); @@ -293,9 +294,9 @@ static int lowlevel_hole(struct drm_i915_private *i915, mock_vma.node.size = BIT_ULL(size); mock_vma.node.start = addr; - intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(i915); vm->insert_entries(vm, &mock_vma, I915_CACHE_NONE, 0); - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); } count = n; @@ -1001,7 +1002,7 @@ static int exercise_ppgtt(struct drm_i915_private *dev_priv, IGT_TIMEOUT(end_time); int err; - if (!USES_FULL_PPGTT(dev_priv)) + if (!HAS_FULL_PPGTT(dev_priv)) return 0; file = mock_file(dev_priv); @@ -1144,6 +1145,7 @@ static int igt_ggtt_page(void *arg) struct drm_i915_private *i915 = arg; struct i915_ggtt *ggtt = &i915->ggtt; struct drm_i915_gem_object *obj; + intel_wakeref_t wakeref; struct drm_mm_node tmp; unsigned int *order, n; int err; @@ -1169,7 +1171,7 @@ static int igt_ggtt_page(void *arg) if (err) goto out_unpin; - intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(i915); for (n = 0; n < count; n++) { u64 offset = tmp.start + n * PAGE_SIZE; @@ -1216,7 +1218,7 @@ static int igt_ggtt_page(void *arg) kfree(order); out_remove: ggtt->vm.clear_range(&ggtt->vm, tmp.start, tmp.size); - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); drm_mm_remove_node(&tmp); out_unpin: i915_gem_object_unpin_pages(obj); @@ -1235,7 +1237,10 @@ static void track_vma_bind(struct i915_vma *vma) __i915_gem_object_pin_pages(obj); vma->pages = obj->mm.pages; - list_move_tail(&vma->vm_link, &vma->vm->inactive_list); + + mutex_lock(&vma->vm->mutex); + list_move_tail(&vma->vm_link, &vma->vm->bound_list); + mutex_unlock(&vma->vm->mutex); } static int exercise_mock(struct drm_i915_private *i915, @@ -1244,7 +1249,7 @@ static int exercise_mock(struct drm_i915_private *i915, u64 hole_start, u64 hole_end, unsigned long end_time)) { - const u64 limit = totalram_pages << PAGE_SHIFT; + const u64 limit = totalram_pages() << PAGE_SHIFT; struct i915_gem_context *ctx; struct i915_hw_ppgtt *ppgtt; IGT_TIMEOUT(end_time); @@ -1265,27 +1270,35 @@ static int exercise_mock(struct drm_i915_private *i915, static int igt_mock_fill(void *arg) { - return exercise_mock(arg, fill_hole); + struct i915_ggtt *ggtt = arg; + + return exercise_mock(ggtt->vm.i915, fill_hole); } static int igt_mock_walk(void *arg) { - return exercise_mock(arg, walk_hole); + struct i915_ggtt *ggtt = arg; + + return exercise_mock(ggtt->vm.i915, walk_hole); } static int igt_mock_pot(void *arg) { - return exercise_mock(arg, pot_hole); + struct i915_ggtt *ggtt = arg; + + return exercise_mock(ggtt->vm.i915, pot_hole); } static int igt_mock_drunk(void *arg) { - return exercise_mock(arg, drunk_hole); + struct i915_ggtt *ggtt = arg; + + return exercise_mock(ggtt->vm.i915, drunk_hole); } static int igt_gtt_reserve(void *arg) { - struct drm_i915_private *i915 = arg; + struct i915_ggtt *ggtt = arg; struct drm_i915_gem_object *obj, *on; LIST_HEAD(objects); u64 total; @@ -1298,11 +1311,12 @@ static int igt_gtt_reserve(void *arg) /* Start by filling the GGTT */ for (total = 0; - total + 2*I915_GTT_PAGE_SIZE <= i915->ggtt.vm.total; - total += 2*I915_GTT_PAGE_SIZE) { + total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total; + total += 2 * I915_GTT_PAGE_SIZE) { struct i915_vma *vma; - obj = i915_gem_object_create_internal(i915, 2*PAGE_SIZE); + obj = i915_gem_object_create_internal(ggtt->vm.i915, + 2 * PAGE_SIZE); if (IS_ERR(obj)) { err = PTR_ERR(obj); goto out; @@ -1316,20 +1330,20 @@ static int igt_gtt_reserve(void *arg) list_add(&obj->st_link, &objects); - vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); + vma = i915_vma_instance(obj, &ggtt->vm, NULL); if (IS_ERR(vma)) { err = PTR_ERR(vma); goto out; } - err = i915_gem_gtt_reserve(&i915->ggtt.vm, &vma->node, + err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node, obj->base.size, total, obj->cache_level, 0); if (err) { pr_err("i915_gem_gtt_reserve (pass 1) failed at %llu/%llu with err=%d\n", - total, i915->ggtt.vm.total, err); + total, ggtt->vm.total, err); goto out; } track_vma_bind(vma); @@ -1347,11 +1361,12 @@ static int igt_gtt_reserve(void *arg) /* Now we start forcing evictions */ for (total = I915_GTT_PAGE_SIZE; - total + 2*I915_GTT_PAGE_SIZE <= i915->ggtt.vm.total; - total += 2*I915_GTT_PAGE_SIZE) { + total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total; + total += 2 * I915_GTT_PAGE_SIZE) { struct i915_vma *vma; - obj = i915_gem_object_create_internal(i915, 2*PAGE_SIZE); + obj = i915_gem_object_create_internal(ggtt->vm.i915, + 2 * PAGE_SIZE); if (IS_ERR(obj)) { err = PTR_ERR(obj); goto out; @@ -1365,20 +1380,20 @@ static int igt_gtt_reserve(void *arg) list_add(&obj->st_link, &objects); - vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); + vma = i915_vma_instance(obj, &ggtt->vm, NULL); if (IS_ERR(vma)) { err = PTR_ERR(vma); goto out; } - err = i915_gem_gtt_reserve(&i915->ggtt.vm, &vma->node, + err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node, obj->base.size, total, obj->cache_level, 0); if (err) { pr_err("i915_gem_gtt_reserve (pass 2) failed at %llu/%llu with err=%d\n", - total, i915->ggtt.vm.total, err); + total, ggtt->vm.total, err); goto out; } track_vma_bind(vma); @@ -1399,7 +1414,7 @@ static int igt_gtt_reserve(void *arg) struct i915_vma *vma; u64 offset; - vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); + vma = i915_vma_instance(obj, &ggtt->vm, NULL); if (IS_ERR(vma)) { err = PTR_ERR(vma); goto out; @@ -1411,18 +1426,18 @@ static int igt_gtt_reserve(void *arg) goto out; } - offset = random_offset(0, i915->ggtt.vm.total, + offset = random_offset(0, ggtt->vm.total, 2*I915_GTT_PAGE_SIZE, I915_GTT_MIN_ALIGNMENT); - err = i915_gem_gtt_reserve(&i915->ggtt.vm, &vma->node, + err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node, obj->base.size, offset, obj->cache_level, 0); if (err) { pr_err("i915_gem_gtt_reserve (pass 3) failed at %llu/%llu with err=%d\n", - total, i915->ggtt.vm.total, err); + total, ggtt->vm.total, err); goto out; } track_vma_bind(vma); @@ -1448,7 +1463,7 @@ out: static int igt_gtt_insert(void *arg) { - struct drm_i915_private *i915 = arg; + struct i915_ggtt *ggtt = arg; struct drm_i915_gem_object *obj, *on; struct drm_mm_node tmp = {}; const struct invalid_insert { @@ -1457,8 +1472,8 @@ static int igt_gtt_insert(void *arg) u64 start, end; } invalid_insert[] = { { - i915->ggtt.vm.total + I915_GTT_PAGE_SIZE, 0, - 0, i915->ggtt.vm.total, + ggtt->vm.total + I915_GTT_PAGE_SIZE, 0, + 0, ggtt->vm.total, }, { 2*I915_GTT_PAGE_SIZE, 0, @@ -1488,7 +1503,7 @@ static int igt_gtt_insert(void *arg) /* Check a couple of obviously invalid requests */ for (ii = invalid_insert; ii->size; ii++) { - err = i915_gem_gtt_insert(&i915->ggtt.vm, &tmp, + err = i915_gem_gtt_insert(&ggtt->vm, &tmp, ii->size, ii->alignment, I915_COLOR_UNEVICTABLE, ii->start, ii->end, @@ -1503,11 +1518,12 @@ static int igt_gtt_insert(void *arg) /* Start by filling the GGTT */ for (total = 0; - total + I915_GTT_PAGE_SIZE <= i915->ggtt.vm.total; + total + I915_GTT_PAGE_SIZE <= ggtt->vm.total; total += I915_GTT_PAGE_SIZE) { struct i915_vma *vma; - obj = i915_gem_object_create_internal(i915, I915_GTT_PAGE_SIZE); + obj = i915_gem_object_create_internal(ggtt->vm.i915, + I915_GTT_PAGE_SIZE); if (IS_ERR(obj)) { err = PTR_ERR(obj); goto out; @@ -1521,15 +1537,15 @@ static int igt_gtt_insert(void *arg) list_add(&obj->st_link, &objects); - vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); + vma = i915_vma_instance(obj, &ggtt->vm, NULL); if (IS_ERR(vma)) { err = PTR_ERR(vma); goto out; } - err = i915_gem_gtt_insert(&i915->ggtt.vm, &vma->node, + err = i915_gem_gtt_insert(&ggtt->vm, &vma->node, obj->base.size, 0, obj->cache_level, - 0, i915->ggtt.vm.total, + 0, ggtt->vm.total, 0); if (err == -ENOSPC) { /* maxed out the GGTT space */ @@ -1538,7 +1554,7 @@ static int igt_gtt_insert(void *arg) } if (err) { pr_err("i915_gem_gtt_insert (pass 1) failed at %llu/%llu with err=%d\n", - total, i915->ggtt.vm.total, err); + total, ggtt->vm.total, err); goto out; } track_vma_bind(vma); @@ -1550,7 +1566,7 @@ static int igt_gtt_insert(void *arg) list_for_each_entry(obj, &objects, st_link) { struct i915_vma *vma; - vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); + vma = i915_vma_instance(obj, &ggtt->vm, NULL); if (IS_ERR(vma)) { err = PTR_ERR(vma); goto out; @@ -1570,7 +1586,7 @@ static int igt_gtt_insert(void *arg) struct i915_vma *vma; u64 offset; - vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); + vma = i915_vma_instance(obj, &ggtt->vm, NULL); if (IS_ERR(vma)) { err = PTR_ERR(vma); goto out; @@ -1585,13 +1601,13 @@ static int igt_gtt_insert(void *arg) goto out; } - err = i915_gem_gtt_insert(&i915->ggtt.vm, &vma->node, + err = i915_gem_gtt_insert(&ggtt->vm, &vma->node, obj->base.size, 0, obj->cache_level, - 0, i915->ggtt.vm.total, + 0, ggtt->vm.total, 0); if (err) { pr_err("i915_gem_gtt_insert (pass 2) failed at %llu/%llu with err=%d\n", - total, i915->ggtt.vm.total, err); + total, ggtt->vm.total, err); goto out; } track_vma_bind(vma); @@ -1607,11 +1623,12 @@ static int igt_gtt_insert(void *arg) /* And then force evictions */ for (total = 0; - total + 2*I915_GTT_PAGE_SIZE <= i915->ggtt.vm.total; - total += 2*I915_GTT_PAGE_SIZE) { + total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total; + total += 2 * I915_GTT_PAGE_SIZE) { struct i915_vma *vma; - obj = i915_gem_object_create_internal(i915, 2*I915_GTT_PAGE_SIZE); + obj = i915_gem_object_create_internal(ggtt->vm.i915, + 2 * I915_GTT_PAGE_SIZE); if (IS_ERR(obj)) { err = PTR_ERR(obj); goto out; @@ -1625,19 +1642,19 @@ static int igt_gtt_insert(void *arg) list_add(&obj->st_link, &objects); - vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); + vma = i915_vma_instance(obj, &ggtt->vm, NULL); if (IS_ERR(vma)) { err = PTR_ERR(vma); goto out; } - err = i915_gem_gtt_insert(&i915->ggtt.vm, &vma->node, + err = i915_gem_gtt_insert(&ggtt->vm, &vma->node, obj->base.size, 0, obj->cache_level, - 0, i915->ggtt.vm.total, + 0, ggtt->vm.total, 0); if (err) { pr_err("i915_gem_gtt_insert (pass 3) failed at %llu/%llu with err=%d\n", - total, i915->ggtt.vm.total, err); + total, ggtt->vm.total, err); goto out; } track_vma_bind(vma); @@ -1664,17 +1681,25 @@ int i915_gem_gtt_mock_selftests(void) SUBTEST(igt_gtt_insert), }; struct drm_i915_private *i915; + struct i915_ggtt ggtt; int err; i915 = mock_gem_device(); if (!i915) return -ENOMEM; + mock_init_ggtt(i915, &ggtt); + mutex_lock(&i915->drm.struct_mutex); - err = i915_subtests(tests, i915); + err = i915_subtests(tests, &ggtt); + mock_device_flush(i915); mutex_unlock(&i915->drm.struct_mutex); + i915_gem_drain_freed_objects(i915); + + mock_fini_ggtt(&ggtt); drm_dev_put(&i915->drm); + return err; } diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_object.c b/drivers/gpu/drm/i915/selftests/i915_gem_object.c index c3999dd2021e..395ae878e0f7 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem_object.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem_object.c @@ -238,6 +238,7 @@ static int check_partial_mapping(struct drm_i915_gem_object *obj, u32 *cpu; GEM_BUG_ON(view.partial.size > nreal); + cond_resched(); err = i915_gem_object_set_to_gtt_domain(obj, true); if (err) { @@ -307,6 +308,7 @@ static int igt_partial_tiling(void *arg) const unsigned int nreal = 1 << 12; /* largest tile row x2 */ struct drm_i915_private *i915 = arg; struct drm_i915_gem_object *obj; + intel_wakeref_t wakeref; int tiling; int err; @@ -332,7 +334,7 @@ static int igt_partial_tiling(void *arg) } mutex_lock(&i915->drm.struct_mutex); - intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(i915); if (1) { IGT_TIMEOUT(end); @@ -443,7 +445,7 @@ next_tiling: ; } out_unlock: - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); mutex_unlock(&i915->drm.struct_mutex); i915_gem_object_unpin_pages(obj); out: @@ -505,11 +507,13 @@ static void disable_retire_worker(struct drm_i915_private *i915) mutex_lock(&i915->drm.struct_mutex); if (!i915->gt.active_requests++) { - intel_runtime_pm_get(i915); - i915_gem_unpark(i915); - intel_runtime_pm_put(i915); + intel_wakeref_t wakeref; + + with_intel_runtime_pm(i915, wakeref) + i915_gem_unpark(i915); } mutex_unlock(&i915->drm.struct_mutex); + cancel_delayed_work_sync(&i915->gt.retire_work); cancel_delayed_work_sync(&i915->gt.idle_work); } @@ -577,6 +581,8 @@ static int igt_mmap_offset_exhaustion(void *arg) /* Now fill with busy dead objects that we expect to reap */ for (loop = 0; loop < 3; loop++) { + intel_wakeref_t wakeref; + if (i915_terminally_wedged(&i915->gpu_error)) break; @@ -586,10 +592,10 @@ static int igt_mmap_offset_exhaustion(void *arg) goto out; } + err = 0; mutex_lock(&i915->drm.struct_mutex); - intel_runtime_pm_get(i915); - err = make_obj_busy(obj); - intel_runtime_pm_put(i915); + with_intel_runtime_pm(i915, wakeref) + err = make_obj_busy(obj); mutex_unlock(&i915->drm.struct_mutex); if (err) { pr_err("[loop %d] Failed to busy the object\n", loop); diff --git a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h index a15713cae3b3..6d766925ad04 100644 --- a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h +++ b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h @@ -12,7 +12,9 @@ selftest(sanitycheck, i915_live_sanitycheck) /* keep first (igt selfcheck) */ selftest(uncore, intel_uncore_live_selftests) selftest(workarounds, intel_workarounds_live_selftests) +selftest(timelines, i915_timeline_live_selftests) selftest(requests, i915_request_live_selftests) +selftest(active, i915_active_live_selftests) selftest(objects, i915_gem_object_live_selftests) selftest(dmabuf, i915_gem_dmabuf_live_selftests) selftest(coherency, i915_gem_coherency_live_selftests) diff --git a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h index 1b70208eeea7..88e5ab586337 100644 --- a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h +++ b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h @@ -15,8 +15,7 @@ selftest(scatterlist, scatterlist_mock_selftests) selftest(syncmap, i915_syncmap_mock_selftests) selftest(uncore, intel_uncore_mock_selftests) selftest(engine, intel_engine_cs_mock_selftests) -selftest(breadcrumbs, intel_breadcrumbs_mock_selftests) -selftest(timelines, i915_gem_timeline_mock_selftests) +selftest(timelines, i915_timeline_mock_selftests) selftest(requests, i915_request_mock_selftests) selftest(objects, i915_gem_object_mock_selftests) selftest(dmabuf, i915_gem_dmabuf_mock_selftests) diff --git a/drivers/gpu/drm/i915/selftests/i915_random.c b/drivers/gpu/drm/i915/selftests/i915_random.c index 1f415ce47018..716a3f19f030 100644 --- a/drivers/gpu/drm/i915/selftests/i915_random.c +++ b/drivers/gpu/drm/i915/selftests/i915_random.c @@ -41,18 +41,37 @@ u64 i915_prandom_u64_state(struct rnd_state *rnd) return x; } -void i915_random_reorder(unsigned int *order, unsigned int count, - struct rnd_state *state) +void i915_prandom_shuffle(void *arr, size_t elsz, size_t count, + struct rnd_state *state) { - unsigned int i, j; + char stack[128]; + + if (WARN_ON(elsz > sizeof(stack) || count > U32_MAX)) + return; + + if (!elsz || !count) + return; + + /* Fisher-Yates shuffle courtesy of Knuth */ + while (--count) { + size_t swp; + + swp = i915_prandom_u32_max_state(count + 1, state); + if (swp == count) + continue; - for (i = 0; i < count; i++) { - BUILD_BUG_ON(sizeof(unsigned int) > sizeof(u32)); - j = i915_prandom_u32_max_state(count, state); - swap(order[i], order[j]); + memcpy(stack, arr + count * elsz, elsz); + memcpy(arr + count * elsz, arr + swp * elsz, elsz); + memcpy(arr + swp * elsz, stack, elsz); } } +void i915_random_reorder(unsigned int *order, unsigned int count, + struct rnd_state *state) +{ + i915_prandom_shuffle(order, sizeof(*order), count, state); +} + unsigned int *i915_random_order(unsigned int count, struct rnd_state *state) { unsigned int *order, i; diff --git a/drivers/gpu/drm/i915/selftests/i915_random.h b/drivers/gpu/drm/i915/selftests/i915_random.h index 7dffedc501ca..8e1ff9c105b6 100644 --- a/drivers/gpu/drm/i915/selftests/i915_random.h +++ b/drivers/gpu/drm/i915/selftests/i915_random.h @@ -54,4 +54,7 @@ void i915_random_reorder(unsigned int *order, unsigned int count, struct rnd_state *state); +void i915_prandom_shuffle(void *arr, size_t elsz, size_t count, + struct rnd_state *state); + #endif /* !__I915_SELFTESTS_RANDOM_H__ */ diff --git a/drivers/gpu/drm/i915/selftests/i915_request.c b/drivers/gpu/drm/i915/selftests/i915_request.c index 07e557815308..6733dc5b6b4c 100644 --- a/drivers/gpu/drm/i915/selftests/i915_request.c +++ b/drivers/gpu/drm/i915/selftests/i915_request.c @@ -25,8 +25,12 @@ #include <linux/prime_numbers.h> #include "../i915_selftest.h" +#include "i915_random.h" +#include "igt_live_test.h" +#include "lib_sw_fence.h" #include "mock_context.h" +#include "mock_drm.h" #include "mock_gem_device.h" static int igt_add_request(void *arg) @@ -246,93 +250,285 @@ err_context_0: return err; } -int i915_request_mock_selftests(void) +struct smoketest { + struct intel_engine_cs *engine; + struct i915_gem_context **contexts; + atomic_long_t num_waits, num_fences; + int ncontexts, max_batch; + struct i915_request *(*request_alloc)(struct i915_gem_context *, + struct intel_engine_cs *); +}; + +static struct i915_request * +__mock_request_alloc(struct i915_gem_context *ctx, + struct intel_engine_cs *engine) { - static const struct i915_subtest tests[] = { - SUBTEST(igt_add_request), - SUBTEST(igt_wait_request), - SUBTEST(igt_fence_wait), - SUBTEST(igt_request_rewind), - }; - struct drm_i915_private *i915; - int err; + return mock_request(engine, ctx, 0); +} - i915 = mock_gem_device(); - if (!i915) +static struct i915_request * +__live_request_alloc(struct i915_gem_context *ctx, + struct intel_engine_cs *engine) +{ + return i915_request_alloc(engine, ctx); +} + +static int __igt_breadcrumbs_smoketest(void *arg) +{ + struct smoketest *t = arg; + struct mutex * const BKL = &t->engine->i915->drm.struct_mutex; + const unsigned int max_batch = min(t->ncontexts, t->max_batch) - 1; + const unsigned int total = 4 * t->ncontexts + 1; + unsigned int num_waits = 0, num_fences = 0; + struct i915_request **requests; + I915_RND_STATE(prng); + unsigned int *order; + int err = 0; + + /* + * A very simple test to catch the most egregious of list handling bugs. + * + * At its heart, we simply create oodles of requests running across + * multiple kthreads and enable signaling on them, for the sole purpose + * of stressing our breadcrumb handling. The only inspection we do is + * that the fences were marked as signaled. + */ + + requests = kmalloc_array(total, sizeof(*requests), GFP_KERNEL); + if (!requests) return -ENOMEM; - err = i915_subtests(tests, i915); - drm_dev_put(&i915->drm); + order = i915_random_order(total, &prng); + if (!order) { + err = -ENOMEM; + goto out_requests; + } - return err; -} + while (!kthread_should_stop()) { + struct i915_sw_fence *submit, *wait; + unsigned int n, count; -struct live_test { - struct drm_i915_private *i915; - const char *func; - const char *name; + submit = heap_fence_create(GFP_KERNEL); + if (!submit) { + err = -ENOMEM; + break; + } - unsigned int reset_count; -}; + wait = heap_fence_create(GFP_KERNEL); + if (!wait) { + i915_sw_fence_commit(submit); + heap_fence_put(submit); + err = ENOMEM; + break; + } -static int begin_live_test(struct live_test *t, - struct drm_i915_private *i915, - const char *func, - const char *name) -{ - int err; + i915_random_reorder(order, total, &prng); + count = 1 + i915_prandom_u32_max_state(max_batch, &prng); - t->i915 = i915; - t->func = func; - t->name = name; + for (n = 0; n < count; n++) { + struct i915_gem_context *ctx = + t->contexts[order[n] % t->ncontexts]; + struct i915_request *rq; - err = i915_gem_wait_for_idle(i915, - I915_WAIT_LOCKED, - MAX_SCHEDULE_TIMEOUT); - if (err) { - pr_err("%s(%s): failed to idle before, with err=%d!", - func, name, err); - return err; + mutex_lock(BKL); + + rq = t->request_alloc(ctx, t->engine); + if (IS_ERR(rq)) { + mutex_unlock(BKL); + err = PTR_ERR(rq); + count = n; + break; + } + + err = i915_sw_fence_await_sw_fence_gfp(&rq->submit, + submit, + GFP_KERNEL); + + requests[n] = i915_request_get(rq); + i915_request_add(rq); + + mutex_unlock(BKL); + + if (err >= 0) + err = i915_sw_fence_await_dma_fence(wait, + &rq->fence, + 0, + GFP_KERNEL); + + if (err < 0) { + i915_request_put(rq); + count = n; + break; + } + } + + i915_sw_fence_commit(submit); + i915_sw_fence_commit(wait); + + if (!wait_event_timeout(wait->wait, + i915_sw_fence_done(wait), + HZ / 2)) { + struct i915_request *rq = requests[count - 1]; + + pr_err("waiting for %d fences (last %llx:%lld) on %s timed out!\n", + count, + rq->fence.context, rq->fence.seqno, + t->engine->name); + i915_gem_set_wedged(t->engine->i915); + GEM_BUG_ON(!i915_request_completed(rq)); + i915_sw_fence_wait(wait); + err = -EIO; + } + + for (n = 0; n < count; n++) { + struct i915_request *rq = requests[n]; + + if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, + &rq->fence.flags)) { + pr_err("%llu:%llu was not signaled!\n", + rq->fence.context, rq->fence.seqno); + err = -EINVAL; + } + + i915_request_put(rq); + } + + heap_fence_put(wait); + heap_fence_put(submit); + + if (err < 0) + break; + + num_fences += count; + num_waits++; + + cond_resched(); } - i915->gpu_error.missed_irq_rings = 0; - t->reset_count = i915_reset_count(&i915->gpu_error); + atomic_long_add(num_fences, &t->num_fences); + atomic_long_add(num_waits, &t->num_waits); - return 0; + kfree(order); +out_requests: + kfree(requests); + return err; } -static int end_live_test(struct live_test *t) +static int mock_breadcrumbs_smoketest(void *arg) { - struct drm_i915_private *i915 = t->i915; + struct drm_i915_private *i915 = arg; + struct smoketest t = { + .engine = i915->engine[RCS], + .ncontexts = 1024, + .max_batch = 1024, + .request_alloc = __mock_request_alloc + }; + unsigned int ncpus = num_online_cpus(); + struct task_struct **threads; + unsigned int n; + int ret = 0; + + /* + * Smoketest our breadcrumb/signal handling for requests across multiple + * threads. A very simple test to only catch the most egregious of bugs. + * See __igt_breadcrumbs_smoketest(); + */ - i915_retire_requests(i915); + threads = kmalloc_array(ncpus, sizeof(*threads), GFP_KERNEL); + if (!threads) + return -ENOMEM; - if (wait_for(intel_engines_are_idle(i915), 10)) { - pr_err("%s(%s): GPU not idle\n", t->func, t->name); - return -EIO; + t.contexts = + kmalloc_array(t.ncontexts, sizeof(*t.contexts), GFP_KERNEL); + if (!t.contexts) { + ret = -ENOMEM; + goto out_threads; } - if (t->reset_count != i915_reset_count(&i915->gpu_error)) { - pr_err("%s(%s): GPU was reset %d times!\n", - t->func, t->name, - i915_reset_count(&i915->gpu_error) - t->reset_count); - return -EIO; + mutex_lock(&t.engine->i915->drm.struct_mutex); + for (n = 0; n < t.ncontexts; n++) { + t.contexts[n] = mock_context(t.engine->i915, "mock"); + if (!t.contexts[n]) { + ret = -ENOMEM; + goto out_contexts; + } } + mutex_unlock(&t.engine->i915->drm.struct_mutex); + + for (n = 0; n < ncpus; n++) { + threads[n] = kthread_run(__igt_breadcrumbs_smoketest, + &t, "igt/%d", n); + if (IS_ERR(threads[n])) { + ret = PTR_ERR(threads[n]); + ncpus = n; + break; + } - if (i915->gpu_error.missed_irq_rings) { - pr_err("%s(%s): Missed interrupts on engines %lx\n", - t->func, t->name, i915->gpu_error.missed_irq_rings); - return -EIO; + get_task_struct(threads[n]); } - return 0; + msleep(jiffies_to_msecs(i915_selftest.timeout_jiffies)); + + for (n = 0; n < ncpus; n++) { + int err; + + err = kthread_stop(threads[n]); + if (err < 0 && !ret) + ret = err; + + put_task_struct(threads[n]); + } + pr_info("Completed %lu waits for %lu fence across %d cpus\n", + atomic_long_read(&t.num_waits), + atomic_long_read(&t.num_fences), + ncpus); + + mutex_lock(&t.engine->i915->drm.struct_mutex); +out_contexts: + for (n = 0; n < t.ncontexts; n++) { + if (!t.contexts[n]) + break; + mock_context_close(t.contexts[n]); + } + mutex_unlock(&t.engine->i915->drm.struct_mutex); + kfree(t.contexts); +out_threads: + kfree(threads); + + return ret; +} + +int i915_request_mock_selftests(void) +{ + static const struct i915_subtest tests[] = { + SUBTEST(igt_add_request), + SUBTEST(igt_wait_request), + SUBTEST(igt_fence_wait), + SUBTEST(igt_request_rewind), + SUBTEST(mock_breadcrumbs_smoketest), + }; + struct drm_i915_private *i915; + intel_wakeref_t wakeref; + int err = 0; + + i915 = mock_gem_device(); + if (!i915) + return -ENOMEM; + + with_intel_runtime_pm(i915, wakeref) + err = i915_subtests(tests, i915); + + drm_dev_put(&i915->drm); + + return err; } static int live_nop_request(void *arg) { struct drm_i915_private *i915 = arg; struct intel_engine_cs *engine; - struct live_test t; + intel_wakeref_t wakeref; + struct igt_live_test t; unsigned int id; int err = -ENODEV; @@ -342,7 +538,7 @@ static int live_nop_request(void *arg) */ mutex_lock(&i915->drm.struct_mutex); - intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(i915); for_each_engine(engine, i915, id) { struct i915_request *request = NULL; @@ -350,7 +546,7 @@ static int live_nop_request(void *arg) IGT_TIMEOUT(end_time); ktime_t times[2] = {}; - err = begin_live_test(&t, i915, __func__, engine->name); + err = igt_live_test_begin(&t, i915, __func__, engine->name); if (err) goto out_unlock; @@ -392,7 +588,7 @@ static int live_nop_request(void *arg) break; } - err = end_live_test(&t); + err = igt_live_test_end(&t); if (err) goto out_unlock; @@ -403,7 +599,7 @@ static int live_nop_request(void *arg) } out_unlock: - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); mutex_unlock(&i915->drm.struct_mutex); return err; } @@ -478,7 +674,8 @@ static int live_empty_request(void *arg) { struct drm_i915_private *i915 = arg; struct intel_engine_cs *engine; - struct live_test t; + intel_wakeref_t wakeref; + struct igt_live_test t; struct i915_vma *batch; unsigned int id; int err = 0; @@ -489,7 +686,7 @@ static int live_empty_request(void *arg) */ mutex_lock(&i915->drm.struct_mutex); - intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(i915); batch = empty_batch(i915); if (IS_ERR(batch)) { @@ -503,7 +700,7 @@ static int live_empty_request(void *arg) unsigned long n, prime; ktime_t times[2] = {}; - err = begin_live_test(&t, i915, __func__, engine->name); + err = igt_live_test_begin(&t, i915, __func__, engine->name); if (err) goto out_batch; @@ -539,7 +736,7 @@ static int live_empty_request(void *arg) break; } - err = end_live_test(&t); + err = igt_live_test_end(&t); if (err) goto out_batch; @@ -553,7 +750,7 @@ out_batch: i915_vma_unpin(batch); i915_vma_put(batch); out_unlock: - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); mutex_unlock(&i915->drm.struct_mutex); return err; } @@ -637,8 +834,9 @@ static int live_all_engines(void *arg) struct drm_i915_private *i915 = arg; struct intel_engine_cs *engine; struct i915_request *request[I915_NUM_ENGINES]; + intel_wakeref_t wakeref; + struct igt_live_test t; struct i915_vma *batch; - struct live_test t; unsigned int id; int err; @@ -648,9 +846,9 @@ static int live_all_engines(void *arg) */ mutex_lock(&i915->drm.struct_mutex); - intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(i915); - err = begin_live_test(&t, i915, __func__, ""); + err = igt_live_test_begin(&t, i915, __func__, ""); if (err) goto out_unlock; @@ -722,7 +920,7 @@ static int live_all_engines(void *arg) request[id] = NULL; } - err = end_live_test(&t); + err = igt_live_test_end(&t); out_request: for_each_engine(engine, i915, id) @@ -731,7 +929,7 @@ out_request: i915_vma_unpin(batch); i915_vma_put(batch); out_unlock: - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); mutex_unlock(&i915->drm.struct_mutex); return err; } @@ -742,7 +940,8 @@ static int live_sequential_engines(void *arg) struct i915_request *request[I915_NUM_ENGINES] = {}; struct i915_request *prev = NULL; struct intel_engine_cs *engine; - struct live_test t; + intel_wakeref_t wakeref; + struct igt_live_test t; unsigned int id; int err; @@ -753,9 +952,9 @@ static int live_sequential_engines(void *arg) */ mutex_lock(&i915->drm.struct_mutex); - intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(i915); - err = begin_live_test(&t, i915, __func__, ""); + err = igt_live_test_begin(&t, i915, __func__, ""); if (err) goto out_unlock; @@ -838,7 +1037,7 @@ static int live_sequential_engines(void *arg) GEM_BUG_ON(!i915_request_completed(request[id])); } - err = end_live_test(&t); + err = igt_live_test_end(&t); out_request: for_each_engine(engine, i915, id) { @@ -860,11 +1059,183 @@ out_request: i915_request_put(request[id]); } out_unlock: - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); mutex_unlock(&i915->drm.struct_mutex); return err; } +static int +max_batches(struct i915_gem_context *ctx, struct intel_engine_cs *engine) +{ + struct i915_request *rq; + int ret; + + /* + * Before execlists, all contexts share the same ringbuffer. With + * execlists, each context/engine has a separate ringbuffer and + * for the purposes of this test, inexhaustible. + * + * For the global ringbuffer though, we have to be very careful + * that we do not wrap while preventing the execution of requests + * with a unsignaled fence. + */ + if (HAS_EXECLISTS(ctx->i915)) + return INT_MAX; + + rq = i915_request_alloc(engine, ctx); + if (IS_ERR(rq)) { + ret = PTR_ERR(rq); + } else { + int sz; + + ret = rq->ring->size - rq->reserved_space; + i915_request_add(rq); + + sz = rq->ring->emit - rq->head; + if (sz < 0) + sz += rq->ring->size; + ret /= sz; + ret /= 2; /* leave half spare, in case of emergency! */ + } + + return ret; +} + +static int live_breadcrumbs_smoketest(void *arg) +{ + struct drm_i915_private *i915 = arg; + struct smoketest t[I915_NUM_ENGINES]; + unsigned int ncpus = num_online_cpus(); + unsigned long num_waits, num_fences; + struct intel_engine_cs *engine; + struct task_struct **threads; + struct igt_live_test live; + enum intel_engine_id id; + intel_wakeref_t wakeref; + struct drm_file *file; + unsigned int n; + int ret = 0; + + /* + * Smoketest our breadcrumb/signal handling for requests across multiple + * threads. A very simple test to only catch the most egregious of bugs. + * See __igt_breadcrumbs_smoketest(); + * + * On real hardware this time. + */ + + wakeref = intel_runtime_pm_get(i915); + + file = mock_file(i915); + if (IS_ERR(file)) { + ret = PTR_ERR(file); + goto out_rpm; + } + + threads = kcalloc(ncpus * I915_NUM_ENGINES, + sizeof(*threads), + GFP_KERNEL); + if (!threads) { + ret = -ENOMEM; + goto out_file; + } + + memset(&t[0], 0, sizeof(t[0])); + t[0].request_alloc = __live_request_alloc; + t[0].ncontexts = 64; + t[0].contexts = kmalloc_array(t[0].ncontexts, + sizeof(*t[0].contexts), + GFP_KERNEL); + if (!t[0].contexts) { + ret = -ENOMEM; + goto out_threads; + } + + mutex_lock(&i915->drm.struct_mutex); + for (n = 0; n < t[0].ncontexts; n++) { + t[0].contexts[n] = live_context(i915, file); + if (!t[0].contexts[n]) { + ret = -ENOMEM; + goto out_contexts; + } + } + + ret = igt_live_test_begin(&live, i915, __func__, ""); + if (ret) + goto out_contexts; + + for_each_engine(engine, i915, id) { + t[id] = t[0]; + t[id].engine = engine; + t[id].max_batch = max_batches(t[0].contexts[0], engine); + if (t[id].max_batch < 0) { + ret = t[id].max_batch; + mutex_unlock(&i915->drm.struct_mutex); + goto out_flush; + } + /* One ring interleaved between requests from all cpus */ + t[id].max_batch /= num_online_cpus() + 1; + pr_debug("Limiting batches to %d requests on %s\n", + t[id].max_batch, engine->name); + + for (n = 0; n < ncpus; n++) { + struct task_struct *tsk; + + tsk = kthread_run(__igt_breadcrumbs_smoketest, + &t[id], "igt/%d.%d", id, n); + if (IS_ERR(tsk)) { + ret = PTR_ERR(tsk); + mutex_unlock(&i915->drm.struct_mutex); + goto out_flush; + } + + get_task_struct(tsk); + threads[id * ncpus + n] = tsk; + } + } + mutex_unlock(&i915->drm.struct_mutex); + + msleep(jiffies_to_msecs(i915_selftest.timeout_jiffies)); + +out_flush: + num_waits = 0; + num_fences = 0; + for_each_engine(engine, i915, id) { + for (n = 0; n < ncpus; n++) { + struct task_struct *tsk = threads[id * ncpus + n]; + int err; + + if (!tsk) + continue; + + err = kthread_stop(tsk); + if (err < 0 && !ret) + ret = err; + + put_task_struct(tsk); + } + + num_waits += atomic_long_read(&t[id].num_waits); + num_fences += atomic_long_read(&t[id].num_fences); + } + pr_info("Completed %lu waits for %lu fences across %d engines and %d cpus\n", + num_waits, num_fences, RUNTIME_INFO(i915)->num_rings, ncpus); + + mutex_lock(&i915->drm.struct_mutex); + ret = igt_live_test_end(&live) ?: ret; +out_contexts: + mutex_unlock(&i915->drm.struct_mutex); + kfree(t[0].contexts); +out_threads: + kfree(threads); +out_file: + mock_file_free(i915, file); +out_rpm: + intel_runtime_pm_put(i915, wakeref); + + return ret; +} + int i915_request_live_selftests(struct drm_i915_private *i915) { static const struct i915_subtest tests[] = { @@ -872,6 +1243,7 @@ int i915_request_live_selftests(struct drm_i915_private *i915) SUBTEST(live_all_engines), SUBTEST(live_sequential_engines), SUBTEST(live_empty_request), + SUBTEST(live_breadcrumbs_smoketest), }; if (i915_terminally_wedged(&i915->gpu_error)) diff --git a/drivers/gpu/drm/i915/selftests/i915_selftest.c b/drivers/gpu/drm/i915/selftests/i915_selftest.c index 86c54ea37f48..10ef0e636a24 100644 --- a/drivers/gpu/drm/i915/selftests/i915_selftest.c +++ b/drivers/gpu/drm/i915/selftests/i915_selftest.c @@ -197,6 +197,49 @@ int i915_live_selftests(struct pci_dev *pdev) return 0; } +static bool apply_subtest_filter(const char *caller, const char *name) +{ + char *filter, *sep, *tok; + bool result = true; + + filter = kstrdup(i915_selftest.filter, GFP_KERNEL); + for (sep = filter; (tok = strsep(&sep, ","));) { + bool allow = true; + char *sl; + + if (*tok == '!') { + allow = false; + tok++; + } + + if (*tok == '\0') + continue; + + sl = strchr(tok, '/'); + if (sl) { + *sl++ = '\0'; + if (strcmp(tok, caller)) { + if (allow) + result = false; + continue; + } + tok = sl; + } + + if (strcmp(tok, name)) { + if (allow) + result = false; + continue; + } + + result = allow; + break; + } + kfree(filter); + + return result; +} + int __i915_subtests(const char *caller, const struct i915_subtest *st, unsigned int count, @@ -209,6 +252,9 @@ int __i915_subtests(const char *caller, if (signal_pending(current)) return -EINTR; + if (!apply_subtest_filter(caller, st->name)) + continue; + pr_debug(DRIVER_NAME ": Running %s/%s\n", caller, st->name); GEM_TRACE("Running %s/%s\n", caller, st->name); @@ -244,6 +290,7 @@ bool __igt_timeout(unsigned long timeout, const char *fmt, ...) module_param_named(st_random_seed, i915_selftest.random_seed, uint, 0400); module_param_named(st_timeout, i915_selftest.timeout_ms, uint, 0400); +module_param_named(st_filter, i915_selftest.filter, charp, 0400); module_param_named_unsafe(mock_selftests, i915_selftest.mock, int, 0400); MODULE_PARM_DESC(mock_selftests, "Run selftests before loading, using mock hardware (0:disabled [default], 1:run tests then load driver, -1:run tests then exit module)"); diff --git a/drivers/gpu/drm/i915/selftests/i915_timeline.c b/drivers/gpu/drm/i915/selftests/i915_timeline.c index 19f1c6a5c8fb..12ea69b1a1e5 100644 --- a/drivers/gpu/drm/i915/selftests/i915_timeline.c +++ b/drivers/gpu/drm/i915/selftests/i915_timeline.c @@ -4,12 +4,155 @@ * Copyright © 2017-2018 Intel Corporation */ +#include <linux/prime_numbers.h> + #include "../i915_selftest.h" #include "i915_random.h" +#include "igt_flush_test.h" #include "mock_gem_device.h" #include "mock_timeline.h" +static struct page *hwsp_page(struct i915_timeline *tl) +{ + struct drm_i915_gem_object *obj = tl->hwsp_ggtt->obj; + + GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); + return sg_page(obj->mm.pages->sgl); +} + +static unsigned long hwsp_cacheline(struct i915_timeline *tl) +{ + unsigned long address = (unsigned long)page_address(hwsp_page(tl)); + + return (address + tl->hwsp_offset) / CACHELINE_BYTES; +} + +#define CACHELINES_PER_PAGE (PAGE_SIZE / CACHELINE_BYTES) + +struct mock_hwsp_freelist { + struct drm_i915_private *i915; + struct radix_tree_root cachelines; + struct i915_timeline **history; + unsigned long count, max; + struct rnd_state prng; +}; + +enum { + SHUFFLE = BIT(0), +}; + +static void __mock_hwsp_record(struct mock_hwsp_freelist *state, + unsigned int idx, + struct i915_timeline *tl) +{ + tl = xchg(&state->history[idx], tl); + if (tl) { + radix_tree_delete(&state->cachelines, hwsp_cacheline(tl)); + i915_timeline_put(tl); + } +} + +static int __mock_hwsp_timeline(struct mock_hwsp_freelist *state, + unsigned int count, + unsigned int flags) +{ + struct i915_timeline *tl; + unsigned int idx; + + while (count--) { + unsigned long cacheline; + int err; + + tl = i915_timeline_create(state->i915, "mock", NULL); + if (IS_ERR(tl)) + return PTR_ERR(tl); + + cacheline = hwsp_cacheline(tl); + err = radix_tree_insert(&state->cachelines, cacheline, tl); + if (err) { + if (err == -EEXIST) { + pr_err("HWSP cacheline %lu already used; duplicate allocation!\n", + cacheline); + } + i915_timeline_put(tl); + return err; + } + + idx = state->count++ % state->max; + __mock_hwsp_record(state, idx, tl); + } + + if (flags & SHUFFLE) + i915_prandom_shuffle(state->history, + sizeof(*state->history), + min(state->count, state->max), + &state->prng); + + count = i915_prandom_u32_max_state(min(state->count, state->max), + &state->prng); + while (count--) { + idx = --state->count % state->max; + __mock_hwsp_record(state, idx, NULL); + } + + return 0; +} + +static int mock_hwsp_freelist(void *arg) +{ + struct mock_hwsp_freelist state; + const struct { + const char *name; + unsigned int flags; + } phases[] = { + { "linear", 0 }, + { "shuffled", SHUFFLE }, + { }, + }, *p; + unsigned int na; + int err = 0; + + INIT_RADIX_TREE(&state.cachelines, GFP_KERNEL); + state.prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed); + + state.i915 = mock_gem_device(); + if (!state.i915) + return -ENOMEM; + + /* + * Create a bunch of timelines and check that their HWSP do not overlap. + * Free some, and try again. + */ + + state.max = PAGE_SIZE / sizeof(*state.history); + state.count = 0; + state.history = kcalloc(state.max, sizeof(*state.history), GFP_KERNEL); + if (!state.history) { + err = -ENOMEM; + goto err_put; + } + + mutex_lock(&state.i915->drm.struct_mutex); + for (p = phases; p->name; p++) { + pr_debug("%s(%s)\n", __func__, p->name); + for_each_prime_number_from(na, 1, 2 * CACHELINES_PER_PAGE) { + err = __mock_hwsp_timeline(&state, na, p->flags); + if (err) + goto out; + } + } + +out: + for (na = 0; na < state.max; na++) + __mock_hwsp_record(&state, na, NULL); + mutex_unlock(&state.i915->drm.struct_mutex); + kfree(state.history); +err_put: + drm_dev_put(&state.i915->drm); + return err; +} + struct __igt_sync { const char *name; u32 seqno; @@ -256,12 +399,331 @@ static int bench_sync(void *arg) return 0; } -int i915_gem_timeline_mock_selftests(void) +int i915_timeline_mock_selftests(void) { static const struct i915_subtest tests[] = { + SUBTEST(mock_hwsp_freelist), SUBTEST(igt_sync), SUBTEST(bench_sync), }; return i915_subtests(tests, NULL); } + +static int emit_ggtt_store_dw(struct i915_request *rq, u32 addr, u32 value) +{ + u32 *cs; + + cs = intel_ring_begin(rq, 4); + if (IS_ERR(cs)) + return PTR_ERR(cs); + + if (INTEL_GEN(rq->i915) >= 8) { + *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; + *cs++ = addr; + *cs++ = 0; + *cs++ = value; + } else if (INTEL_GEN(rq->i915) >= 4) { + *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; + *cs++ = 0; + *cs++ = addr; + *cs++ = value; + } else { + *cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; + *cs++ = addr; + *cs++ = value; + *cs++ = MI_NOOP; + } + + intel_ring_advance(rq, cs); + + return 0; +} + +static struct i915_request * +tl_write(struct i915_timeline *tl, struct intel_engine_cs *engine, u32 value) +{ + struct i915_request *rq; + int err; + + lockdep_assert_held(&tl->i915->drm.struct_mutex); /* lazy rq refs */ + + err = i915_timeline_pin(tl); + if (err) { + rq = ERR_PTR(err); + goto out; + } + + rq = i915_request_alloc(engine, engine->i915->kernel_context); + if (IS_ERR(rq)) + goto out_unpin; + + err = emit_ggtt_store_dw(rq, tl->hwsp_offset, value); + i915_request_add(rq); + if (err) + rq = ERR_PTR(err); + +out_unpin: + i915_timeline_unpin(tl); +out: + if (IS_ERR(rq)) + pr_err("Failed to write to timeline!\n"); + return rq; +} + +static struct i915_timeline * +checked_i915_timeline_create(struct drm_i915_private *i915) +{ + struct i915_timeline *tl; + + tl = i915_timeline_create(i915, "live", NULL); + if (IS_ERR(tl)) + return tl; + + if (*tl->hwsp_seqno != tl->seqno) { + pr_err("Timeline created with incorrect breadcrumb, found %x, expected %x\n", + *tl->hwsp_seqno, tl->seqno); + i915_timeline_put(tl); + return ERR_PTR(-EINVAL); + } + + return tl; +} + +static int live_hwsp_engine(void *arg) +{ +#define NUM_TIMELINES 4096 + struct drm_i915_private *i915 = arg; + struct i915_timeline **timelines; + struct intel_engine_cs *engine; + enum intel_engine_id id; + intel_wakeref_t wakeref; + unsigned long count, n; + int err = 0; + + /* + * Create a bunch of timelines and check we can write + * independently to each of their breadcrumb slots. + */ + + timelines = kvmalloc_array(NUM_TIMELINES * I915_NUM_ENGINES, + sizeof(*timelines), + GFP_KERNEL); + if (!timelines) + return -ENOMEM; + + mutex_lock(&i915->drm.struct_mutex); + wakeref = intel_runtime_pm_get(i915); + + count = 0; + for_each_engine(engine, i915, id) { + if (!intel_engine_can_store_dword(engine)) + continue; + + for (n = 0; n < NUM_TIMELINES; n++) { + struct i915_timeline *tl; + struct i915_request *rq; + + tl = checked_i915_timeline_create(i915); + if (IS_ERR(tl)) { + err = PTR_ERR(tl); + goto out; + } + + rq = tl_write(tl, engine, count); + if (IS_ERR(rq)) { + i915_timeline_put(tl); + err = PTR_ERR(rq); + goto out; + } + + timelines[count++] = tl; + } + } + +out: + if (igt_flush_test(i915, I915_WAIT_LOCKED)) + err = -EIO; + + for (n = 0; n < count; n++) { + struct i915_timeline *tl = timelines[n]; + + if (!err && *tl->hwsp_seqno != n) { + pr_err("Invalid seqno stored in timeline %lu, found 0x%x\n", + n, *tl->hwsp_seqno); + err = -EINVAL; + } + i915_timeline_put(tl); + } + + intel_runtime_pm_put(i915, wakeref); + mutex_unlock(&i915->drm.struct_mutex); + + kvfree(timelines); + + return err; +#undef NUM_TIMELINES +} + +static int live_hwsp_alternate(void *arg) +{ +#define NUM_TIMELINES 4096 + struct drm_i915_private *i915 = arg; + struct i915_timeline **timelines; + struct intel_engine_cs *engine; + enum intel_engine_id id; + intel_wakeref_t wakeref; + unsigned long count, n; + int err = 0; + + /* + * Create a bunch of timelines and check we can write + * independently to each of their breadcrumb slots with adjacent + * engines. + */ + + timelines = kvmalloc_array(NUM_TIMELINES * I915_NUM_ENGINES, + sizeof(*timelines), + GFP_KERNEL); + if (!timelines) + return -ENOMEM; + + mutex_lock(&i915->drm.struct_mutex); + wakeref = intel_runtime_pm_get(i915); + + count = 0; + for (n = 0; n < NUM_TIMELINES; n++) { + for_each_engine(engine, i915, id) { + struct i915_timeline *tl; + struct i915_request *rq; + + if (!intel_engine_can_store_dword(engine)) + continue; + + tl = checked_i915_timeline_create(i915); + if (IS_ERR(tl)) { + err = PTR_ERR(tl); + goto out; + } + + rq = tl_write(tl, engine, count); + if (IS_ERR(rq)) { + i915_timeline_put(tl); + err = PTR_ERR(rq); + goto out; + } + + timelines[count++] = tl; + } + } + +out: + if (igt_flush_test(i915, I915_WAIT_LOCKED)) + err = -EIO; + + for (n = 0; n < count; n++) { + struct i915_timeline *tl = timelines[n]; + + if (!err && *tl->hwsp_seqno != n) { + pr_err("Invalid seqno stored in timeline %lu, found 0x%x\n", + n, *tl->hwsp_seqno); + err = -EINVAL; + } + i915_timeline_put(tl); + } + + intel_runtime_pm_put(i915, wakeref); + mutex_unlock(&i915->drm.struct_mutex); + + kvfree(timelines); + + return err; +#undef NUM_TIMELINES +} + +static int live_hwsp_recycle(void *arg) +{ + struct drm_i915_private *i915 = arg; + struct intel_engine_cs *engine; + enum intel_engine_id id; + intel_wakeref_t wakeref; + unsigned long count; + int err = 0; + + /* + * Check seqno writes into one timeline at a time. We expect to + * recycle the breadcrumb slot between iterations and neither + * want to confuse ourselves or the GPU. + */ + + mutex_lock(&i915->drm.struct_mutex); + wakeref = intel_runtime_pm_get(i915); + + count = 0; + for_each_engine(engine, i915, id) { + IGT_TIMEOUT(end_time); + + if (!intel_engine_can_store_dword(engine)) + continue; + + do { + struct i915_timeline *tl; + struct i915_request *rq; + + tl = checked_i915_timeline_create(i915); + if (IS_ERR(tl)) { + err = PTR_ERR(tl); + goto out; + } + + rq = tl_write(tl, engine, count); + if (IS_ERR(rq)) { + i915_timeline_put(tl); + err = PTR_ERR(rq); + goto out; + } + + if (i915_request_wait(rq, + I915_WAIT_LOCKED, + HZ / 5) < 0) { + pr_err("Wait for timeline writes timed out!\n"); + i915_timeline_put(tl); + err = -EIO; + goto out; + } + + if (*tl->hwsp_seqno != count) { + pr_err("Invalid seqno stored in timeline %lu, found 0x%x\n", + count, *tl->hwsp_seqno); + err = -EINVAL; + } + + i915_timeline_put(tl); + count++; + + if (err) + goto out; + + i915_timelines_park(i915); /* Encourage recycling! */ + } while (!__igt_timeout(end_time, NULL)); + } + +out: + if (igt_flush_test(i915, I915_WAIT_LOCKED)) + err = -EIO; + intel_runtime_pm_put(i915, wakeref); + mutex_unlock(&i915->drm.struct_mutex); + + return err; +} + +int i915_timeline_live_selftests(struct drm_i915_private *i915) +{ + static const struct i915_subtest tests[] = { + SUBTEST(live_hwsp_recycle), + SUBTEST(live_hwsp_engine), + SUBTEST(live_hwsp_alternate), + }; + + return i915_subtests(tests, i915); +} diff --git a/drivers/gpu/drm/i915/selftests/i915_vma.c b/drivers/gpu/drm/i915/selftests/i915_vma.c index ffa74290e054..cf1de82741fa 100644 --- a/drivers/gpu/drm/i915/selftests/i915_vma.c +++ b/drivers/gpu/drm/i915/selftests/i915_vma.c @@ -28,6 +28,7 @@ #include "mock_gem_device.h" #include "mock_context.h" +#include "mock_gtt.h" static bool assert_vma(struct i915_vma *vma, struct drm_i915_gem_object *obj, @@ -141,7 +142,8 @@ static int create_vmas(struct drm_i915_private *i915, static int igt_vma_create(void *arg) { - struct drm_i915_private *i915 = arg; + struct i915_ggtt *ggtt = arg; + struct drm_i915_private *i915 = ggtt->vm.i915; struct drm_i915_gem_object *obj, *on; struct i915_gem_context *ctx, *cn; unsigned long num_obj, num_ctx; @@ -245,7 +247,7 @@ static bool assert_pin_einval(const struct i915_vma *vma, static int igt_vma_pin1(void *arg) { - struct drm_i915_private *i915 = arg; + struct i915_ggtt *ggtt = arg; const struct pin_mode modes[] = { #define VALID(sz, fl) { .size = (sz), .flags = (fl), .assert = assert_pin_valid, .string = #sz ", " #fl ", (valid) " } #define __INVALID(sz, fl, check, eval) { .size = (sz), .flags = (fl), .assert = (check), .string = #sz ", " #fl ", (invalid " #eval ")" } @@ -256,30 +258,30 @@ static int igt_vma_pin1(void *arg) VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | 4096), VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | 8192), - VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | (i915->ggtt.mappable_end - 4096)), - VALID(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_BIAS | (i915->ggtt.mappable_end - 4096)), - VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | (i915->ggtt.vm.total - 4096)), - - VALID(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_FIXED | (i915->ggtt.mappable_end - 4096)), - INVALID(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_FIXED | i915->ggtt.mappable_end), - VALID(0, PIN_GLOBAL | PIN_OFFSET_FIXED | (i915->ggtt.vm.total - 4096)), - INVALID(0, PIN_GLOBAL | PIN_OFFSET_FIXED | i915->ggtt.vm.total), + VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)), + VALID(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)), + VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->vm.total - 4096)), + + VALID(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_FIXED | (ggtt->mappable_end - 4096)), + INVALID(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_FIXED | ggtt->mappable_end), + VALID(0, PIN_GLOBAL | PIN_OFFSET_FIXED | (ggtt->vm.total - 4096)), + INVALID(0, PIN_GLOBAL | PIN_OFFSET_FIXED | ggtt->vm.total), INVALID(0, PIN_GLOBAL | PIN_OFFSET_FIXED | round_down(U64_MAX, PAGE_SIZE)), VALID(4096, PIN_GLOBAL), VALID(8192, PIN_GLOBAL), - VALID(i915->ggtt.mappable_end - 4096, PIN_GLOBAL | PIN_MAPPABLE), - VALID(i915->ggtt.mappable_end, PIN_GLOBAL | PIN_MAPPABLE), - NOSPACE(i915->ggtt.mappable_end + 4096, PIN_GLOBAL | PIN_MAPPABLE), - VALID(i915->ggtt.vm.total - 4096, PIN_GLOBAL), - VALID(i915->ggtt.vm.total, PIN_GLOBAL), - NOSPACE(i915->ggtt.vm.total + 4096, PIN_GLOBAL), + VALID(ggtt->mappable_end - 4096, PIN_GLOBAL | PIN_MAPPABLE), + VALID(ggtt->mappable_end, PIN_GLOBAL | PIN_MAPPABLE), + NOSPACE(ggtt->mappable_end + 4096, PIN_GLOBAL | PIN_MAPPABLE), + VALID(ggtt->vm.total - 4096, PIN_GLOBAL), + VALID(ggtt->vm.total, PIN_GLOBAL), + NOSPACE(ggtt->vm.total + 4096, PIN_GLOBAL), NOSPACE(round_down(U64_MAX, PAGE_SIZE), PIN_GLOBAL), - INVALID(8192, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_FIXED | (i915->ggtt.mappable_end - 4096)), - INVALID(8192, PIN_GLOBAL | PIN_OFFSET_FIXED | (i915->ggtt.vm.total - 4096)), + INVALID(8192, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_FIXED | (ggtt->mappable_end - 4096)), + INVALID(8192, PIN_GLOBAL | PIN_OFFSET_FIXED | (ggtt->vm.total - 4096)), INVALID(8192, PIN_GLOBAL | PIN_OFFSET_FIXED | (round_down(U64_MAX, PAGE_SIZE) - 4096)), - VALID(8192, PIN_GLOBAL | PIN_OFFSET_BIAS | (i915->ggtt.mappable_end - 4096)), + VALID(8192, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)), #if !IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) /* Misusing BIAS is a programming error (it is not controllable @@ -287,10 +289,10 @@ static int igt_vma_pin1(void *arg) * However, the tests are still quite interesting for checking * variable start, end and size. */ - NOSPACE(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_BIAS | i915->ggtt.mappable_end), - NOSPACE(0, PIN_GLOBAL | PIN_OFFSET_BIAS | i915->ggtt.vm.total), - NOSPACE(8192, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_BIAS | (i915->ggtt.mappable_end - 4096)), - NOSPACE(8192, PIN_GLOBAL | PIN_OFFSET_BIAS | (i915->ggtt.vm.total - 4096)), + NOSPACE(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_BIAS | ggtt->mappable_end), + NOSPACE(0, PIN_GLOBAL | PIN_OFFSET_BIAS | ggtt->vm.total), + NOSPACE(8192, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)), + NOSPACE(8192, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->vm.total - 4096)), #endif { }, #undef NOSPACE @@ -306,13 +308,13 @@ static int igt_vma_pin1(void *arg) * focusing on error handling of boundary conditions. */ - GEM_BUG_ON(!drm_mm_clean(&i915->ggtt.vm.mm)); + GEM_BUG_ON(!drm_mm_clean(&ggtt->vm.mm)); - obj = i915_gem_object_create_internal(i915, PAGE_SIZE); + obj = i915_gem_object_create_internal(ggtt->vm.i915, PAGE_SIZE); if (IS_ERR(obj)) return PTR_ERR(obj); - vma = checked_vma_instance(obj, &i915->ggtt.vm, NULL); + vma = checked_vma_instance(obj, &ggtt->vm, NULL); if (IS_ERR(vma)) goto out; @@ -403,8 +405,8 @@ static unsigned int rotated_size(const struct intel_rotation_plane_info *a, static int igt_vma_rotate(void *arg) { - struct drm_i915_private *i915 = arg; - struct i915_address_space *vm = &i915->ggtt.vm; + struct i915_ggtt *ggtt = arg; + struct i915_address_space *vm = &ggtt->vm; struct drm_i915_gem_object *obj; const struct intel_rotation_plane_info planes[] = { { .width = 1, .height = 1, .stride = 1 }, @@ -431,7 +433,7 @@ static int igt_vma_rotate(void *arg) * that the page layout within the rotated VMA match our expectations. */ - obj = i915_gem_object_create_internal(i915, max_pages * PAGE_SIZE); + obj = i915_gem_object_create_internal(vm->i915, max_pages * PAGE_SIZE); if (IS_ERR(obj)) goto out; @@ -602,8 +604,8 @@ static bool assert_pin(struct i915_vma *vma, static int igt_vma_partial(void *arg) { - struct drm_i915_private *i915 = arg; - struct i915_address_space *vm = &i915->ggtt.vm; + struct i915_ggtt *ggtt = arg; + struct i915_address_space *vm = &ggtt->vm; const unsigned int npages = 1021; /* prime! */ struct drm_i915_gem_object *obj; const struct phase { @@ -621,7 +623,7 @@ static int igt_vma_partial(void *arg) * we are returned the same VMA when we later request the same range. */ - obj = i915_gem_object_create_internal(i915, npages*PAGE_SIZE); + obj = i915_gem_object_create_internal(vm->i915, npages * PAGE_SIZE); if (IS_ERR(obj)) goto out; @@ -670,7 +672,7 @@ static int igt_vma_partial(void *arg) } count = 0; - list_for_each_entry(vma, &obj->vma_list, obj_link) + list_for_each_entry(vma, &obj->vma.list, obj_link) count++; if (count != nvma) { pr_err("(%s) All partial vma were not recorded on the obj->vma_list: found %u, expected %u\n", @@ -699,7 +701,7 @@ static int igt_vma_partial(void *arg) i915_vma_unpin(vma); count = 0; - list_for_each_entry(vma, &obj->vma_list, obj_link) + list_for_each_entry(vma, &obj->vma.list, obj_link) count++; if (count != nvma) { pr_err("(%s) allocated an extra full vma!\n", p->name); @@ -723,17 +725,24 @@ int i915_vma_mock_selftests(void) SUBTEST(igt_vma_partial), }; struct drm_i915_private *i915; + struct i915_ggtt ggtt; int err; i915 = mock_gem_device(); if (!i915) return -ENOMEM; + mock_init_ggtt(i915, &ggtt); + mutex_lock(&i915->drm.struct_mutex); - err = i915_subtests(tests, i915); + err = i915_subtests(tests, &ggtt); + mock_device_flush(i915); mutex_unlock(&i915->drm.struct_mutex); + i915_gem_drain_freed_objects(i915); + + mock_fini_ggtt(&ggtt); drm_dev_put(&i915->drm); + return err; } - diff --git a/drivers/gpu/drm/i915/selftests/igt_live_test.c b/drivers/gpu/drm/i915/selftests/igt_live_test.c new file mode 100644 index 000000000000..3e902761cd16 --- /dev/null +++ b/drivers/gpu/drm/i915/selftests/igt_live_test.c @@ -0,0 +1,78 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright © 2018 Intel Corporation + */ + +#include "../i915_drv.h" + +#include "../i915_selftest.h" +#include "igt_flush_test.h" +#include "igt_live_test.h" + +int igt_live_test_begin(struct igt_live_test *t, + struct drm_i915_private *i915, + const char *func, + const char *name) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + int err; + + lockdep_assert_held(&i915->drm.struct_mutex); + + t->i915 = i915; + t->func = func; + t->name = name; + + err = i915_gem_wait_for_idle(i915, + I915_WAIT_INTERRUPTIBLE | + I915_WAIT_LOCKED, + MAX_SCHEDULE_TIMEOUT); + if (err) { + pr_err("%s(%s): failed to idle before, with err=%d!", + func, name, err); + return err; + } + + t->reset_global = i915_reset_count(&i915->gpu_error); + + for_each_engine(engine, i915, id) + t->reset_engine[id] = + i915_reset_engine_count(&i915->gpu_error, engine); + + return 0; +} + +int igt_live_test_end(struct igt_live_test *t) +{ + struct drm_i915_private *i915 = t->i915; + struct intel_engine_cs *engine; + enum intel_engine_id id; + + lockdep_assert_held(&i915->drm.struct_mutex); + + if (igt_flush_test(i915, I915_WAIT_LOCKED)) + return -EIO; + + if (t->reset_global != i915_reset_count(&i915->gpu_error)) { + pr_err("%s(%s): GPU was reset %d times!\n", + t->func, t->name, + i915_reset_count(&i915->gpu_error) - t->reset_global); + return -EIO; + } + + for_each_engine(engine, i915, id) { + if (t->reset_engine[id] == + i915_reset_engine_count(&i915->gpu_error, engine)) + continue; + + pr_err("%s(%s): engine '%s' was reset %d times!\n", + t->func, t->name, engine->name, + i915_reset_engine_count(&i915->gpu_error, engine) - + t->reset_engine[id]); + return -EIO; + } + + return 0; +} diff --git a/drivers/gpu/drm/i915/selftests/igt_live_test.h b/drivers/gpu/drm/i915/selftests/igt_live_test.h new file mode 100644 index 000000000000..c0e9f99d50de --- /dev/null +++ b/drivers/gpu/drm/i915/selftests/igt_live_test.h @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright © 2019 Intel Corporation + */ + +#ifndef IGT_LIVE_TEST_H +#define IGT_LIVE_TEST_H + +#include "../i915_gem.h" + +struct drm_i915_private; + +struct igt_live_test { + struct drm_i915_private *i915; + const char *func; + const char *name; + + unsigned int reset_global; + unsigned int reset_engine[I915_NUM_ENGINES]; +}; + +/* + * Flush the GPU state before and after the test to ensure that no residual + * code is running on the GPU that may affect this test. Also compare the + * state before and after the test and alert if it unexpectedly changes, + * e.g. if the GPU was reset. + */ +int igt_live_test_begin(struct igt_live_test *t, + struct drm_i915_private *i915, + const char *func, + const char *name); +int igt_live_test_end(struct igt_live_test *t); + +#endif /* IGT_LIVE_TEST_H */ diff --git a/drivers/gpu/drm/i915/selftests/igt_reset.c b/drivers/gpu/drm/i915/selftests/igt_reset.c new file mode 100644 index 000000000000..208a966da8ca --- /dev/null +++ b/drivers/gpu/drm/i915/selftests/igt_reset.c @@ -0,0 +1,44 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright © 2018 Intel Corporation + */ + +#include "igt_reset.h" + +#include "../i915_drv.h" +#include "../intel_ringbuffer.h" + +void igt_global_reset_lock(struct drm_i915_private *i915) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + + pr_debug("%s: current gpu_error=%08lx\n", + __func__, i915->gpu_error.flags); + + while (test_and_set_bit(I915_RESET_BACKOFF, &i915->gpu_error.flags)) + wait_event(i915->gpu_error.reset_queue, + !test_bit(I915_RESET_BACKOFF, + &i915->gpu_error.flags)); + + for_each_engine(engine, i915, id) { + while (test_and_set_bit(I915_RESET_ENGINE + id, + &i915->gpu_error.flags)) + wait_on_bit(&i915->gpu_error.flags, + I915_RESET_ENGINE + id, + TASK_UNINTERRUPTIBLE); + } +} + +void igt_global_reset_unlock(struct drm_i915_private *i915) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + + for_each_engine(engine, i915, id) + clear_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags); + + clear_bit(I915_RESET_BACKOFF, &i915->gpu_error.flags); + wake_up_all(&i915->gpu_error.reset_queue); +} diff --git a/drivers/gpu/drm/i915/selftests/igt_reset.h b/drivers/gpu/drm/i915/selftests/igt_reset.h new file mode 100644 index 000000000000..5f0234d045d5 --- /dev/null +++ b/drivers/gpu/drm/i915/selftests/igt_reset.h @@ -0,0 +1,15 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright © 2018 Intel Corporation + */ + +#ifndef __I915_SELFTESTS_IGT_RESET_H__ +#define __I915_SELFTESTS_IGT_RESET_H__ + +#include "../i915_drv.h" + +void igt_global_reset_lock(struct drm_i915_private *i915); +void igt_global_reset_unlock(struct drm_i915_private *i915); + +#endif diff --git a/drivers/gpu/drm/i915/selftests/igt_spinner.c b/drivers/gpu/drm/i915/selftests/igt_spinner.c new file mode 100644 index 000000000000..9ebd9225684e --- /dev/null +++ b/drivers/gpu/drm/i915/selftests/igt_spinner.c @@ -0,0 +1,194 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright © 2018 Intel Corporation + */ + +#include "igt_spinner.h" + +int igt_spinner_init(struct igt_spinner *spin, struct drm_i915_private *i915) +{ + unsigned int mode; + void *vaddr; + int err; + + GEM_BUG_ON(INTEL_GEN(i915) < 8); + + memset(spin, 0, sizeof(*spin)); + spin->i915 = i915; + + spin->hws = i915_gem_object_create_internal(i915, PAGE_SIZE); + if (IS_ERR(spin->hws)) { + err = PTR_ERR(spin->hws); + goto err; + } + + spin->obj = i915_gem_object_create_internal(i915, PAGE_SIZE); + if (IS_ERR(spin->obj)) { + err = PTR_ERR(spin->obj); + goto err_hws; + } + + i915_gem_object_set_cache_level(spin->hws, I915_CACHE_LLC); + vaddr = i915_gem_object_pin_map(spin->hws, I915_MAP_WB); + if (IS_ERR(vaddr)) { + err = PTR_ERR(vaddr); + goto err_obj; + } + spin->seqno = memset(vaddr, 0xff, PAGE_SIZE); + + mode = i915_coherent_map_type(i915); + vaddr = i915_gem_object_pin_map(spin->obj, mode); + if (IS_ERR(vaddr)) { + err = PTR_ERR(vaddr); + goto err_unpin_hws; + } + spin->batch = vaddr; + + return 0; + +err_unpin_hws: + i915_gem_object_unpin_map(spin->hws); +err_obj: + i915_gem_object_put(spin->obj); +err_hws: + i915_gem_object_put(spin->hws); +err: + return err; +} + +static unsigned int seqno_offset(u64 fence) +{ + return offset_in_page(sizeof(u32) * fence); +} + +static u64 hws_address(const struct i915_vma *hws, + const struct i915_request *rq) +{ + return hws->node.start + seqno_offset(rq->fence.context); +} + +static int move_to_active(struct i915_vma *vma, + struct i915_request *rq, + unsigned int flags) +{ + int err; + + err = i915_vma_move_to_active(vma, rq, flags); + if (err) + return err; + + if (!i915_gem_object_has_active_reference(vma->obj)) { + i915_gem_object_get(vma->obj); + i915_gem_object_set_active_reference(vma->obj); + } + + return 0; +} + +struct i915_request * +igt_spinner_create_request(struct igt_spinner *spin, + struct i915_gem_context *ctx, + struct intel_engine_cs *engine, + u32 arbitration_command) +{ + struct i915_address_space *vm = &ctx->ppgtt->vm; + struct i915_request *rq = NULL; + struct i915_vma *hws, *vma; + u32 *batch; + int err; + + vma = i915_vma_instance(spin->obj, vm, NULL); + if (IS_ERR(vma)) + return ERR_CAST(vma); + + hws = i915_vma_instance(spin->hws, vm, NULL); + if (IS_ERR(hws)) + return ERR_CAST(hws); + + err = i915_vma_pin(vma, 0, 0, PIN_USER); + if (err) + return ERR_PTR(err); + + err = i915_vma_pin(hws, 0, 0, PIN_USER); + if (err) + goto unpin_vma; + + rq = i915_request_alloc(engine, ctx); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto unpin_hws; + } + + err = move_to_active(vma, rq, 0); + if (err) + goto cancel_rq; + + err = move_to_active(hws, rq, 0); + if (err) + goto cancel_rq; + + batch = spin->batch; + + *batch++ = MI_STORE_DWORD_IMM_GEN4; + *batch++ = lower_32_bits(hws_address(hws, rq)); + *batch++ = upper_32_bits(hws_address(hws, rq)); + *batch++ = rq->fence.seqno; + + *batch++ = arbitration_command; + + *batch++ = MI_BATCH_BUFFER_START | 1 << 8 | 1; + *batch++ = lower_32_bits(vma->node.start); + *batch++ = upper_32_bits(vma->node.start); + *batch++ = MI_BATCH_BUFFER_END; /* not reached */ + + i915_gem_chipset_flush(spin->i915); + + err = engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, 0); + +cancel_rq: + if (err) { + i915_request_skip(rq, err); + i915_request_add(rq); + } +unpin_hws: + i915_vma_unpin(hws); +unpin_vma: + i915_vma_unpin(vma); + return err ? ERR_PTR(err) : rq; +} + +static u32 +hws_seqno(const struct igt_spinner *spin, const struct i915_request *rq) +{ + u32 *seqno = spin->seqno + seqno_offset(rq->fence.context); + + return READ_ONCE(*seqno); +} + +void igt_spinner_end(struct igt_spinner *spin) +{ + *spin->batch = MI_BATCH_BUFFER_END; + i915_gem_chipset_flush(spin->i915); +} + +void igt_spinner_fini(struct igt_spinner *spin) +{ + igt_spinner_end(spin); + + i915_gem_object_unpin_map(spin->obj); + i915_gem_object_put(spin->obj); + + i915_gem_object_unpin_map(spin->hws); + i915_gem_object_put(spin->hws); +} + +bool igt_wait_for_spinner(struct igt_spinner *spin, struct i915_request *rq) +{ + return !(wait_for_us(i915_seqno_passed(hws_seqno(spin, rq), + rq->fence.seqno), + 10) && + wait_for(i915_seqno_passed(hws_seqno(spin, rq), + rq->fence.seqno), + 1000)); +} diff --git a/drivers/gpu/drm/i915/selftests/igt_spinner.h b/drivers/gpu/drm/i915/selftests/igt_spinner.h new file mode 100644 index 000000000000..391777c76dc7 --- /dev/null +++ b/drivers/gpu/drm/i915/selftests/igt_spinner.h @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright © 2018 Intel Corporation + */ + +#ifndef __I915_SELFTESTS_IGT_SPINNER_H__ +#define __I915_SELFTESTS_IGT_SPINNER_H__ + +#include "../i915_selftest.h" + +#include "../i915_drv.h" +#include "../i915_request.h" +#include "../intel_ringbuffer.h" +#include "../i915_gem_context.h" + +struct igt_spinner { + struct drm_i915_private *i915; + struct drm_i915_gem_object *hws; + struct drm_i915_gem_object *obj; + u32 *batch; + void *seqno; +}; + +int igt_spinner_init(struct igt_spinner *spin, struct drm_i915_private *i915); +void igt_spinner_fini(struct igt_spinner *spin); + +struct i915_request * +igt_spinner_create_request(struct igt_spinner *spin, + struct i915_gem_context *ctx, + struct intel_engine_cs *engine, + u32 arbitration_command); +void igt_spinner_end(struct igt_spinner *spin); + +bool igt_wait_for_spinner(struct igt_spinner *spin, struct i915_request *rq); + +#endif diff --git a/drivers/gpu/drm/i915/selftests/intel_breadcrumbs.c b/drivers/gpu/drm/i915/selftests/intel_breadcrumbs.c deleted file mode 100644 index f03b407fdbe2..000000000000 --- a/drivers/gpu/drm/i915/selftests/intel_breadcrumbs.c +++ /dev/null @@ -1,470 +0,0 @@ -/* - * Copyright © 2016 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - * - */ - -#include "../i915_selftest.h" -#include "i915_random.h" - -#include "mock_gem_device.h" -#include "mock_engine.h" - -static int check_rbtree(struct intel_engine_cs *engine, - const unsigned long *bitmap, - const struct intel_wait *waiters, - const int count) -{ - struct intel_breadcrumbs *b = &engine->breadcrumbs; - struct rb_node *rb; - int n; - - if (&b->irq_wait->node != rb_first(&b->waiters)) { - pr_err("First waiter does not match first element of wait-tree\n"); - return -EINVAL; - } - - n = find_first_bit(bitmap, count); - for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) { - struct intel_wait *w = container_of(rb, typeof(*w), node); - int idx = w - waiters; - - if (!test_bit(idx, bitmap)) { - pr_err("waiter[%d, seqno=%d] removed but still in wait-tree\n", - idx, w->seqno); - return -EINVAL; - } - - if (n != idx) { - pr_err("waiter[%d, seqno=%d] does not match expected next element in tree [%d]\n", - idx, w->seqno, n); - return -EINVAL; - } - - n = find_next_bit(bitmap, count, n + 1); - } - - return 0; -} - -static int check_completion(struct intel_engine_cs *engine, - const unsigned long *bitmap, - const struct intel_wait *waiters, - const int count) -{ - int n; - - for (n = 0; n < count; n++) { - if (intel_wait_complete(&waiters[n]) != !!test_bit(n, bitmap)) - continue; - - pr_err("waiter[%d, seqno=%d] is %s, but expected %s\n", - n, waiters[n].seqno, - intel_wait_complete(&waiters[n]) ? "complete" : "active", - test_bit(n, bitmap) ? "active" : "complete"); - return -EINVAL; - } - - return 0; -} - -static int check_rbtree_empty(struct intel_engine_cs *engine) -{ - struct intel_breadcrumbs *b = &engine->breadcrumbs; - - if (b->irq_wait) { - pr_err("Empty breadcrumbs still has a waiter\n"); - return -EINVAL; - } - - if (!RB_EMPTY_ROOT(&b->waiters)) { - pr_err("Empty breadcrumbs, but wait-tree not empty\n"); - return -EINVAL; - } - - return 0; -} - -static int igt_random_insert_remove(void *arg) -{ - const u32 seqno_bias = 0x1000; - I915_RND_STATE(prng); - struct intel_engine_cs *engine = arg; - struct intel_wait *waiters; - const int count = 4096; - unsigned int *order; - unsigned long *bitmap; - int err = -ENOMEM; - int n; - - mock_engine_reset(engine); - - waiters = kvmalloc_array(count, sizeof(*waiters), GFP_KERNEL); - if (!waiters) - goto out_engines; - - bitmap = kcalloc(DIV_ROUND_UP(count, BITS_PER_LONG), sizeof(*bitmap), - GFP_KERNEL); - if (!bitmap) - goto out_waiters; - - order = i915_random_order(count, &prng); - if (!order) - goto out_bitmap; - - for (n = 0; n < count; n++) - intel_wait_init_for_seqno(&waiters[n], seqno_bias + n); - - err = check_rbtree(engine, bitmap, waiters, count); - if (err) - goto out_order; - - /* Add and remove waiters into the rbtree in random order. At each - * step, we verify that the rbtree is correctly ordered. - */ - for (n = 0; n < count; n++) { - int i = order[n]; - - intel_engine_add_wait(engine, &waiters[i]); - __set_bit(i, bitmap); - - err = check_rbtree(engine, bitmap, waiters, count); - if (err) - goto out_order; - } - - i915_random_reorder(order, count, &prng); - for (n = 0; n < count; n++) { - int i = order[n]; - - intel_engine_remove_wait(engine, &waiters[i]); - __clear_bit(i, bitmap); - - err = check_rbtree(engine, bitmap, waiters, count); - if (err) - goto out_order; - } - - err = check_rbtree_empty(engine); -out_order: - kfree(order); -out_bitmap: - kfree(bitmap); -out_waiters: - kvfree(waiters); -out_engines: - mock_engine_flush(engine); - return err; -} - -static int igt_insert_complete(void *arg) -{ - const u32 seqno_bias = 0x1000; - struct intel_engine_cs *engine = arg; - struct intel_wait *waiters; - const int count = 4096; - unsigned long *bitmap; - int err = -ENOMEM; - int n, m; - - mock_engine_reset(engine); - - waiters = kvmalloc_array(count, sizeof(*waiters), GFP_KERNEL); - if (!waiters) - goto out_engines; - - bitmap = kcalloc(DIV_ROUND_UP(count, BITS_PER_LONG), sizeof(*bitmap), - GFP_KERNEL); - if (!bitmap) - goto out_waiters; - - for (n = 0; n < count; n++) { - intel_wait_init_for_seqno(&waiters[n], n + seqno_bias); - intel_engine_add_wait(engine, &waiters[n]); - __set_bit(n, bitmap); - } - err = check_rbtree(engine, bitmap, waiters, count); - if (err) - goto out_bitmap; - - /* On each step, we advance the seqno so that several waiters are then - * complete (we increase the seqno by increasingly larger values to - * retire more and more waiters at once). All retired waiters should - * be woken and removed from the rbtree, and so that we check. - */ - for (n = 0; n < count; n = m) { - int seqno = 2 * n; - - GEM_BUG_ON(find_first_bit(bitmap, count) != n); - - if (intel_wait_complete(&waiters[n])) { - pr_err("waiter[%d, seqno=%d] completed too early\n", - n, waiters[n].seqno); - err = -EINVAL; - goto out_bitmap; - } - - /* complete the following waiters */ - mock_seqno_advance(engine, seqno + seqno_bias); - for (m = n; m <= seqno; m++) { - if (m == count) - break; - - GEM_BUG_ON(!test_bit(m, bitmap)); - __clear_bit(m, bitmap); - } - - intel_engine_remove_wait(engine, &waiters[n]); - RB_CLEAR_NODE(&waiters[n].node); - - err = check_rbtree(engine, bitmap, waiters, count); - if (err) { - pr_err("rbtree corrupt after seqno advance to %d\n", - seqno + seqno_bias); - goto out_bitmap; - } - - err = check_completion(engine, bitmap, waiters, count); - if (err) { - pr_err("completions after seqno advance to %d failed\n", - seqno + seqno_bias); - goto out_bitmap; - } - } - - err = check_rbtree_empty(engine); -out_bitmap: - kfree(bitmap); -out_waiters: - kvfree(waiters); -out_engines: - mock_engine_flush(engine); - return err; -} - -struct igt_wakeup { - struct task_struct *tsk; - atomic_t *ready, *set, *done; - struct intel_engine_cs *engine; - unsigned long flags; -#define STOP 0 -#define IDLE 1 - wait_queue_head_t *wq; - u32 seqno; -}; - -static bool wait_for_ready(struct igt_wakeup *w) -{ - DEFINE_WAIT(ready); - - set_bit(IDLE, &w->flags); - if (atomic_dec_and_test(w->done)) - wake_up_var(w->done); - - if (test_bit(STOP, &w->flags)) - goto out; - - for (;;) { - prepare_to_wait(w->wq, &ready, TASK_INTERRUPTIBLE); - if (atomic_read(w->ready) == 0) - break; - - schedule(); - } - finish_wait(w->wq, &ready); - -out: - clear_bit(IDLE, &w->flags); - if (atomic_dec_and_test(w->set)) - wake_up_var(w->set); - - return !test_bit(STOP, &w->flags); -} - -static int igt_wakeup_thread(void *arg) -{ - struct igt_wakeup *w = arg; - struct intel_wait wait; - - while (wait_for_ready(w)) { - GEM_BUG_ON(kthread_should_stop()); - - intel_wait_init_for_seqno(&wait, w->seqno); - intel_engine_add_wait(w->engine, &wait); - for (;;) { - set_current_state(TASK_UNINTERRUPTIBLE); - if (i915_seqno_passed(intel_engine_get_seqno(w->engine), - w->seqno)) - break; - - if (test_bit(STOP, &w->flags)) /* emergency escape */ - break; - - schedule(); - } - intel_engine_remove_wait(w->engine, &wait); - __set_current_state(TASK_RUNNING); - } - - return 0; -} - -static void igt_wake_all_sync(atomic_t *ready, - atomic_t *set, - atomic_t *done, - wait_queue_head_t *wq, - int count) -{ - atomic_set(set, count); - atomic_set(ready, 0); - wake_up_all(wq); - - wait_var_event(set, !atomic_read(set)); - atomic_set(ready, count); - atomic_set(done, count); -} - -static int igt_wakeup(void *arg) -{ - I915_RND_STATE(prng); - struct intel_engine_cs *engine = arg; - struct igt_wakeup *waiters; - DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq); - const int count = 4096; - const u32 max_seqno = count / 4; - atomic_t ready, set, done; - int err = -ENOMEM; - int n, step; - - mock_engine_reset(engine); - - waiters = kvmalloc_array(count, sizeof(*waiters), GFP_KERNEL); - if (!waiters) - goto out_engines; - - /* Create a large number of threads, each waiting on a random seqno. - * Multiple waiters will be waiting for the same seqno. - */ - atomic_set(&ready, count); - for (n = 0; n < count; n++) { - waiters[n].wq = &wq; - waiters[n].ready = &ready; - waiters[n].set = &set; - waiters[n].done = &done; - waiters[n].engine = engine; - waiters[n].flags = BIT(IDLE); - - waiters[n].tsk = kthread_run(igt_wakeup_thread, &waiters[n], - "i915/igt:%d", n); - if (IS_ERR(waiters[n].tsk)) - goto out_waiters; - - get_task_struct(waiters[n].tsk); - } - - for (step = 1; step <= max_seqno; step <<= 1) { - u32 seqno; - - /* The waiter threads start paused as we assign them a random - * seqno and reset the engine. Once the engine is reset, - * we signal that the threads may begin their wait upon their - * seqno. - */ - for (n = 0; n < count; n++) { - GEM_BUG_ON(!test_bit(IDLE, &waiters[n].flags)); - waiters[n].seqno = - 1 + prandom_u32_state(&prng) % max_seqno; - } - mock_seqno_advance(engine, 0); - igt_wake_all_sync(&ready, &set, &done, &wq, count); - - /* Simulate the GPU doing chunks of work, with one or more - * seqno appearing to finish at the same time. A random number - * of threads will be waiting upon the update and hopefully be - * woken. - */ - for (seqno = 1; seqno <= max_seqno + step; seqno += step) { - usleep_range(50, 500); - mock_seqno_advance(engine, seqno); - } - GEM_BUG_ON(intel_engine_get_seqno(engine) < 1 + max_seqno); - - /* With the seqno now beyond any of the waiting threads, they - * should all be woken, see that they are complete and signal - * that they are ready for the next test. We wait until all - * threads are complete and waiting for us (i.e. not a seqno). - */ - if (!wait_var_event_timeout(&done, - !atomic_read(&done), 10 * HZ)) { - pr_err("Timed out waiting for %d remaining waiters\n", - atomic_read(&done)); - err = -ETIMEDOUT; - break; - } - - err = check_rbtree_empty(engine); - if (err) - break; - } - -out_waiters: - for (n = 0; n < count; n++) { - if (IS_ERR(waiters[n].tsk)) - break; - - set_bit(STOP, &waiters[n].flags); - } - mock_seqno_advance(engine, INT_MAX); /* wakeup any broken waiters */ - igt_wake_all_sync(&ready, &set, &done, &wq, n); - - for (n = 0; n < count; n++) { - if (IS_ERR(waiters[n].tsk)) - break; - - kthread_stop(waiters[n].tsk); - put_task_struct(waiters[n].tsk); - } - - kvfree(waiters); -out_engines: - mock_engine_flush(engine); - return err; -} - -int intel_breadcrumbs_mock_selftests(void) -{ - static const struct i915_subtest tests[] = { - SUBTEST(igt_random_insert_remove), - SUBTEST(igt_insert_complete), - SUBTEST(igt_wakeup), - }; - struct drm_i915_private *i915; - int err; - - i915 = mock_gem_device(); - if (!i915) - return -ENOMEM; - - err = i915_subtests(tests, i915->engine[RCS]); - drm_dev_put(&i915->drm); - - return err; -} diff --git a/drivers/gpu/drm/i915/selftests/intel_guc.c b/drivers/gpu/drm/i915/selftests/intel_guc.c index 0c0ab82b6228..c5e0a0e98fcb 100644 --- a/drivers/gpu/drm/i915/selftests/intel_guc.c +++ b/drivers/gpu/drm/i915/selftests/intel_guc.c @@ -137,12 +137,13 @@ static bool client_doorbell_in_sync(struct intel_guc_client *client) static int igt_guc_clients(void *args) { struct drm_i915_private *dev_priv = args; + intel_wakeref_t wakeref; struct intel_guc *guc; int err = 0; GEM_BUG_ON(!HAS_GUC(dev_priv)); mutex_lock(&dev_priv->drm.struct_mutex); - intel_runtime_pm_get(dev_priv); + wakeref = intel_runtime_pm_get(dev_priv); guc = &dev_priv->guc; if (!guc) { @@ -159,6 +160,7 @@ static int igt_guc_clients(void *args) * Get rid of clients created during driver load because the test will * recreate them. */ + guc_clients_disable(guc); guc_clients_destroy(guc); if (guc->execbuf_client || guc->preempt_client) { pr_err("guc_clients_destroy lied!\n"); @@ -197,8 +199,8 @@ static int igt_guc_clients(void *args) goto out; } - /* Now create the doorbells */ - guc_clients_doorbell_init(guc); + /* Now enable the clients */ + guc_clients_enable(guc); /* each client should now have received a doorbell */ if (!client_doorbell_in_sync(guc->execbuf_client) || @@ -212,65 +214,19 @@ static int igt_guc_clients(void *args) * Basic test - an attempt to reallocate a valid doorbell to the * client it is currently assigned should not cause a failure. */ - err = guc_clients_doorbell_init(guc); - if (err) - goto out; - - /* - * Negative test - a client with no doorbell (invalid db id). - * After destroying the doorbell, the db id is changed to - * GUC_DOORBELL_INVALID and the firmware will reject any attempt to - * allocate a doorbell with an invalid id (db has to be reserved before - * allocation). - */ - destroy_doorbell(guc->execbuf_client); - if (client_doorbell_in_sync(guc->execbuf_client)) { - pr_err("destroy db did not work\n"); - err = -EINVAL; - goto out; - } - - unreserve_doorbell(guc->execbuf_client); - - __create_doorbell(guc->execbuf_client); - err = __guc_allocate_doorbell(guc, guc->execbuf_client->stage_id); - if (err != -EIO) { - pr_err("unexpected (err = %d)", err); - goto out_db; - } - - if (!available_dbs(guc, guc->execbuf_client->priority)) { - pr_err("doorbell not available when it should\n"); - err = -EIO; - goto out_db; - } - -out_db: - /* clean after test */ - __destroy_doorbell(guc->execbuf_client); - err = reserve_doorbell(guc->execbuf_client); - if (err) { - pr_err("failed to reserve back the doorbell back\n"); - } err = create_doorbell(guc->execbuf_client); - if (err) { - pr_err("recreate doorbell failed\n"); - goto out; - } out: /* * Leave clean state for other test, plus the driver always destroy the * clients during unload. */ - destroy_doorbell(guc->execbuf_client); - if (guc->preempt_client) - destroy_doorbell(guc->preempt_client); + guc_clients_disable(guc); guc_clients_destroy(guc); guc_clients_create(guc); - guc_clients_doorbell_init(guc); + guc_clients_enable(guc); unlock: - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, wakeref); mutex_unlock(&dev_priv->drm.struct_mutex); return err; } @@ -283,13 +239,14 @@ unlock: static int igt_guc_doorbells(void *arg) { struct drm_i915_private *dev_priv = arg; + intel_wakeref_t wakeref; struct intel_guc *guc; int i, err = 0; u16 db_id; GEM_BUG_ON(!HAS_GUC(dev_priv)); mutex_lock(&dev_priv->drm.struct_mutex); - intel_runtime_pm_get(dev_priv); + wakeref = intel_runtime_pm_get(dev_priv); guc = &dev_priv->guc; if (!guc) { @@ -352,7 +309,7 @@ static int igt_guc_doorbells(void *arg) db_id = clients[i]->doorbell_id; - err = create_doorbell(clients[i]); + err = __guc_client_enable(clients[i]); if (err) { pr_err("[%d] Failed to create a doorbell\n", i); goto out; @@ -378,11 +335,11 @@ static int igt_guc_doorbells(void *arg) out: for (i = 0; i < ATTEMPTS; i++) if (!IS_ERR_OR_NULL(clients[i])) { - destroy_doorbell(clients[i]); + __guc_client_disable(clients[i]); guc_client_free(clients[i]); } unlock: - intel_runtime_pm_put(dev_priv); + intel_runtime_pm_put(dev_priv, wakeref); mutex_unlock(&dev_priv->drm.struct_mutex); return err; } diff --git a/drivers/gpu/drm/i915/selftests/intel_hangcheck.c b/drivers/gpu/drm/i915/selftests/intel_hangcheck.c index db378226ac10..7b6f3bea9ef8 100644 --- a/drivers/gpu/drm/i915/selftests/intel_hangcheck.c +++ b/drivers/gpu/drm/i915/selftests/intel_hangcheck.c @@ -27,6 +27,7 @@ #include "../i915_selftest.h" #include "i915_random.h" #include "igt_flush_test.h" +#include "igt_reset.h" #include "igt_wedge_me.h" #include "mock_context.h" @@ -76,7 +77,7 @@ static int hang_init(struct hang *h, struct drm_i915_private *i915) h->seqno = memset(vaddr, 0xff, PAGE_SIZE); vaddr = i915_gem_object_pin_map(h->obj, - HAS_LLC(i915) ? I915_MAP_WB : I915_MAP_WC); + i915_coherent_map_type(i915)); if (IS_ERR(vaddr)) { err = PTR_ERR(vaddr); goto err_unpin_hws; @@ -102,52 +103,87 @@ static u64 hws_address(const struct i915_vma *hws, return hws->node.start + offset_in_page(sizeof(u32)*rq->fence.context); } -static int emit_recurse_batch(struct hang *h, - struct i915_request *rq) +static int move_to_active(struct i915_vma *vma, + struct i915_request *rq, + unsigned int flags) +{ + int err; + + err = i915_vma_move_to_active(vma, rq, flags); + if (err) + return err; + + if (!i915_gem_object_has_active_reference(vma->obj)) { + i915_gem_object_get(vma->obj); + i915_gem_object_set_active_reference(vma->obj); + } + + return 0; +} + +static struct i915_request * +hang_create_request(struct hang *h, struct intel_engine_cs *engine) { struct drm_i915_private *i915 = h->i915; struct i915_address_space *vm = - rq->gem_context->ppgtt ? - &rq->gem_context->ppgtt->vm : - &i915->ggtt.vm; + h->ctx->ppgtt ? &h->ctx->ppgtt->vm : &i915->ggtt.vm; + struct i915_request *rq = NULL; struct i915_vma *hws, *vma; unsigned int flags; u32 *batch; int err; + if (i915_gem_object_is_active(h->obj)) { + struct drm_i915_gem_object *obj; + void *vaddr; + + obj = i915_gem_object_create_internal(h->i915, PAGE_SIZE); + if (IS_ERR(obj)) + return ERR_CAST(obj); + + vaddr = i915_gem_object_pin_map(obj, + i915_coherent_map_type(h->i915)); + if (IS_ERR(vaddr)) { + i915_gem_object_put(obj); + return ERR_CAST(vaddr); + } + + i915_gem_object_unpin_map(h->obj); + i915_gem_object_put(h->obj); + + h->obj = obj; + h->batch = vaddr; + } + vma = i915_vma_instance(h->obj, vm, NULL); if (IS_ERR(vma)) - return PTR_ERR(vma); + return ERR_CAST(vma); hws = i915_vma_instance(h->hws, vm, NULL); if (IS_ERR(hws)) - return PTR_ERR(hws); + return ERR_CAST(hws); err = i915_vma_pin(vma, 0, 0, PIN_USER); if (err) - return err; + return ERR_PTR(err); err = i915_vma_pin(hws, 0, 0, PIN_USER); if (err) goto unpin_vma; - err = i915_vma_move_to_active(vma, rq, 0); - if (err) + rq = i915_request_alloc(engine, h->ctx); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); goto unpin_hws; - - if (!i915_gem_object_has_active_reference(vma->obj)) { - i915_gem_object_get(vma->obj); - i915_gem_object_set_active_reference(vma->obj); } - err = i915_vma_move_to_active(hws, rq, 0); + err = move_to_active(vma, rq, 0); if (err) - goto unpin_hws; + goto cancel_rq; - if (!i915_gem_object_has_active_reference(hws->obj)) { - i915_gem_object_get(hws->obj); - i915_gem_object_set_active_reference(hws->obj); - } + err = move_to_active(hws, rq, 0); + if (err) + goto cancel_rq; batch = h->batch; if (INTEL_GEN(i915) >= 8) { @@ -212,52 +248,16 @@ static int emit_recurse_batch(struct hang *h, err = rq->engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, flags); +cancel_rq: + if (err) { + i915_request_skip(rq, err); + i915_request_add(rq); + } unpin_hws: i915_vma_unpin(hws); unpin_vma: i915_vma_unpin(vma); - return err; -} - -static struct i915_request * -hang_create_request(struct hang *h, struct intel_engine_cs *engine) -{ - struct i915_request *rq; - int err; - - if (i915_gem_object_is_active(h->obj)) { - struct drm_i915_gem_object *obj; - void *vaddr; - - obj = i915_gem_object_create_internal(h->i915, PAGE_SIZE); - if (IS_ERR(obj)) - return ERR_CAST(obj); - - vaddr = i915_gem_object_pin_map(obj, - HAS_LLC(h->i915) ? I915_MAP_WB : I915_MAP_WC); - if (IS_ERR(vaddr)) { - i915_gem_object_put(obj); - return ERR_CAST(vaddr); - } - - i915_gem_object_unpin_map(h->obj); - i915_gem_object_put(h->obj); - - h->obj = obj; - h->batch = vaddr; - } - - rq = i915_request_alloc(engine, h->ctx); - if (IS_ERR(rq)) - return rq; - - err = emit_recurse_batch(h, rq); - if (err) { - i915_request_add(rq); - return ERR_PTR(err); - } - - return rq; + return err ? ERR_PTR(err) : rq; } static u32 hws_seqno(const struct hang *h, const struct i915_request *rq) @@ -308,6 +308,7 @@ static int igt_hang_sanitycheck(void *arg) goto unlock; for_each_engine(engine, i915, id) { + struct igt_wedge_me w; long timeout; if (!intel_engine_can_store_dword(engine)) @@ -328,9 +329,14 @@ static int igt_hang_sanitycheck(void *arg) i915_request_add(rq); - timeout = i915_request_wait(rq, - I915_WAIT_LOCKED, - MAX_SCHEDULE_TIMEOUT); + timeout = 0; + igt_wedge_on_timeout(&w, i915, HZ / 10 /* 100ms timeout*/) + timeout = i915_request_wait(rq, + I915_WAIT_LOCKED, + MAX_SCHEDULE_TIMEOUT); + if (i915_terminally_wedged(&i915->gpu_error)) + timeout = -EIO; + i915_request_put(rq); if (timeout < 0) { @@ -348,40 +354,6 @@ unlock: return err; } -static void global_reset_lock(struct drm_i915_private *i915) -{ - struct intel_engine_cs *engine; - enum intel_engine_id id; - - pr_debug("%s: current gpu_error=%08lx\n", - __func__, i915->gpu_error.flags); - - while (test_and_set_bit(I915_RESET_BACKOFF, &i915->gpu_error.flags)) - wait_event(i915->gpu_error.reset_queue, - !test_bit(I915_RESET_BACKOFF, - &i915->gpu_error.flags)); - - for_each_engine(engine, i915, id) { - while (test_and_set_bit(I915_RESET_ENGINE + id, - &i915->gpu_error.flags)) - wait_on_bit(&i915->gpu_error.flags, - I915_RESET_ENGINE + id, - TASK_UNINTERRUPTIBLE); - } -} - -static void global_reset_unlock(struct drm_i915_private *i915) -{ - struct intel_engine_cs *engine; - enum intel_engine_id id; - - for_each_engine(engine, i915, id) - clear_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags); - - clear_bit(I915_RESET_BACKOFF, &i915->gpu_error.flags); - wake_up_all(&i915->gpu_error.reset_queue); -} - static int igt_global_reset(void *arg) { struct drm_i915_private *i915 = arg; @@ -390,10 +362,8 @@ static int igt_global_reset(void *arg) /* Check that we can issue a global GPU reset */ - global_reset_lock(i915); - set_bit(I915_RESET_HANDOFF, &i915->gpu_error.flags); + igt_global_reset_lock(i915); - mutex_lock(&i915->drm.struct_mutex); reset_count = i915_reset_count(&i915->gpu_error); i915_reset(i915, ALL_ENGINES, NULL); @@ -402,10 +372,8 @@ static int igt_global_reset(void *arg) pr_err("No GPU reset recorded!\n"); err = -EINVAL; } - mutex_unlock(&i915->drm.struct_mutex); - GEM_BUG_ON(test_bit(I915_RESET_HANDOFF, &i915->gpu_error.flags)); - global_reset_unlock(i915); + igt_global_reset_unlock(i915); if (i915_terminally_wedged(&i915->gpu_error)) err = -EIO; @@ -413,6 +381,29 @@ static int igt_global_reset(void *arg) return err; } +static int igt_wedged_reset(void *arg) +{ + struct drm_i915_private *i915 = arg; + intel_wakeref_t wakeref; + + /* Check that we can recover a wedged device with a GPU reset */ + + igt_global_reset_lock(i915); + wakeref = intel_runtime_pm_get(i915); + + i915_gem_set_wedged(i915); + + mutex_lock(&i915->drm.struct_mutex); + GEM_BUG_ON(!i915_terminally_wedged(&i915->gpu_error)); + i915_reset(i915, ALL_ENGINES, NULL); + mutex_unlock(&i915->drm.struct_mutex); + + intel_runtime_pm_put(i915, wakeref); + igt_global_reset_unlock(i915); + + return i915_terminally_wedged(&i915->gpu_error) ? -EIO : 0; +} + static bool wait_for_idle(struct intel_engine_cs *engine) { return wait_for(intel_engine_is_idle(engine), IGT_IDLE_TIMEOUT) == 0; @@ -458,8 +449,6 @@ static int __igt_reset_engine(struct drm_i915_private *i915, bool active) set_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags); do { - u32 seqno = intel_engine_get_seqno(engine); - if (active) { struct i915_request *rq; @@ -478,7 +467,7 @@ static int __igt_reset_engine(struct drm_i915_private *i915, bool active) if (!wait_until_running(&h, rq)) { struct drm_printer p = drm_info_printer(i915->drm.dev); - pr_err("%s: Failed to start request %x, at %x\n", + pr_err("%s: Failed to start request %llx, at %x\n", __func__, rq->fence.seqno, hws_seqno(&h, rq)); intel_engine_dump(engine, &p, "%s\n", engine->name); @@ -488,8 +477,6 @@ static int __igt_reset_engine(struct drm_i915_private *i915, bool active) break; } - GEM_BUG_ON(!rq->global_seqno); - seqno = rq->global_seqno - 1; i915_request_put(rq); } @@ -505,16 +492,15 @@ static int __igt_reset_engine(struct drm_i915_private *i915, bool active) break; } - reset_engine_count += active; if (i915_reset_engine_count(&i915->gpu_error, engine) != - reset_engine_count) { - pr_err("%s engine reset %srecorded!\n", - engine->name, active ? "not " : ""); + ++reset_engine_count) { + pr_err("%s engine reset not recorded!\n", + engine->name); err = -EINVAL; break; } - if (!wait_for_idle(engine)) { + if (!i915_reset_flush(i915)) { struct drm_printer p = drm_info_printer(i915->drm.dev); @@ -579,7 +565,7 @@ static int active_request_put(struct i915_request *rq) return 0; if (i915_request_wait(rq, 0, 5 * HZ) < 0) { - GEM_TRACE("%s timed out waiting for completion of fence %llx:%d, seqno %d.\n", + GEM_TRACE("%s timed out waiting for completion of fence %llx:%lld, seqno %d.\n", rq->engine->name, rq->fence.context, rq->fence.seqno, @@ -737,7 +723,6 @@ static int __igt_reset_engines(struct drm_i915_private *i915, set_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags); do { - u32 seqno = intel_engine_get_seqno(engine); struct i915_request *rq = NULL; if (flags & TEST_ACTIVE) { @@ -756,7 +741,7 @@ static int __igt_reset_engines(struct drm_i915_private *i915, if (!wait_until_running(&h, rq)) { struct drm_printer p = drm_info_printer(i915->drm.dev); - pr_err("%s: Failed to start request %x, at %x\n", + pr_err("%s: Failed to start request %llx, at %x\n", __func__, rq->fence.seqno, hws_seqno(&h, rq)); intel_engine_dump(engine, &p, "%s\n", engine->name); @@ -765,9 +750,6 @@ static int __igt_reset_engines(struct drm_i915_private *i915, err = -EIO; break; } - - GEM_BUG_ON(!rq->global_seqno); - seqno = rq->global_seqno - 1; } err = i915_reset_engine(engine, NULL); @@ -804,10 +786,9 @@ static int __igt_reset_engines(struct drm_i915_private *i915, reported = i915_reset_engine_count(&i915->gpu_error, engine); reported -= threads[engine->id].resets; - if (reported != (flags & TEST_ACTIVE ? count : 0)) { - pr_err("i915_reset_engine(%s:%s): reset %lu times, but reported %lu, expected %lu reported\n", - engine->name, test_name, count, reported, - (flags & TEST_ACTIVE ? count : 0)); + if (reported != count) { + pr_err("i915_reset_engine(%s:%s): reset %lu times, but reported %lu\n", + engine->name, test_name, count, reported); if (!err) err = -EINVAL; } @@ -906,20 +887,13 @@ static int igt_reset_engines(void *arg) return 0; } -static u32 fake_hangcheck(struct i915_request *rq, u32 mask) +static u32 fake_hangcheck(struct drm_i915_private *i915, u32 mask) { - struct i915_gpu_error *error = &rq->i915->gpu_error; - u32 reset_count = i915_reset_count(error); + u32 count = i915_reset_count(&i915->gpu_error); - error->stalled_mask = mask; + i915_reset(i915, mask, NULL); - /* set_bit() must be after we have setup the backchannel (mask) */ - smp_mb__before_atomic(); - set_bit(I915_RESET_HANDOFF, &error->flags); - - wake_up_all(&error->wait_queue); - - return reset_count; + return count; } static int igt_reset_wait(void *arg) @@ -936,7 +910,7 @@ static int igt_reset_wait(void *arg) /* Check that we detect a stuck waiter and issue a reset */ - global_reset_lock(i915); + igt_global_reset_lock(i915); mutex_lock(&i915->drm.struct_mutex); err = hang_init(&h, i915); @@ -955,7 +929,7 @@ static int igt_reset_wait(void *arg) if (!wait_until_running(&h, rq)) { struct drm_printer p = drm_info_printer(i915->drm.dev); - pr_err("%s: Failed to start request %x, at %x\n", + pr_err("%s: Failed to start request %llx, at %x\n", __func__, rq->fence.seqno, hws_seqno(&h, rq)); intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name); @@ -965,7 +939,7 @@ static int igt_reset_wait(void *arg) goto out_rq; } - reset_count = fake_hangcheck(rq, ALL_ENGINES); + reset_count = fake_hangcheck(i915, ALL_ENGINES); timeout = i915_request_wait(rq, I915_WAIT_LOCKED, 10); if (timeout < 0) { @@ -975,7 +949,6 @@ static int igt_reset_wait(void *arg) goto out_rq; } - GEM_BUG_ON(test_bit(I915_RESET_HANDOFF, &i915->gpu_error.flags)); if (i915_reset_count(&i915->gpu_error) == reset_count) { pr_err("No GPU reset recorded!\n"); err = -EINVAL; @@ -988,7 +961,7 @@ fini: hang_fini(&h); unlock: mutex_unlock(&i915->drm.struct_mutex); - global_reset_unlock(i915); + igt_global_reset_unlock(i915); if (i915_terminally_wedged(&i915->gpu_error)) return -EIO; @@ -1066,7 +1039,7 @@ static int __igt_reset_evict_vma(struct drm_i915_private *i915, /* Check that we can recover an unbind stuck on a hanging request */ - global_reset_lock(i915); + igt_global_reset_lock(i915); mutex_lock(&i915->drm.struct_mutex); err = hang_init(&h, i915); @@ -1134,7 +1107,7 @@ static int __igt_reset_evict_vma(struct drm_i915_private *i915, if (!wait_until_running(&h, rq)) { struct drm_printer p = drm_info_printer(i915->drm.dev); - pr_err("%s: Failed to start request %x, at %x\n", + pr_err("%s: Failed to start request %llx, at %x\n", __func__, rq->fence.seqno, hws_seqno(&h, rq)); intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name); @@ -1150,10 +1123,11 @@ static int __igt_reset_evict_vma(struct drm_i915_private *i915, tsk = NULL; goto out_reset; } + get_task_struct(tsk); wait_for_completion(&arg.completion); - if (wait_for(waitqueue_active(&rq->execute), 10)) { + if (wait_for(!list_empty(&rq->fence.cb_list), 10)) { struct drm_printer p = drm_info_printer(i915->drm.dev); pr_err("igt/evict_vma kthread did not wait\n"); @@ -1164,7 +1138,7 @@ static int __igt_reset_evict_vma(struct drm_i915_private *i915, } out_reset: - fake_hangcheck(rq, intel_engine_flag(rq->engine)); + fake_hangcheck(rq->i915, intel_engine_flag(rq->engine)); if (tsk) { struct igt_wedge_me w; @@ -1172,6 +1146,8 @@ out_reset: /* The reset, even indirectly, should take less than 10ms. */ igt_wedge_on_timeout(&w, i915, HZ / 10 /* 100ms timeout*/) err = kthread_stop(tsk); + + put_task_struct(tsk); } mutex_lock(&i915->drm.struct_mutex); @@ -1183,7 +1159,7 @@ fini: hang_fini(&h); unlock: mutex_unlock(&i915->drm.struct_mutex); - global_reset_unlock(i915); + igt_global_reset_unlock(i915); if (i915_terminally_wedged(&i915->gpu_error)) return -EIO; @@ -1263,7 +1239,7 @@ static int igt_reset_queue(void *arg) /* Check that we replay pending requests following a hang */ - global_reset_lock(i915); + igt_global_reset_lock(i915); mutex_lock(&i915->drm.struct_mutex); err = hang_init(&h, i915); @@ -1326,7 +1302,7 @@ static int igt_reset_queue(void *arg) if (!wait_until_running(&h, prev)) { struct drm_printer p = drm_info_printer(i915->drm.dev); - pr_err("%s(%s): Failed to start request %x, at %x\n", + pr_err("%s(%s): Failed to start request %llx, at %x\n", __func__, engine->name, prev->fence.seqno, hws_seqno(&h, prev)); intel_engine_dump(engine, &p, @@ -1341,12 +1317,7 @@ static int igt_reset_queue(void *arg) goto fini; } - reset_count = fake_hangcheck(prev, ENGINE_MASK(id)); - - i915_reset(i915, ENGINE_MASK(id), NULL); - - GEM_BUG_ON(test_bit(I915_RESET_HANDOFF, - &i915->gpu_error.flags)); + reset_count = fake_hangcheck(i915, ENGINE_MASK(id)); if (prev->fence.error != -EIO) { pr_err("GPU reset not recorded on hanging request [fence.error=%d]!\n", @@ -1394,7 +1365,7 @@ fini: hang_fini(&h); unlock: mutex_unlock(&i915->drm.struct_mutex); - global_reset_unlock(i915); + igt_global_reset_unlock(i915); if (i915_terminally_wedged(&i915->gpu_error)) return -EIO; @@ -1437,7 +1408,7 @@ static int igt_handle_error(void *arg) if (!wait_until_running(&h, rq)) { struct drm_printer p = drm_info_printer(i915->drm.dev); - pr_err("%s: Failed to start request %x, at %x\n", + pr_err("%s: Failed to start request %llx, at %x\n", __func__, rq->fence.seqno, hws_seqno(&h, rq)); intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name); @@ -1473,10 +1444,203 @@ err_unlock: return err; } +static void __preempt_begin(void) +{ + preempt_disable(); +} + +static void __preempt_end(void) +{ + preempt_enable(); +} + +static void __softirq_begin(void) +{ + local_bh_disable(); +} + +static void __softirq_end(void) +{ + local_bh_enable(); +} + +static void __hardirq_begin(void) +{ + local_irq_disable(); +} + +static void __hardirq_end(void) +{ + local_irq_enable(); +} + +struct atomic_section { + const char *name; + void (*critical_section_begin)(void); + void (*critical_section_end)(void); +}; + +static int __igt_atomic_reset_engine(struct intel_engine_cs *engine, + const struct atomic_section *p, + const char *mode) +{ + struct tasklet_struct * const t = &engine->execlists.tasklet; + int err; + + GEM_TRACE("i915_reset_engine(%s:%s) under %s\n", + engine->name, mode, p->name); + + tasklet_disable_nosync(t); + p->critical_section_begin(); + + err = i915_reset_engine(engine, NULL); + + p->critical_section_end(); + tasklet_enable(t); + + if (err) + pr_err("i915_reset_engine(%s:%s) failed under %s\n", + engine->name, mode, p->name); + + return err; +} + +static int igt_atomic_reset_engine(struct intel_engine_cs *engine, + const struct atomic_section *p) +{ + struct drm_i915_private *i915 = engine->i915; + struct i915_request *rq; + struct hang h; + int err; + + err = __igt_atomic_reset_engine(engine, p, "idle"); + if (err) + return err; + + err = hang_init(&h, i915); + if (err) + return err; + + rq = hang_create_request(&h, engine); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto out; + } + + i915_request_get(rq); + i915_request_add(rq); + + if (wait_until_running(&h, rq)) { + err = __igt_atomic_reset_engine(engine, p, "active"); + } else { + pr_err("%s(%s): Failed to start request %llx, at %x\n", + __func__, engine->name, + rq->fence.seqno, hws_seqno(&h, rq)); + i915_gem_set_wedged(i915); + err = -EIO; + } + + if (err == 0) { + struct igt_wedge_me w; + + igt_wedge_on_timeout(&w, i915, HZ / 20 /* 50ms timeout*/) + i915_request_wait(rq, + I915_WAIT_LOCKED, + MAX_SCHEDULE_TIMEOUT); + if (i915_terminally_wedged(&i915->gpu_error)) + err = -EIO; + } + + i915_request_put(rq); +out: + hang_fini(&h); + return err; +} + +static void force_reset(struct drm_i915_private *i915) +{ + i915_gem_set_wedged(i915); + i915_reset(i915, 0, NULL); +} + +static int igt_atomic_reset(void *arg) +{ + static const struct atomic_section phases[] = { + { "preempt", __preempt_begin, __preempt_end }, + { "softirq", __softirq_begin, __softirq_end }, + { "hardirq", __hardirq_begin, __hardirq_end }, + { } + }; + struct drm_i915_private *i915 = arg; + intel_wakeref_t wakeref; + int err = 0; + + /* Check that the resets are usable from atomic context */ + + if (USES_GUC_SUBMISSION(i915)) + return 0; /* guc is dead; long live the guc */ + + igt_global_reset_lock(i915); + mutex_lock(&i915->drm.struct_mutex); + wakeref = intel_runtime_pm_get(i915); + + /* Flush any requests before we get started and check basics */ + force_reset(i915); + if (i915_terminally_wedged(&i915->gpu_error)) + goto unlock; + + if (intel_has_gpu_reset(i915)) { + const typeof(*phases) *p; + + for (p = phases; p->name; p++) { + GEM_TRACE("intel_gpu_reset under %s\n", p->name); + + p->critical_section_begin(); + err = intel_gpu_reset(i915, ALL_ENGINES); + p->critical_section_end(); + + if (err) { + pr_err("intel_gpu_reset failed under %s\n", + p->name); + goto out; + } + } + + force_reset(i915); + } + + if (intel_has_reset_engine(i915)) { + struct intel_engine_cs *engine; + enum intel_engine_id id; + + for_each_engine(engine, i915, id) { + const typeof(*phases) *p; + + for (p = phases; p->name; p++) { + err = igt_atomic_reset_engine(engine, p); + if (err) + goto out; + } + } + } + +out: + /* As we poke around the guts, do a full reset before continuing. */ + force_reset(i915); + +unlock: + intel_runtime_pm_put(i915, wakeref); + mutex_unlock(&i915->drm.struct_mutex); + igt_global_reset_unlock(i915); + + return err; +} + int intel_hangcheck_live_selftests(struct drm_i915_private *i915) { static const struct i915_subtest tests[] = { SUBTEST(igt_global_reset), /* attempt to recover GPU first */ + SUBTEST(igt_wedged_reset), SUBTEST(igt_hang_sanitycheck), SUBTEST(igt_reset_idle_engine), SUBTEST(igt_reset_active_engine), @@ -1487,7 +1651,9 @@ int intel_hangcheck_live_selftests(struct drm_i915_private *i915) SUBTEST(igt_reset_evict_ppgtt), SUBTEST(igt_reset_evict_fence), SUBTEST(igt_handle_error), + SUBTEST(igt_atomic_reset), }; + intel_wakeref_t wakeref; bool saved_hangcheck; int err; @@ -1497,8 +1663,9 @@ int intel_hangcheck_live_selftests(struct drm_i915_private *i915) if (i915_terminally_wedged(&i915->gpu_error)) return -EIO; /* we're long past hope of a successful reset */ - intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(i915); saved_hangcheck = fetch_and_zero(&i915_modparams.enable_hangcheck); + drain_delayed_work(&i915->gpu_error.hangcheck_work); /* flush param */ err = i915_subtests(tests, i915); @@ -1507,7 +1674,7 @@ int intel_hangcheck_live_selftests(struct drm_i915_private *i915) mutex_unlock(&i915->drm.struct_mutex); i915_modparams.enable_hangcheck = saved_hangcheck; - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); return err; } diff --git a/drivers/gpu/drm/i915/selftests/intel_lrc.c b/drivers/gpu/drm/i915/selftests/intel_lrc.c index 1aea7a8f2224..58144e024751 100644 --- a/drivers/gpu/drm/i915/selftests/intel_lrc.c +++ b/drivers/gpu/drm/i915/selftests/intel_lrc.c @@ -4,226 +4,34 @@ * Copyright © 2018 Intel Corporation */ +#include <linux/prime_numbers.h> + +#include "../i915_reset.h" + #include "../i915_selftest.h" #include "igt_flush_test.h" +#include "igt_spinner.h" +#include "i915_random.h" #include "mock_context.h" -struct spinner { - struct drm_i915_private *i915; - struct drm_i915_gem_object *hws; - struct drm_i915_gem_object *obj; - u32 *batch; - void *seqno; -}; - -static int spinner_init(struct spinner *spin, struct drm_i915_private *i915) -{ - unsigned int mode; - void *vaddr; - int err; - - GEM_BUG_ON(INTEL_GEN(i915) < 8); - - memset(spin, 0, sizeof(*spin)); - spin->i915 = i915; - - spin->hws = i915_gem_object_create_internal(i915, PAGE_SIZE); - if (IS_ERR(spin->hws)) { - err = PTR_ERR(spin->hws); - goto err; - } - - spin->obj = i915_gem_object_create_internal(i915, PAGE_SIZE); - if (IS_ERR(spin->obj)) { - err = PTR_ERR(spin->obj); - goto err_hws; - } - - i915_gem_object_set_cache_level(spin->hws, I915_CACHE_LLC); - vaddr = i915_gem_object_pin_map(spin->hws, I915_MAP_WB); - if (IS_ERR(vaddr)) { - err = PTR_ERR(vaddr); - goto err_obj; - } - spin->seqno = memset(vaddr, 0xff, PAGE_SIZE); - - mode = HAS_LLC(i915) ? I915_MAP_WB : I915_MAP_WC; - vaddr = i915_gem_object_pin_map(spin->obj, mode); - if (IS_ERR(vaddr)) { - err = PTR_ERR(vaddr); - goto err_unpin_hws; - } - spin->batch = vaddr; - - return 0; - -err_unpin_hws: - i915_gem_object_unpin_map(spin->hws); -err_obj: - i915_gem_object_put(spin->obj); -err_hws: - i915_gem_object_put(spin->hws); -err: - return err; -} - -static unsigned int seqno_offset(u64 fence) -{ - return offset_in_page(sizeof(u32) * fence); -} - -static u64 hws_address(const struct i915_vma *hws, - const struct i915_request *rq) -{ - return hws->node.start + seqno_offset(rq->fence.context); -} - -static int emit_recurse_batch(struct spinner *spin, - struct i915_request *rq, - u32 arbitration_command) -{ - struct i915_address_space *vm = &rq->gem_context->ppgtt->vm; - struct i915_vma *hws, *vma; - u32 *batch; - int err; - - vma = i915_vma_instance(spin->obj, vm, NULL); - if (IS_ERR(vma)) - return PTR_ERR(vma); - - hws = i915_vma_instance(spin->hws, vm, NULL); - if (IS_ERR(hws)) - return PTR_ERR(hws); - - err = i915_vma_pin(vma, 0, 0, PIN_USER); - if (err) - return err; - - err = i915_vma_pin(hws, 0, 0, PIN_USER); - if (err) - goto unpin_vma; - - err = i915_vma_move_to_active(vma, rq, 0); - if (err) - goto unpin_hws; - - if (!i915_gem_object_has_active_reference(vma->obj)) { - i915_gem_object_get(vma->obj); - i915_gem_object_set_active_reference(vma->obj); - } - - err = i915_vma_move_to_active(hws, rq, 0); - if (err) - goto unpin_hws; - - if (!i915_gem_object_has_active_reference(hws->obj)) { - i915_gem_object_get(hws->obj); - i915_gem_object_set_active_reference(hws->obj); - } - - batch = spin->batch; - - *batch++ = MI_STORE_DWORD_IMM_GEN4; - *batch++ = lower_32_bits(hws_address(hws, rq)); - *batch++ = upper_32_bits(hws_address(hws, rq)); - *batch++ = rq->fence.seqno; - - *batch++ = arbitration_command; - - *batch++ = MI_BATCH_BUFFER_START | 1 << 8 | 1; - *batch++ = lower_32_bits(vma->node.start); - *batch++ = upper_32_bits(vma->node.start); - *batch++ = MI_BATCH_BUFFER_END; /* not reached */ - - i915_gem_chipset_flush(spin->i915); - - err = rq->engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, 0); - -unpin_hws: - i915_vma_unpin(hws); -unpin_vma: - i915_vma_unpin(vma); - return err; -} - -static struct i915_request * -spinner_create_request(struct spinner *spin, - struct i915_gem_context *ctx, - struct intel_engine_cs *engine, - u32 arbitration_command) -{ - struct i915_request *rq; - int err; - - rq = i915_request_alloc(engine, ctx); - if (IS_ERR(rq)) - return rq; - - err = emit_recurse_batch(spin, rq, arbitration_command); - if (err) { - i915_request_add(rq); - return ERR_PTR(err); - } - - return rq; -} - -static u32 hws_seqno(const struct spinner *spin, const struct i915_request *rq) -{ - u32 *seqno = spin->seqno + seqno_offset(rq->fence.context); - - return READ_ONCE(*seqno); -} - -static void spinner_end(struct spinner *spin) -{ - *spin->batch = MI_BATCH_BUFFER_END; - i915_gem_chipset_flush(spin->i915); -} - -static void spinner_fini(struct spinner *spin) -{ - spinner_end(spin); - - i915_gem_object_unpin_map(spin->obj); - i915_gem_object_put(spin->obj); - - i915_gem_object_unpin_map(spin->hws); - i915_gem_object_put(spin->hws); -} - -static bool wait_for_spinner(struct spinner *spin, struct i915_request *rq) -{ - if (!wait_event_timeout(rq->execute, - READ_ONCE(rq->global_seqno), - msecs_to_jiffies(10))) - return false; - - return !(wait_for_us(i915_seqno_passed(hws_seqno(spin, rq), - rq->fence.seqno), - 10) && - wait_for(i915_seqno_passed(hws_seqno(spin, rq), - rq->fence.seqno), - 1000)); -} - static int live_sanitycheck(void *arg) { struct drm_i915_private *i915 = arg; struct intel_engine_cs *engine; struct i915_gem_context *ctx; enum intel_engine_id id; - struct spinner spin; + struct igt_spinner spin; + intel_wakeref_t wakeref; int err = -ENOMEM; if (!HAS_LOGICAL_RING_CONTEXTS(i915)) return 0; mutex_lock(&i915->drm.struct_mutex); - intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(i915); - if (spinner_init(&spin, i915)) + if (igt_spinner_init(&spin, i915)) goto err_unlock; ctx = kernel_context(i915); @@ -233,14 +41,14 @@ static int live_sanitycheck(void *arg) for_each_engine(engine, i915, id) { struct i915_request *rq; - rq = spinner_create_request(&spin, ctx, engine, MI_NOOP); + rq = igt_spinner_create_request(&spin, ctx, engine, MI_NOOP); if (IS_ERR(rq)) { err = PTR_ERR(rq); goto err_ctx; } i915_request_add(rq); - if (!wait_for_spinner(&spin, rq)) { + if (!igt_wait_for_spinner(&spin, rq)) { GEM_TRACE("spinner failed to start\n"); GEM_TRACE_DUMP(); i915_gem_set_wedged(i915); @@ -248,7 +56,7 @@ static int live_sanitycheck(void *arg) goto err_ctx; } - spinner_end(&spin); + igt_spinner_end(&spin); if (igt_flush_test(i915, I915_WAIT_LOCKED)) { err = -EIO; goto err_ctx; @@ -259,10 +67,10 @@ static int live_sanitycheck(void *arg) err_ctx: kernel_context_close(ctx); err_spin: - spinner_fini(&spin); + igt_spinner_fini(&spin); err_unlock: igt_flush_test(i915, I915_WAIT_LOCKED); - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); mutex_unlock(&i915->drm.struct_mutex); return err; } @@ -271,45 +79,48 @@ static int live_preempt(void *arg) { struct drm_i915_private *i915 = arg; struct i915_gem_context *ctx_hi, *ctx_lo; - struct spinner spin_hi, spin_lo; + struct igt_spinner spin_hi, spin_lo; struct intel_engine_cs *engine; enum intel_engine_id id; + intel_wakeref_t wakeref; int err = -ENOMEM; if (!HAS_LOGICAL_RING_PREEMPTION(i915)) return 0; mutex_lock(&i915->drm.struct_mutex); - intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(i915); - if (spinner_init(&spin_hi, i915)) + if (igt_spinner_init(&spin_hi, i915)) goto err_unlock; - if (spinner_init(&spin_lo, i915)) + if (igt_spinner_init(&spin_lo, i915)) goto err_spin_hi; ctx_hi = kernel_context(i915); if (!ctx_hi) goto err_spin_lo; - ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY; + ctx_hi->sched.priority = + I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY); ctx_lo = kernel_context(i915); if (!ctx_lo) goto err_ctx_hi; - ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY; + ctx_lo->sched.priority = + I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY); for_each_engine(engine, i915, id) { struct i915_request *rq; - rq = spinner_create_request(&spin_lo, ctx_lo, engine, - MI_ARB_CHECK); + rq = igt_spinner_create_request(&spin_lo, ctx_lo, engine, + MI_ARB_CHECK); if (IS_ERR(rq)) { err = PTR_ERR(rq); goto err_ctx_lo; } i915_request_add(rq); - if (!wait_for_spinner(&spin_lo, rq)) { + if (!igt_wait_for_spinner(&spin_lo, rq)) { GEM_TRACE("lo spinner failed to start\n"); GEM_TRACE_DUMP(); i915_gem_set_wedged(i915); @@ -317,16 +128,16 @@ static int live_preempt(void *arg) goto err_ctx_lo; } - rq = spinner_create_request(&spin_hi, ctx_hi, engine, - MI_ARB_CHECK); + rq = igt_spinner_create_request(&spin_hi, ctx_hi, engine, + MI_ARB_CHECK); if (IS_ERR(rq)) { - spinner_end(&spin_lo); + igt_spinner_end(&spin_lo); err = PTR_ERR(rq); goto err_ctx_lo; } i915_request_add(rq); - if (!wait_for_spinner(&spin_hi, rq)) { + if (!igt_wait_for_spinner(&spin_hi, rq)) { GEM_TRACE("hi spinner failed to start\n"); GEM_TRACE_DUMP(); i915_gem_set_wedged(i915); @@ -334,8 +145,8 @@ static int live_preempt(void *arg) goto err_ctx_lo; } - spinner_end(&spin_hi); - spinner_end(&spin_lo); + igt_spinner_end(&spin_hi); + igt_spinner_end(&spin_lo); if (igt_flush_test(i915, I915_WAIT_LOCKED)) { err = -EIO; goto err_ctx_lo; @@ -348,12 +159,12 @@ err_ctx_lo: err_ctx_hi: kernel_context_close(ctx_hi); err_spin_lo: - spinner_fini(&spin_lo); + igt_spinner_fini(&spin_lo); err_spin_hi: - spinner_fini(&spin_hi); + igt_spinner_fini(&spin_hi); err_unlock: igt_flush_test(i915, I915_WAIT_LOCKED); - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); mutex_unlock(&i915->drm.struct_mutex); return err; } @@ -362,22 +173,23 @@ static int live_late_preempt(void *arg) { struct drm_i915_private *i915 = arg; struct i915_gem_context *ctx_hi, *ctx_lo; - struct spinner spin_hi, spin_lo; + struct igt_spinner spin_hi, spin_lo; struct intel_engine_cs *engine; struct i915_sched_attr attr = {}; enum intel_engine_id id; + intel_wakeref_t wakeref; int err = -ENOMEM; if (!HAS_LOGICAL_RING_PREEMPTION(i915)) return 0; mutex_lock(&i915->drm.struct_mutex); - intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(i915); - if (spinner_init(&spin_hi, i915)) + if (igt_spinner_init(&spin_hi, i915)) goto err_unlock; - if (spinner_init(&spin_lo, i915)) + if (igt_spinner_init(&spin_lo, i915)) goto err_spin_hi; ctx_hi = kernel_context(i915); @@ -391,43 +203,44 @@ static int live_late_preempt(void *arg) for_each_engine(engine, i915, id) { struct i915_request *rq; - rq = spinner_create_request(&spin_lo, ctx_lo, engine, - MI_ARB_CHECK); + rq = igt_spinner_create_request(&spin_lo, ctx_lo, engine, + MI_ARB_CHECK); if (IS_ERR(rq)) { err = PTR_ERR(rq); goto err_ctx_lo; } i915_request_add(rq); - if (!wait_for_spinner(&spin_lo, rq)) { + if (!igt_wait_for_spinner(&spin_lo, rq)) { pr_err("First context failed to start\n"); goto err_wedged; } - rq = spinner_create_request(&spin_hi, ctx_hi, engine, MI_NOOP); + rq = igt_spinner_create_request(&spin_hi, ctx_hi, engine, + MI_NOOP); if (IS_ERR(rq)) { - spinner_end(&spin_lo); + igt_spinner_end(&spin_lo); err = PTR_ERR(rq); goto err_ctx_lo; } i915_request_add(rq); - if (wait_for_spinner(&spin_hi, rq)) { + if (igt_wait_for_spinner(&spin_hi, rq)) { pr_err("Second context overtook first?\n"); goto err_wedged; } - attr.priority = I915_PRIORITY_MAX; + attr.priority = I915_USER_PRIORITY(I915_PRIORITY_MAX); engine->schedule(rq, &attr); - if (!wait_for_spinner(&spin_hi, rq)) { + if (!igt_wait_for_spinner(&spin_hi, rq)) { pr_err("High priority context failed to preempt the low priority context\n"); GEM_TRACE_DUMP(); goto err_wedged; } - spinner_end(&spin_hi); - spinner_end(&spin_lo); + igt_spinner_end(&spin_hi); + igt_spinner_end(&spin_lo); if (igt_flush_test(i915, I915_WAIT_LOCKED)) { err = -EIO; goto err_ctx_lo; @@ -440,30 +253,268 @@ err_ctx_lo: err_ctx_hi: kernel_context_close(ctx_hi); err_spin_lo: - spinner_fini(&spin_lo); + igt_spinner_fini(&spin_lo); err_spin_hi: - spinner_fini(&spin_hi); + igt_spinner_fini(&spin_hi); err_unlock: igt_flush_test(i915, I915_WAIT_LOCKED); - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); mutex_unlock(&i915->drm.struct_mutex); return err; err_wedged: - spinner_end(&spin_hi); - spinner_end(&spin_lo); + igt_spinner_end(&spin_hi); + igt_spinner_end(&spin_lo); i915_gem_set_wedged(i915); err = -EIO; goto err_ctx_lo; } +struct preempt_client { + struct igt_spinner spin; + struct i915_gem_context *ctx; +}; + +static int preempt_client_init(struct drm_i915_private *i915, + struct preempt_client *c) +{ + c->ctx = kernel_context(i915); + if (!c->ctx) + return -ENOMEM; + + if (igt_spinner_init(&c->spin, i915)) + goto err_ctx; + + return 0; + +err_ctx: + kernel_context_close(c->ctx); + return -ENOMEM; +} + +static void preempt_client_fini(struct preempt_client *c) +{ + igt_spinner_fini(&c->spin); + kernel_context_close(c->ctx); +} + +static int live_suppress_self_preempt(void *arg) +{ + struct drm_i915_private *i915 = arg; + struct intel_engine_cs *engine; + struct i915_sched_attr attr = { + .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX) + }; + struct preempt_client a, b; + enum intel_engine_id id; + intel_wakeref_t wakeref; + int err = -ENOMEM; + + /* + * Verify that if a preemption request does not cause a change in + * the current execution order, the preempt-to-idle injection is + * skipped and that we do not accidentally apply it after the CS + * completion event. + */ + + if (!HAS_LOGICAL_RING_PREEMPTION(i915)) + return 0; + + if (USES_GUC_SUBMISSION(i915)) + return 0; /* presume black blox */ + + mutex_lock(&i915->drm.struct_mutex); + wakeref = intel_runtime_pm_get(i915); + + if (preempt_client_init(i915, &a)) + goto err_unlock; + if (preempt_client_init(i915, &b)) + goto err_client_a; + + for_each_engine(engine, i915, id) { + struct i915_request *rq_a, *rq_b; + int depth; + + engine->execlists.preempt_hang.count = 0; + + rq_a = igt_spinner_create_request(&a.spin, + a.ctx, engine, + MI_NOOP); + if (IS_ERR(rq_a)) { + err = PTR_ERR(rq_a); + goto err_client_b; + } + + i915_request_add(rq_a); + if (!igt_wait_for_spinner(&a.spin, rq_a)) { + pr_err("First client failed to start\n"); + goto err_wedged; + } + + for (depth = 0; depth < 8; depth++) { + rq_b = igt_spinner_create_request(&b.spin, + b.ctx, engine, + MI_NOOP); + if (IS_ERR(rq_b)) { + err = PTR_ERR(rq_b); + goto err_client_b; + } + i915_request_add(rq_b); + + GEM_BUG_ON(i915_request_completed(rq_a)); + engine->schedule(rq_a, &attr); + igt_spinner_end(&a.spin); + + if (!igt_wait_for_spinner(&b.spin, rq_b)) { + pr_err("Second client failed to start\n"); + goto err_wedged; + } + + swap(a, b); + rq_a = rq_b; + } + igt_spinner_end(&a.spin); + + if (engine->execlists.preempt_hang.count) { + pr_err("Preemption recorded x%d, depth %d; should have been suppressed!\n", + engine->execlists.preempt_hang.count, + depth); + err = -EINVAL; + goto err_client_b; + } + + if (igt_flush_test(i915, I915_WAIT_LOCKED)) + goto err_wedged; + } + + err = 0; +err_client_b: + preempt_client_fini(&b); +err_client_a: + preempt_client_fini(&a); +err_unlock: + if (igt_flush_test(i915, I915_WAIT_LOCKED)) + err = -EIO; + intel_runtime_pm_put(i915, wakeref); + mutex_unlock(&i915->drm.struct_mutex); + return err; + +err_wedged: + igt_spinner_end(&b.spin); + igt_spinner_end(&a.spin); + i915_gem_set_wedged(i915); + err = -EIO; + goto err_client_b; +} + +static int live_chain_preempt(void *arg) +{ + struct drm_i915_private *i915 = arg; + struct intel_engine_cs *engine; + struct preempt_client hi, lo; + enum intel_engine_id id; + intel_wakeref_t wakeref; + int err = -ENOMEM; + + /* + * Build a chain AB...BA between two contexts (A, B) and request + * preemption of the last request. It should then complete before + * the previously submitted spinner in B. + */ + + if (!HAS_LOGICAL_RING_PREEMPTION(i915)) + return 0; + + mutex_lock(&i915->drm.struct_mutex); + wakeref = intel_runtime_pm_get(i915); + + if (preempt_client_init(i915, &hi)) + goto err_unlock; + + if (preempt_client_init(i915, &lo)) + goto err_client_hi; + + for_each_engine(engine, i915, id) { + struct i915_sched_attr attr = { + .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX), + }; + int count, i; + + for_each_prime_number_from(count, 1, 32) { /* must fit ring! */ + struct i915_request *rq; + + rq = igt_spinner_create_request(&hi.spin, + hi.ctx, engine, + MI_ARB_CHECK); + if (IS_ERR(rq)) + goto err_wedged; + i915_request_add(rq); + if (!igt_wait_for_spinner(&hi.spin, rq)) + goto err_wedged; + + rq = igt_spinner_create_request(&lo.spin, + lo.ctx, engine, + MI_ARB_CHECK); + if (IS_ERR(rq)) + goto err_wedged; + i915_request_add(rq); + + for (i = 0; i < count; i++) { + rq = i915_request_alloc(engine, lo.ctx); + if (IS_ERR(rq)) + goto err_wedged; + i915_request_add(rq); + } + + rq = i915_request_alloc(engine, hi.ctx); + if (IS_ERR(rq)) + goto err_wedged; + i915_request_add(rq); + engine->schedule(rq, &attr); + + igt_spinner_end(&hi.spin); + if (i915_request_wait(rq, I915_WAIT_LOCKED, HZ / 5) < 0) { + struct drm_printer p = + drm_info_printer(i915->drm.dev); + + pr_err("Failed to preempt over chain of %d\n", + count); + intel_engine_dump(engine, &p, + "%s\n", engine->name); + goto err_wedged; + } + igt_spinner_end(&lo.spin); + } + } + + err = 0; +err_client_lo: + preempt_client_fini(&lo); +err_client_hi: + preempt_client_fini(&hi); +err_unlock: + if (igt_flush_test(i915, I915_WAIT_LOCKED)) + err = -EIO; + intel_runtime_pm_put(i915, wakeref); + mutex_unlock(&i915->drm.struct_mutex); + return err; + +err_wedged: + igt_spinner_end(&hi.spin); + igt_spinner_end(&lo.spin); + i915_gem_set_wedged(i915); + err = -EIO; + goto err_client_lo; +} + static int live_preempt_hang(void *arg) { struct drm_i915_private *i915 = arg; struct i915_gem_context *ctx_hi, *ctx_lo; - struct spinner spin_hi, spin_lo; + struct igt_spinner spin_hi, spin_lo; struct intel_engine_cs *engine; enum intel_engine_id id; + intel_wakeref_t wakeref; int err = -ENOMEM; if (!HAS_LOGICAL_RING_PREEMPTION(i915)) @@ -473,12 +524,12 @@ static int live_preempt_hang(void *arg) return 0; mutex_lock(&i915->drm.struct_mutex); - intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(i915); - if (spinner_init(&spin_hi, i915)) + if (igt_spinner_init(&spin_hi, i915)) goto err_unlock; - if (spinner_init(&spin_lo, i915)) + if (igt_spinner_init(&spin_lo, i915)) goto err_spin_hi; ctx_hi = kernel_context(i915); @@ -497,15 +548,15 @@ static int live_preempt_hang(void *arg) if (!intel_engine_has_preemption(engine)) continue; - rq = spinner_create_request(&spin_lo, ctx_lo, engine, - MI_ARB_CHECK); + rq = igt_spinner_create_request(&spin_lo, ctx_lo, engine, + MI_ARB_CHECK); if (IS_ERR(rq)) { err = PTR_ERR(rq); goto err_ctx_lo; } i915_request_add(rq); - if (!wait_for_spinner(&spin_lo, rq)) { + if (!igt_wait_for_spinner(&spin_lo, rq)) { GEM_TRACE("lo spinner failed to start\n"); GEM_TRACE_DUMP(); i915_gem_set_wedged(i915); @@ -513,10 +564,10 @@ static int live_preempt_hang(void *arg) goto err_ctx_lo; } - rq = spinner_create_request(&spin_hi, ctx_hi, engine, - MI_ARB_CHECK); + rq = igt_spinner_create_request(&spin_hi, ctx_hi, engine, + MI_ARB_CHECK); if (IS_ERR(rq)) { - spinner_end(&spin_lo); + igt_spinner_end(&spin_lo); err = PTR_ERR(rq); goto err_ctx_lo; } @@ -541,7 +592,7 @@ static int live_preempt_hang(void *arg) engine->execlists.preempt_hang.inject_hang = false; - if (!wait_for_spinner(&spin_hi, rq)) { + if (!igt_wait_for_spinner(&spin_hi, rq)) { GEM_TRACE("hi spinner failed to start\n"); GEM_TRACE_DUMP(); i915_gem_set_wedged(i915); @@ -549,8 +600,8 @@ static int live_preempt_hang(void *arg) goto err_ctx_lo; } - spinner_end(&spin_hi); - spinner_end(&spin_lo); + igt_spinner_end(&spin_hi); + igt_spinner_end(&spin_lo); if (igt_flush_test(i915, I915_WAIT_LOCKED)) { err = -EIO; goto err_ctx_lo; @@ -563,23 +614,282 @@ err_ctx_lo: err_ctx_hi: kernel_context_close(ctx_hi); err_spin_lo: - spinner_fini(&spin_lo); + igt_spinner_fini(&spin_lo); err_spin_hi: - spinner_fini(&spin_hi); + igt_spinner_fini(&spin_hi); err_unlock: igt_flush_test(i915, I915_WAIT_LOCKED); - intel_runtime_pm_put(i915); + intel_runtime_pm_put(i915, wakeref); mutex_unlock(&i915->drm.struct_mutex); return err; } +static int random_range(struct rnd_state *rnd, int min, int max) +{ + return i915_prandom_u32_max_state(max - min, rnd) + min; +} + +static int random_priority(struct rnd_state *rnd) +{ + return random_range(rnd, I915_PRIORITY_MIN, I915_PRIORITY_MAX); +} + +struct preempt_smoke { + struct drm_i915_private *i915; + struct i915_gem_context **contexts; + struct intel_engine_cs *engine; + struct drm_i915_gem_object *batch; + unsigned int ncontext; + struct rnd_state prng; + unsigned long count; +}; + +static struct i915_gem_context *smoke_context(struct preempt_smoke *smoke) +{ + return smoke->contexts[i915_prandom_u32_max_state(smoke->ncontext, + &smoke->prng)]; +} + +static int smoke_submit(struct preempt_smoke *smoke, + struct i915_gem_context *ctx, int prio, + struct drm_i915_gem_object *batch) +{ + struct i915_request *rq; + struct i915_vma *vma = NULL; + int err = 0; + + if (batch) { + vma = i915_vma_instance(batch, &ctx->ppgtt->vm, NULL); + if (IS_ERR(vma)) + return PTR_ERR(vma); + + err = i915_vma_pin(vma, 0, 0, PIN_USER); + if (err) + return err; + } + + ctx->sched.priority = prio; + + rq = i915_request_alloc(smoke->engine, ctx); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto unpin; + } + + if (vma) { + err = rq->engine->emit_bb_start(rq, + vma->node.start, + PAGE_SIZE, 0); + if (!err) + err = i915_vma_move_to_active(vma, rq, 0); + } + + i915_request_add(rq); + +unpin: + if (vma) + i915_vma_unpin(vma); + + return err; +} + +static int smoke_crescendo_thread(void *arg) +{ + struct preempt_smoke *smoke = arg; + IGT_TIMEOUT(end_time); + unsigned long count; + + count = 0; + do { + struct i915_gem_context *ctx = smoke_context(smoke); + int err; + + mutex_lock(&smoke->i915->drm.struct_mutex); + err = smoke_submit(smoke, + ctx, count % I915_PRIORITY_MAX, + smoke->batch); + mutex_unlock(&smoke->i915->drm.struct_mutex); + if (err) + return err; + + count++; + } while (!__igt_timeout(end_time, NULL)); + + smoke->count = count; + return 0; +} + +static int smoke_crescendo(struct preempt_smoke *smoke, unsigned int flags) +#define BATCH BIT(0) +{ + struct task_struct *tsk[I915_NUM_ENGINES] = {}; + struct preempt_smoke arg[I915_NUM_ENGINES]; + struct intel_engine_cs *engine; + enum intel_engine_id id; + unsigned long count; + int err = 0; + + mutex_unlock(&smoke->i915->drm.struct_mutex); + + for_each_engine(engine, smoke->i915, id) { + arg[id] = *smoke; + arg[id].engine = engine; + if (!(flags & BATCH)) + arg[id].batch = NULL; + arg[id].count = 0; + + tsk[id] = kthread_run(smoke_crescendo_thread, &arg, + "igt/smoke:%d", id); + if (IS_ERR(tsk[id])) { + err = PTR_ERR(tsk[id]); + break; + } + get_task_struct(tsk[id]); + } + + count = 0; + for_each_engine(engine, smoke->i915, id) { + int status; + + if (IS_ERR_OR_NULL(tsk[id])) + continue; + + status = kthread_stop(tsk[id]); + if (status && !err) + err = status; + + count += arg[id].count; + + put_task_struct(tsk[id]); + } + + mutex_lock(&smoke->i915->drm.struct_mutex); + + pr_info("Submitted %lu crescendo:%x requests across %d engines and %d contexts\n", + count, flags, + RUNTIME_INFO(smoke->i915)->num_rings, smoke->ncontext); + return 0; +} + +static int smoke_random(struct preempt_smoke *smoke, unsigned int flags) +{ + enum intel_engine_id id; + IGT_TIMEOUT(end_time); + unsigned long count; + + count = 0; + do { + for_each_engine(smoke->engine, smoke->i915, id) { + struct i915_gem_context *ctx = smoke_context(smoke); + int err; + + err = smoke_submit(smoke, + ctx, random_priority(&smoke->prng), + flags & BATCH ? smoke->batch : NULL); + if (err) + return err; + + count++; + } + } while (!__igt_timeout(end_time, NULL)); + + pr_info("Submitted %lu random:%x requests across %d engines and %d contexts\n", + count, flags, + RUNTIME_INFO(smoke->i915)->num_rings, smoke->ncontext); + return 0; +} + +static int live_preempt_smoke(void *arg) +{ + struct preempt_smoke smoke = { + .i915 = arg, + .prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed), + .ncontext = 1024, + }; + const unsigned int phase[] = { 0, BATCH }; + intel_wakeref_t wakeref; + int err = -ENOMEM; + u32 *cs; + int n; + + if (!HAS_LOGICAL_RING_PREEMPTION(smoke.i915)) + return 0; + + smoke.contexts = kmalloc_array(smoke.ncontext, + sizeof(*smoke.contexts), + GFP_KERNEL); + if (!smoke.contexts) + return -ENOMEM; + + mutex_lock(&smoke.i915->drm.struct_mutex); + wakeref = intel_runtime_pm_get(smoke.i915); + + smoke.batch = i915_gem_object_create_internal(smoke.i915, PAGE_SIZE); + if (IS_ERR(smoke.batch)) { + err = PTR_ERR(smoke.batch); + goto err_unlock; + } + + cs = i915_gem_object_pin_map(smoke.batch, I915_MAP_WB); + if (IS_ERR(cs)) { + err = PTR_ERR(cs); + goto err_batch; + } + for (n = 0; n < PAGE_SIZE / sizeof(*cs) - 1; n++) + cs[n] = MI_ARB_CHECK; + cs[n] = MI_BATCH_BUFFER_END; + i915_gem_object_unpin_map(smoke.batch); + + err = i915_gem_object_set_to_gtt_domain(smoke.batch, false); + if (err) + goto err_batch; + + for (n = 0; n < smoke.ncontext; n++) { + smoke.contexts[n] = kernel_context(smoke.i915); + if (!smoke.contexts[n]) + goto err_ctx; + } + + for (n = 0; n < ARRAY_SIZE(phase); n++) { + err = smoke_crescendo(&smoke, phase[n]); + if (err) + goto err_ctx; + + err = smoke_random(&smoke, phase[n]); + if (err) + goto err_ctx; + } + +err_ctx: + if (igt_flush_test(smoke.i915, I915_WAIT_LOCKED)) + err = -EIO; + + for (n = 0; n < smoke.ncontext; n++) { + if (!smoke.contexts[n]) + break; + kernel_context_close(smoke.contexts[n]); + } + +err_batch: + i915_gem_object_put(smoke.batch); +err_unlock: + intel_runtime_pm_put(smoke.i915, wakeref); + mutex_unlock(&smoke.i915->drm.struct_mutex); + kfree(smoke.contexts); + + return err; +} + int intel_execlists_live_selftests(struct drm_i915_private *i915) { static const struct i915_subtest tests[] = { SUBTEST(live_sanitycheck), SUBTEST(live_preempt), SUBTEST(live_late_preempt), + SUBTEST(live_suppress_self_preempt), + SUBTEST(live_chain_preempt), SUBTEST(live_preempt_hang), + SUBTEST(live_preempt_smoke), }; if (!HAS_EXECLISTS(i915)) diff --git a/drivers/gpu/drm/i915/selftests/intel_workarounds.c b/drivers/gpu/drm/i915/selftests/intel_workarounds.c index d1a0923d2f38..b15c4f26c593 100644 --- a/drivers/gpu/drm/i915/selftests/intel_workarounds.c +++ b/drivers/gpu/drm/i915/selftests/intel_workarounds.c @@ -5,17 +5,67 @@ */ #include "../i915_selftest.h" +#include "../i915_reset.h" +#include "igt_flush_test.h" +#include "igt_reset.h" +#include "igt_spinner.h" #include "igt_wedge_me.h" #include "mock_context.h" +#define REF_NAME_MAX (INTEL_ENGINE_CS_MAX_NAME + 4) +struct wa_lists { + struct i915_wa_list gt_wa_list; + struct { + char name[REF_NAME_MAX]; + struct i915_wa_list wa_list; + } engine[I915_NUM_ENGINES]; +}; + +static void +reference_lists_init(struct drm_i915_private *i915, struct wa_lists *lists) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + + memset(lists, 0, sizeof(*lists)); + + wa_init_start(&lists->gt_wa_list, "GT_REF"); + gt_init_workarounds(i915, &lists->gt_wa_list); + wa_init_finish(&lists->gt_wa_list); + + for_each_engine(engine, i915, id) { + struct i915_wa_list *wal = &lists->engine[id].wa_list; + char *name = lists->engine[id].name; + + snprintf(name, REF_NAME_MAX, "%s_REF", engine->name); + + wa_init_start(wal, name); + engine_init_workarounds(engine, wal); + wa_init_finish(wal); + } +} + +static void +reference_lists_fini(struct drm_i915_private *i915, struct wa_lists *lists) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + + for_each_engine(engine, i915, id) + intel_wa_list_free(&lists->engine[id].wa_list); + + intel_wa_list_free(&lists->gt_wa_list); +} + static struct drm_i915_gem_object * read_nonprivs(struct i915_gem_context *ctx, struct intel_engine_cs *engine) { + const u32 base = engine->mmio_base; struct drm_i915_gem_object *result; + intel_wakeref_t wakeref; struct i915_request *rq; struct i915_vma *vma; - const u32 base = engine->mmio_base; u32 srm, *cs; int err; int i; @@ -44,9 +94,9 @@ read_nonprivs(struct i915_gem_context *ctx, struct intel_engine_cs *engine) if (err) goto err_obj; - intel_runtime_pm_get(engine->i915); - rq = i915_request_alloc(engine, ctx); - intel_runtime_pm_put(engine->i915); + rq = ERR_PTR(-ENODEV); + with_intel_runtime_pm(engine->i915, wakeref) + rq = i915_request_alloc(engine, ctx); if (IS_ERR(rq)) { err = PTR_ERR(rq); goto err_pin; @@ -91,17 +141,23 @@ err_obj: return ERR_PTR(err); } -static u32 get_whitelist_reg(const struct whitelist *w, unsigned int i) +static u32 +get_whitelist_reg(const struct intel_engine_cs *engine, unsigned int i) { - return i < w->count ? i915_mmio_reg_offset(w->reg[i]) : w->nopid; + i915_reg_t reg = i < engine->whitelist.count ? + engine->whitelist.list[i].reg : + RING_NOPID(engine->mmio_base); + + return i915_mmio_reg_offset(reg); } -static void print_results(const struct whitelist *w, const u32 *results) +static void +print_results(const struct intel_engine_cs *engine, const u32 *results) { unsigned int i; for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { - u32 expected = get_whitelist_reg(w, i); + u32 expected = get_whitelist_reg(engine, i); u32 actual = results[i]; pr_info("RING_NONPRIV[%d]: expected 0x%08x, found 0x%08x\n", @@ -109,8 +165,7 @@ static void print_results(const struct whitelist *w, const u32 *results) } } -static int check_whitelist(const struct whitelist *w, - struct i915_gem_context *ctx, +static int check_whitelist(struct i915_gem_context *ctx, struct intel_engine_cs *engine) { struct drm_i915_gem_object *results; @@ -138,11 +193,11 @@ static int check_whitelist(const struct whitelist *w, } for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { - u32 expected = get_whitelist_reg(w, i); + u32 expected = get_whitelist_reg(engine, i); u32 actual = vaddr[i]; if (expected != actual) { - print_results(w, vaddr); + print_results(engine, vaddr); pr_err("Invalid RING_NONPRIV[%d], expected 0x%08x, found 0x%08x\n", i, expected, actual); @@ -159,66 +214,108 @@ out_put: static int do_device_reset(struct intel_engine_cs *engine) { - i915_reset(engine->i915, ENGINE_MASK(engine->id), NULL); + i915_reset(engine->i915, ENGINE_MASK(engine->id), "live_workarounds"); return 0; } static int do_engine_reset(struct intel_engine_cs *engine) { - return i915_reset_engine(engine, NULL); + return i915_reset_engine(engine, "live_workarounds"); } -static int switch_to_scratch_context(struct intel_engine_cs *engine) +static int +switch_to_scratch_context(struct intel_engine_cs *engine, + struct igt_spinner *spin) { struct i915_gem_context *ctx; struct i915_request *rq; + intel_wakeref_t wakeref; + int err = 0; ctx = kernel_context(engine->i915); if (IS_ERR(ctx)) return PTR_ERR(ctx); - intel_runtime_pm_get(engine->i915); - rq = i915_request_alloc(engine, ctx); - intel_runtime_pm_put(engine->i915); + rq = ERR_PTR(-ENODEV); + with_intel_runtime_pm(engine->i915, wakeref) { + if (spin) + rq = igt_spinner_create_request(spin, + ctx, engine, + MI_NOOP); + else + rq = i915_request_alloc(engine, ctx); + } kernel_context_close(ctx); - if (IS_ERR(rq)) - return PTR_ERR(rq); + + if (IS_ERR(rq)) { + spin = NULL; + err = PTR_ERR(rq); + goto err; + } i915_request_add(rq); - return 0; + if (spin && !igt_wait_for_spinner(spin, rq)) { + pr_err("Spinner failed to start\n"); + err = -ETIMEDOUT; + } + +err: + if (err && spin) + igt_spinner_end(spin); + + return err; } static int check_whitelist_across_reset(struct intel_engine_cs *engine, int (*reset)(struct intel_engine_cs *), - const struct whitelist *w, const char *name) { + struct drm_i915_private *i915 = engine->i915; + bool want_spin = reset == do_engine_reset; struct i915_gem_context *ctx; + struct igt_spinner spin; + intel_wakeref_t wakeref; int err; - ctx = kernel_context(engine->i915); + pr_info("Checking %d whitelisted registers (RING_NONPRIV) [%s]\n", + engine->whitelist.count, name); + + if (want_spin) { + err = igt_spinner_init(&spin, i915); + if (err) + return err; + } + + ctx = kernel_context(i915); if (IS_ERR(ctx)) return PTR_ERR(ctx); - err = check_whitelist(w, ctx, engine); + err = check_whitelist(ctx, engine); if (err) { pr_err("Invalid whitelist *before* %s reset!\n", name); goto out; } - err = switch_to_scratch_context(engine); + err = switch_to_scratch_context(engine, want_spin ? &spin : NULL); if (err) goto out; - err = reset(engine); + with_intel_runtime_pm(i915, wakeref) + err = reset(engine); + + if (want_spin) { + igt_spinner_end(&spin); + igt_spinner_fini(&spin); + } + if (err) { pr_err("%s reset failed\n", name); goto out; } - err = check_whitelist(w, ctx, engine); + err = check_whitelist(ctx, engine); if (err) { pr_err("Whitelist not preserved in context across %s reset!\n", name); @@ -227,11 +324,11 @@ static int check_whitelist_across_reset(struct intel_engine_cs *engine, kernel_context_close(ctx); - ctx = kernel_context(engine->i915); + ctx = kernel_context(i915); if (IS_ERR(ctx)) return PTR_ERR(ctx); - err = check_whitelist(w, ctx, engine); + err = check_whitelist(ctx, engine); if (err) { pr_err("Invalid whitelist *after* %s reset in fresh context!\n", name); @@ -247,26 +344,18 @@ static int live_reset_whitelist(void *arg) { struct drm_i915_private *i915 = arg; struct intel_engine_cs *engine = i915->engine[RCS]; - struct i915_gpu_error *error = &i915->gpu_error; - struct whitelist w; int err = 0; /* If we reset the gpu, we should not lose the RING_NONPRIV */ - if (!engine) + if (!engine || engine->whitelist.count == 0) return 0; - if (!whitelist_build(engine, &w)) - return 0; - - pr_info("Checking %d whitelisted registers (RING_NONPRIV)\n", w.count); - - set_bit(I915_RESET_BACKOFF, &error->flags); - set_bit(I915_RESET_ENGINE + engine->id, &error->flags); + igt_global_reset_lock(i915); if (intel_has_reset_engine(i915)) { err = check_whitelist_across_reset(engine, - do_engine_reset, &w, + do_engine_reset, "engine"); if (err) goto out; @@ -274,22 +363,159 @@ static int live_reset_whitelist(void *arg) if (intel_has_gpu_reset(i915)) { err = check_whitelist_across_reset(engine, - do_device_reset, &w, + do_device_reset, "device"); if (err) goto out; } out: - clear_bit(I915_RESET_ENGINE + engine->id, &error->flags); - clear_bit(I915_RESET_BACKOFF, &error->flags); + igt_global_reset_unlock(i915); return err; } +static bool verify_gt_engine_wa(struct drm_i915_private *i915, + struct wa_lists *lists, const char *str) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + bool ok = true; + + ok &= wa_list_verify(i915, &lists->gt_wa_list, str); + + for_each_engine(engine, i915, id) + ok &= wa_list_verify(i915, &lists->engine[id].wa_list, str); + + return ok; +} + +static int +live_gpu_reset_gt_engine_workarounds(void *arg) +{ + struct drm_i915_private *i915 = arg; + intel_wakeref_t wakeref; + struct wa_lists lists; + bool ok; + + if (!intel_has_gpu_reset(i915)) + return 0; + + pr_info("Verifying after GPU reset...\n"); + + igt_global_reset_lock(i915); + wakeref = intel_runtime_pm_get(i915); + + reference_lists_init(i915, &lists); + + ok = verify_gt_engine_wa(i915, &lists, "before reset"); + if (!ok) + goto out; + + i915_reset(i915, ALL_ENGINES, "live_workarounds"); + + ok = verify_gt_engine_wa(i915, &lists, "after reset"); + +out: + reference_lists_fini(i915, &lists); + intel_runtime_pm_put(i915, wakeref); + igt_global_reset_unlock(i915); + + return ok ? 0 : -ESRCH; +} + +static int +live_engine_reset_gt_engine_workarounds(void *arg) +{ + struct drm_i915_private *i915 = arg; + struct intel_engine_cs *engine; + struct i915_gem_context *ctx; + struct igt_spinner spin; + enum intel_engine_id id; + struct i915_request *rq; + intel_wakeref_t wakeref; + struct wa_lists lists; + int ret = 0; + + if (!intel_has_reset_engine(i915)) + return 0; + + ctx = kernel_context(i915); + if (IS_ERR(ctx)) + return PTR_ERR(ctx); + + igt_global_reset_lock(i915); + wakeref = intel_runtime_pm_get(i915); + + reference_lists_init(i915, &lists); + + for_each_engine(engine, i915, id) { + bool ok; + + pr_info("Verifying after %s reset...\n", engine->name); + + ok = verify_gt_engine_wa(i915, &lists, "before reset"); + if (!ok) { + ret = -ESRCH; + goto err; + } + + i915_reset_engine(engine, "live_workarounds"); + + ok = verify_gt_engine_wa(i915, &lists, "after idle reset"); + if (!ok) { + ret = -ESRCH; + goto err; + } + + ret = igt_spinner_init(&spin, i915); + if (ret) + goto err; + + rq = igt_spinner_create_request(&spin, ctx, engine, MI_NOOP); + if (IS_ERR(rq)) { + ret = PTR_ERR(rq); + igt_spinner_fini(&spin); + goto err; + } + + i915_request_add(rq); + + if (!igt_wait_for_spinner(&spin, rq)) { + pr_err("Spinner failed to start\n"); + igt_spinner_fini(&spin); + ret = -ETIMEDOUT; + goto err; + } + + i915_reset_engine(engine, "live_workarounds"); + + igt_spinner_end(&spin); + igt_spinner_fini(&spin); + + ok = verify_gt_engine_wa(i915, &lists, "after busy reset"); + if (!ok) { + ret = -ESRCH; + goto err; + } + } + +err: + reference_lists_fini(i915, &lists); + intel_runtime_pm_put(i915, wakeref); + igt_global_reset_unlock(i915); + kernel_context_close(ctx); + + igt_flush_test(i915, I915_WAIT_LOCKED); + + return ret; +} + int intel_workarounds_live_selftests(struct drm_i915_private *i915) { static const struct i915_subtest tests[] = { SUBTEST(live_reset_whitelist), + SUBTEST(live_gpu_reset_gt_engine_workarounds), + SUBTEST(live_engine_reset_gt_engine_workarounds), }; int err; diff --git a/drivers/gpu/drm/i915/selftests/lib_sw_fence.c b/drivers/gpu/drm/i915/selftests/lib_sw_fence.c index b26f07b55d86..2bfa72c1654b 100644 --- a/drivers/gpu/drm/i915/selftests/lib_sw_fence.c +++ b/drivers/gpu/drm/i915/selftests/lib_sw_fence.c @@ -76,3 +76,57 @@ void timed_fence_fini(struct timed_fence *tf) destroy_timer_on_stack(&tf->timer); i915_sw_fence_fini(&tf->fence); } + +struct heap_fence { + struct i915_sw_fence fence; + union { + struct kref ref; + struct rcu_head rcu; + }; +}; + +static int __i915_sw_fence_call +heap_fence_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) +{ + struct heap_fence *h = container_of(fence, typeof(*h), fence); + + switch (state) { + case FENCE_COMPLETE: + break; + + case FENCE_FREE: + heap_fence_put(&h->fence); + } + + return NOTIFY_DONE; +} + +struct i915_sw_fence *heap_fence_create(gfp_t gfp) +{ + struct heap_fence *h; + + h = kmalloc(sizeof(*h), gfp); + if (!h) + return NULL; + + i915_sw_fence_init(&h->fence, heap_fence_notify); + refcount_set(&h->ref.refcount, 2); + + return &h->fence; +} + +static void heap_fence_release(struct kref *ref) +{ + struct heap_fence *h = container_of(ref, typeof(*h), ref); + + i915_sw_fence_fini(&h->fence); + + kfree_rcu(h, rcu); +} + +void heap_fence_put(struct i915_sw_fence *fence) +{ + struct heap_fence *h = container_of(fence, typeof(*h), fence); + + kref_put(&h->ref, heap_fence_release); +} diff --git a/drivers/gpu/drm/i915/selftests/lib_sw_fence.h b/drivers/gpu/drm/i915/selftests/lib_sw_fence.h index 474aafb92ae1..1f9927e10f3a 100644 --- a/drivers/gpu/drm/i915/selftests/lib_sw_fence.h +++ b/drivers/gpu/drm/i915/selftests/lib_sw_fence.h @@ -39,4 +39,7 @@ struct timed_fence { void timed_fence_init(struct timed_fence *tf, unsigned long expires); void timed_fence_fini(struct timed_fence *tf); +struct i915_sw_fence *heap_fence_create(gfp_t gfp); +void heap_fence_put(struct i915_sw_fence *fence); + #endif /* _LIB_SW_FENCE_H_ */ diff --git a/drivers/gpu/drm/i915/selftests/mock_context.c b/drivers/gpu/drm/i915/selftests/mock_context.c index d937bdff26f9..b646cdcdd602 100644 --- a/drivers/gpu/drm/i915/selftests/mock_context.c +++ b/drivers/gpu/drm/i915/selftests/mock_context.c @@ -45,11 +45,8 @@ mock_context(struct drm_i915_private *i915, INIT_LIST_HEAD(&ctx->handles_list); INIT_LIST_HEAD(&ctx->hw_id_link); - for (n = 0; n < ARRAY_SIZE(ctx->__engine); n++) { - struct intel_context *ce = &ctx->__engine[n]; - - ce->gem_context = ctx; - } + for (n = 0; n < ARRAY_SIZE(ctx->__engine); n++) + intel_context_init(&ctx->__engine[n], ctx, i915->engine[n]); ret = i915_gem_context_pin_hw_id(ctx); if (ret < 0) diff --git a/drivers/gpu/drm/i915/selftests/mock_engine.c b/drivers/gpu/drm/i915/selftests/mock_engine.c index 22a73da45ad5..08f0cab02e0f 100644 --- a/drivers/gpu/drm/i915/selftests/mock_engine.c +++ b/drivers/gpu/drm/i915/selftests/mock_engine.c @@ -30,6 +30,52 @@ struct mock_ring { struct i915_timeline timeline; }; +static void mock_timeline_pin(struct i915_timeline *tl) +{ + tl->pin_count++; +} + +static void mock_timeline_unpin(struct i915_timeline *tl) +{ + GEM_BUG_ON(!tl->pin_count); + tl->pin_count--; +} + +static struct intel_ring *mock_ring(struct intel_engine_cs *engine) +{ + const unsigned long sz = PAGE_SIZE / 2; + struct mock_ring *ring; + + ring = kzalloc(sizeof(*ring) + sz, GFP_KERNEL); + if (!ring) + return NULL; + + if (i915_timeline_init(engine->i915, + &ring->timeline, engine->name, + NULL)) { + kfree(ring); + return NULL; + } + + ring->base.size = sz; + ring->base.effective_size = sz; + ring->base.vaddr = (void *)(ring + 1); + ring->base.timeline = &ring->timeline; + + INIT_LIST_HEAD(&ring->base.request_list); + intel_ring_update_space(&ring->base); + + return &ring->base; +} + +static void mock_ring_free(struct intel_ring *base) +{ + struct mock_ring *ring = container_of(base, typeof(*ring), base); + + i915_timeline_fini(&ring->timeline); + kfree(ring); +} + static struct mock_request *first_request(struct mock_engine *engine) { return list_first_entry_or_null(&engine->hw_queue, @@ -37,24 +83,29 @@ static struct mock_request *first_request(struct mock_engine *engine) link); } -static void advance(struct mock_engine *engine, - struct mock_request *request) +static void advance(struct mock_request *request) { list_del_init(&request->link); - mock_seqno_advance(&engine->base, request->base.global_seqno); + intel_engine_write_global_seqno(request->base.engine, + request->base.global_seqno); + i915_request_mark_complete(&request->base); + GEM_BUG_ON(!i915_request_completed(&request->base)); + + intel_engine_queue_breadcrumbs(request->base.engine); } static void hw_delay_complete(struct timer_list *t) { struct mock_engine *engine = from_timer(engine, t, hw_delay); struct mock_request *request; + unsigned long flags; - spin_lock(&engine->hw_lock); + spin_lock_irqsave(&engine->hw_lock, flags); /* Timer fired, first request is complete */ request = first_request(engine); if (request) - advance(engine, request); + advance(request); /* * Also immediately signal any subsequent 0-delay requests, but @@ -66,20 +117,24 @@ static void hw_delay_complete(struct timer_list *t) break; } - advance(engine, request); + advance(request); } - spin_unlock(&engine->hw_lock); + spin_unlock_irqrestore(&engine->hw_lock, flags); } static void mock_context_unpin(struct intel_context *ce) { + mock_timeline_unpin(ce->ring->timeline); i915_gem_context_put(ce->gem_context); } static void mock_context_destroy(struct intel_context *ce) { GEM_BUG_ON(ce->pin_count); + + if (ce->ring) + mock_ring_free(ce->ring); } static const struct intel_context_ops mock_context_ops = { @@ -92,14 +147,26 @@ mock_context_pin(struct intel_engine_cs *engine, struct i915_gem_context *ctx) { struct intel_context *ce = to_intel_context(ctx, engine); + int err = -ENOMEM; - if (!ce->pin_count++) { - i915_gem_context_get(ctx); - ce->ring = engine->buffer; - ce->ops = &mock_context_ops; + if (ce->pin_count++) + return ce; + + if (!ce->ring) { + ce->ring = mock_ring(engine); + if (!ce->ring) + goto err; } + mock_timeline_pin(ce->ring->timeline); + + ce->ops = &mock_context_ops; + i915_gem_context_get(ctx); return ce; + +err: + ce->pin_count = 0; + return ERR_PTR(err); } static int mock_request_alloc(struct i915_request *request) @@ -118,9 +185,9 @@ static int mock_emit_flush(struct i915_request *request, return 0; } -static void mock_emit_breadcrumb(struct i915_request *request, - u32 *flags) +static u32 *mock_emit_breadcrumb(struct i915_request *request, u32 *cs) { + return cs; } static void mock_submit_request(struct i915_request *request) @@ -128,51 +195,20 @@ static void mock_submit_request(struct i915_request *request) struct mock_request *mock = container_of(request, typeof(*mock), base); struct mock_engine *engine = container_of(request->engine, typeof(*engine), base); + unsigned long flags; i915_request_submit(request); GEM_BUG_ON(!request->global_seqno); - spin_lock_irq(&engine->hw_lock); + spin_lock_irqsave(&engine->hw_lock, flags); list_add_tail(&mock->link, &engine->hw_queue); if (mock->link.prev == &engine->hw_queue) { if (mock->delay) mod_timer(&engine->hw_delay, jiffies + mock->delay); else - advance(engine, mock); + advance(mock); } - spin_unlock_irq(&engine->hw_lock); -} - -static struct intel_ring *mock_ring(struct intel_engine_cs *engine) -{ - const unsigned long sz = PAGE_SIZE / 2; - struct mock_ring *ring; - - BUILD_BUG_ON(MIN_SPACE_FOR_ADD_REQUEST > sz); - - ring = kzalloc(sizeof(*ring) + sz, GFP_KERNEL); - if (!ring) - return NULL; - - i915_timeline_init(engine->i915, &ring->timeline, engine->name); - - ring->base.size = sz; - ring->base.effective_size = sz; - ring->base.vaddr = (void *)(ring + 1); - ring->base.timeline = &ring->timeline; - - INIT_LIST_HEAD(&ring->base.request_list); - intel_ring_update_space(&ring->base); - - return &ring->base; -} - -static void mock_ring_free(struct intel_ring *base) -{ - struct mock_ring *ring = container_of(base, typeof(*ring), base); - - i915_timeline_fini(&ring->timeline); - kfree(ring); + spin_unlock_irqrestore(&engine->hw_lock, flags); } struct intel_engine_cs *mock_engine(struct drm_i915_private *i915, @@ -191,39 +227,37 @@ struct intel_engine_cs *mock_engine(struct drm_i915_private *i915, engine->base.i915 = i915; snprintf(engine->base.name, sizeof(engine->base.name), "%s", name); engine->base.id = id; - engine->base.status_page.page_addr = (void *)(engine + 1); + engine->base.status_page.addr = (void *)(engine + 1); engine->base.context_pin = mock_context_pin; engine->base.request_alloc = mock_request_alloc; engine->base.emit_flush = mock_emit_flush; - engine->base.emit_breadcrumb = mock_emit_breadcrumb; + engine->base.emit_fini_breadcrumb = mock_emit_breadcrumb; engine->base.submit_request = mock_submit_request; - i915_timeline_init(i915, &engine->base.timeline, engine->base.name); - lockdep_set_subclass(&engine->base.timeline.lock, TIMELINE_ENGINE); + if (i915_timeline_init(i915, + &engine->base.timeline, + engine->base.name, + NULL)) + goto err_free; + i915_timeline_set_subclass(&engine->base.timeline, TIMELINE_ENGINE); intel_engine_init_breadcrumbs(&engine->base); - engine->base.breadcrumbs.mock = true; /* prevent touching HW for irqs */ /* fake hw queue */ spin_lock_init(&engine->hw_lock); timer_setup(&engine->hw_delay, hw_delay_complete, 0); INIT_LIST_HEAD(&engine->hw_queue); - engine->base.buffer = mock_ring(&engine->base); - if (!engine->base.buffer) - goto err_breadcrumbs; - if (IS_ERR(intel_context_pin(i915->kernel_context, &engine->base))) - goto err_ring; + goto err_breadcrumbs; return &engine->base; -err_ring: - mock_ring_free(engine->base.buffer); err_breadcrumbs: intel_engine_fini_breadcrumbs(&engine->base); i915_timeline_fini(&engine->base.timeline); +err_free: kfree(engine); return NULL; } @@ -237,16 +271,14 @@ void mock_engine_flush(struct intel_engine_cs *engine) del_timer_sync(&mock->hw_delay); spin_lock_irq(&mock->hw_lock); - list_for_each_entry_safe(request, rn, &mock->hw_queue, link) { - list_del_init(&request->link); - mock_seqno_advance(&mock->base, request->base.global_seqno); - } + list_for_each_entry_safe(request, rn, &mock->hw_queue, link) + advance(request); spin_unlock_irq(&mock->hw_lock); } void mock_engine_reset(struct intel_engine_cs *engine) { - intel_write_status_page(engine, I915_GEM_HWS_INDEX, 0); + intel_engine_write_global_seqno(engine, 0); } void mock_engine_free(struct intel_engine_cs *engine) @@ -263,8 +295,6 @@ void mock_engine_free(struct intel_engine_cs *engine) __intel_context_unpin(engine->i915->kernel_context, engine); - mock_ring_free(engine->buffer); - intel_engine_fini_breadcrumbs(engine); i915_timeline_fini(&engine->timeline); diff --git a/drivers/gpu/drm/i915/selftests/mock_engine.h b/drivers/gpu/drm/i915/selftests/mock_engine.h index 133d0c21790d..b9cc3a245f16 100644 --- a/drivers/gpu/drm/i915/selftests/mock_engine.h +++ b/drivers/gpu/drm/i915/selftests/mock_engine.h @@ -46,10 +46,4 @@ void mock_engine_flush(struct intel_engine_cs *engine); void mock_engine_reset(struct intel_engine_cs *engine); void mock_engine_free(struct intel_engine_cs *engine); -static inline void mock_seqno_advance(struct intel_engine_cs *engine, u32 seqno) -{ - intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno); - intel_engine_wakeup(engine); -} - #endif /* !__MOCK_ENGINE_H__ */ diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c index 43ed8b28aeaa..14ae46fda49f 100644 --- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c +++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c @@ -58,8 +58,8 @@ static void mock_device_release(struct drm_device *dev) i915_gem_contexts_lost(i915); mutex_unlock(&i915->drm.struct_mutex); - cancel_delayed_work_sync(&i915->gt.retire_work); - cancel_delayed_work_sync(&i915->gt.idle_work); + drain_delayed_work(&i915->gt.retire_work); + drain_delayed_work(&i915->gt.idle_work); i915_gem_drain_workqueue(i915); mutex_lock(&i915->drm.struct_mutex); @@ -68,13 +68,14 @@ static void mock_device_release(struct drm_device *dev) i915_gem_contexts_fini(i915); mutex_unlock(&i915->drm.struct_mutex); + i915_timelines_fini(i915); + drain_workqueue(i915->wq); i915_gem_drain_freed_objects(i915); mutex_lock(&i915->drm.struct_mutex); - mock_fini_ggtt(i915); + mock_fini_ggtt(&i915->ggtt); mutex_unlock(&i915->drm.struct_mutex); - WARN_ON(!list_empty(&i915->gt.timelines)); destroy_workqueue(i915->wq); @@ -147,22 +148,24 @@ struct drm_i915_private *mock_gem_device(void) pdev->class = PCI_BASE_CLASS_DISPLAY << 16; pdev->dev.release = release_dev; dev_set_name(&pdev->dev, "mock"); - dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); #if IS_ENABLED(CONFIG_IOMMU_API) && defined(CONFIG_INTEL_IOMMU) /* hack to disable iommu for the fake device; force identity mapping */ pdev->dev.archdata.iommu = (void *)-1; #endif + i915 = (struct drm_i915_private *)(pdev + 1); + pci_set_drvdata(pdev, i915); + + intel_runtime_pm_init_early(i915); + dev_pm_domain_set(&pdev->dev, &pm_domain); pm_runtime_enable(&pdev->dev); pm_runtime_dont_use_autosuspend(&pdev->dev); if (pm_runtime_enabled(&pdev->dev)) WARN_ON(pm_runtime_get_sync(&pdev->dev)); - i915 = (struct drm_i915_private *)(pdev + 1); - pci_set_drvdata(pdev, i915); - err = drm_dev_init(&i915->drm, &mock_driver, &pdev->dev); if (err) { pr_err("Failed to initialise mock GEM device: err=%d\n", err); @@ -186,6 +189,7 @@ struct drm_i915_private *mock_gem_device(void) init_waitqueue_head(&i915->gpu_error.wait_queue); init_waitqueue_head(&i915->gpu_error.reset_queue); + mutex_init(&i915->gpu_error.wedge_mutex); i915->wq = alloc_ordered_workqueue("mock", 0); if (!i915->wq) @@ -223,13 +227,14 @@ struct drm_i915_private *mock_gem_device(void) if (!i915->priorities) goto err_dependencies; - INIT_LIST_HEAD(&i915->gt.timelines); + i915_timelines_init(i915); + INIT_LIST_HEAD(&i915->gt.active_rings); INIT_LIST_HEAD(&i915->gt.closed_vma); mutex_lock(&i915->drm.struct_mutex); - mock_init_ggtt(i915); + mock_init_ggtt(i915, &i915->ggtt); mkwrite_device_info(i915)->ring_mask = BIT(0); i915->kernel_context = mock_context(i915, NULL); @@ -250,6 +255,7 @@ err_context: i915_gem_contexts_fini(i915); err_unlock: mutex_unlock(&i915->drm.struct_mutex); + i915_timelines_fini(i915); kmem_cache_destroy(i915->priorities); err_dependencies: kmem_cache_destroy(i915->dependencies); diff --git a/drivers/gpu/drm/i915/selftests/mock_gtt.c b/drivers/gpu/drm/i915/selftests/mock_gtt.c index 6ae418c76015..cd83929fde8e 100644 --- a/drivers/gpu/drm/i915/selftests/mock_gtt.c +++ b/drivers/gpu/drm/i915/selftests/mock_gtt.c @@ -70,7 +70,7 @@ mock_ppgtt(struct drm_i915_private *i915, ppgtt->vm.total = round_down(U64_MAX, PAGE_SIZE); ppgtt->vm.file = ERR_PTR(-ENODEV); - i915_address_space_init(&ppgtt->vm, i915); + i915_address_space_init(&ppgtt->vm, VM_CLASS_PPGTT); ppgtt->vm.clear_range = nop_clear_range; ppgtt->vm.insert_page = mock_insert_page; @@ -97,11 +97,12 @@ static void mock_unbind_ggtt(struct i915_vma *vma) { } -void mock_init_ggtt(struct drm_i915_private *i915) +void mock_init_ggtt(struct drm_i915_private *i915, struct i915_ggtt *ggtt) { - struct i915_ggtt *ggtt = &i915->ggtt; + memset(ggtt, 0, sizeof(*ggtt)); ggtt->vm.i915 = i915; + ggtt->vm.is_ggtt = true; ggtt->gmadr = (struct resource) DEFINE_RES_MEM(0, 2048 * PAGE_SIZE); ggtt->mappable_end = resource_size(&ggtt->gmadr); @@ -117,14 +118,10 @@ void mock_init_ggtt(struct drm_i915_private *i915) ggtt->vm.vma_ops.set_pages = ggtt_set_pages; ggtt->vm.vma_ops.clear_pages = clear_pages; - i915_address_space_init(&ggtt->vm, i915); - - ggtt->vm.is_ggtt = true; + i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT); } -void mock_fini_ggtt(struct drm_i915_private *i915) +void mock_fini_ggtt(struct i915_ggtt *ggtt) { - struct i915_ggtt *ggtt = &i915->ggtt; - i915_address_space_fini(&ggtt->vm); } diff --git a/drivers/gpu/drm/i915/selftests/mock_gtt.h b/drivers/gpu/drm/i915/selftests/mock_gtt.h index 9a0a833bb545..40d544bde1d5 100644 --- a/drivers/gpu/drm/i915/selftests/mock_gtt.h +++ b/drivers/gpu/drm/i915/selftests/mock_gtt.h @@ -25,8 +25,8 @@ #ifndef __MOCK_GTT_H #define __MOCK_GTT_H -void mock_init_ggtt(struct drm_i915_private *i915); -void mock_fini_ggtt(struct drm_i915_private *i915); +void mock_init_ggtt(struct drm_i915_private *i915, struct i915_ggtt *ggtt); +void mock_fini_ggtt(struct i915_ggtt *ggtt); struct i915_hw_ppgtt * mock_ppgtt(struct drm_i915_private *i915, diff --git a/drivers/gpu/drm/i915/selftests/mock_timeline.c b/drivers/gpu/drm/i915/selftests/mock_timeline.c index dcf3b16f5a07..d2de9ece2118 100644 --- a/drivers/gpu/drm/i915/selftests/mock_timeline.c +++ b/drivers/gpu/drm/i915/selftests/mock_timeline.c @@ -10,11 +10,13 @@ void mock_timeline_init(struct i915_timeline *timeline, u64 context) { + timeline->i915 = NULL; timeline->fence_context = context; spin_lock_init(&timeline->lock); - init_request_active(&timeline->last_request, NULL); + INIT_ACTIVE_REQUEST(&timeline->barrier); + INIT_ACTIVE_REQUEST(&timeline->last_request); INIT_LIST_HEAD(&timeline->requests); i915_syncmap_init(&timeline->sync); @@ -24,5 +26,5 @@ void mock_timeline_init(struct i915_timeline *timeline, u64 context) void mock_timeline_fini(struct i915_timeline *timeline) { - i915_timeline_fini(timeline); + i915_syncmap_free(&timeline->sync); } diff --git a/drivers/gpu/drm/i915/vlv_dsi.c b/drivers/gpu/drm/i915/vlv_dsi.c index 435a2c35ee8c..31c93c3ccd00 100644 --- a/drivers/gpu/drm/i915/vlv_dsi.c +++ b/drivers/gpu/drm/i915/vlv_dsi.c @@ -23,7 +23,6 @@ * Author: Jani Nikula <jani.nikula@intel.com> */ -#include <drm/drmP.h> #include <drm/drm_atomic_helper.h> #include <drm/drm_crtc.h> #include <drm/drm_edid.h> @@ -206,39 +205,6 @@ static const struct mipi_dsi_host_ops intel_dsi_host_ops = { .transfer = intel_dsi_host_transfer, }; -static struct intel_dsi_host *intel_dsi_host_init(struct intel_dsi *intel_dsi, - enum port port) -{ - struct intel_dsi_host *host; - struct mipi_dsi_device *device; - - host = kzalloc(sizeof(*host), GFP_KERNEL); - if (!host) - return NULL; - - host->base.ops = &intel_dsi_host_ops; - host->intel_dsi = intel_dsi; - host->port = port; - - /* - * We should call mipi_dsi_host_register(&host->base) here, but we don't - * have a host->dev, and we don't have OF stuff either. So just use the - * dsi framework as a library and hope for the best. Create the dsi - * devices by ourselves here too. Need to be careful though, because we - * don't initialize any of the driver model devices here. - */ - device = kzalloc(sizeof(*device), GFP_KERNEL); - if (!device) { - kfree(host); - return NULL; - } - - device->host = &host->base; - host->device = device; - - return host; -} - /* * send a video mode command * @@ -290,19 +256,31 @@ static void band_gap_reset(struct drm_i915_private *dev_priv) mutex_unlock(&dev_priv->sb_lock); } -static inline bool is_vid_mode(struct intel_dsi *intel_dsi) +static int bdw_get_pipemisc_bpp(struct intel_crtc *crtc) { - return intel_dsi->operation_mode == INTEL_DSI_VIDEO_MODE; -} + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + u32 tmp; -static inline bool is_cmd_mode(struct intel_dsi *intel_dsi) -{ - return intel_dsi->operation_mode == INTEL_DSI_COMMAND_MODE; + tmp = I915_READ(PIPEMISC(crtc->pipe)); + + switch (tmp & PIPEMISC_DITHER_BPC_MASK) { + case PIPEMISC_DITHER_6_BPC: + return 18; + case PIPEMISC_DITHER_8_BPC: + return 24; + case PIPEMISC_DITHER_10_BPC: + return 30; + case PIPEMISC_DITHER_12_BPC: + return 36; + default: + MISSING_CASE(tmp); + return 0; + } } -static bool intel_dsi_compute_config(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config, - struct drm_connector_state *conn_state) +static int intel_dsi_compute_config(struct intel_encoder *encoder, + struct intel_crtc_state *pipe_config, + struct drm_connector_state *conn_state) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_dsi *intel_dsi = container_of(encoder, struct intel_dsi, @@ -314,11 +292,12 @@ static bool intel_dsi_compute_config(struct intel_encoder *encoder, int ret; DRM_DEBUG_KMS("\n"); + pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; if (fixed_mode) { intel_fixed_panel_mode(fixed_mode, adjusted_mode); - if (HAS_GMCH_DISPLAY(dev_priv)) + if (HAS_GMCH(dev_priv)) intel_gmch_panel_fitting(crtc, pipe_config, conn_state->scaling_mode); else @@ -327,11 +306,16 @@ static bool intel_dsi_compute_config(struct intel_encoder *encoder, } if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) - return false; + return -EINVAL; /* DSI uses short packets for sync events, so clear mode flags for DSI */ adjusted_mode->flags = 0; + if (intel_dsi->pixel_format == MIPI_DSI_FMT_RGB888) + pipe_config->pipe_bpp = 24; + else + pipe_config->pipe_bpp = 18; + if (IS_GEN9_LP(dev_priv)) { /* Enable Frame time stamp based scanline reporting */ adjusted_mode->private_flags |= @@ -345,16 +329,16 @@ static bool intel_dsi_compute_config(struct intel_encoder *encoder, ret = bxt_dsi_pll_compute(encoder, pipe_config); if (ret) - return false; + return -EINVAL; } else { ret = vlv_dsi_pll_compute(encoder, pipe_config); if (ret) - return false; + return -EINVAL; } pipe_config->clock_set = true; - return true; + return 0; } static bool glk_dsi_enable_io(struct intel_encoder *encoder) @@ -716,6 +700,10 @@ static void intel_dsi_port_enable(struct intel_encoder *encoder, LANE_CONFIGURATION_DUAL_LINK_B : LANE_CONFIGURATION_DUAL_LINK_A; } + + if (intel_dsi->pixel_format != MIPI_DSI_FMT_RGB888) + temp |= DITHERING_ENABLE; + /* assert ip_tg_enable signal */ I915_WRITE(port_ctrl, temp | DPI_ENABLE); POSTING_READ(port_ctrl); @@ -745,17 +733,6 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder, const struct intel_crtc_state *pipe_config); static void intel_dsi_unprepare(struct intel_encoder *encoder); -static void intel_dsi_msleep(struct intel_dsi *intel_dsi, int msec) -{ - struct drm_i915_private *dev_priv = to_i915(intel_dsi->base.base.dev); - - /* For v3 VBTs in vid-mode the delays are part of the VBT sequences */ - if (is_vid_mode(intel_dsi) && dev_priv->vbt.dsi.seq_version >= 3) - return; - - msleep(msec); -} - /* * Panel enable/disable sequences from the VBT spec. * @@ -793,6 +770,10 @@ static void intel_dsi_msleep(struct intel_dsi *intel_dsi, int msec) * - wait t4 - wait t4 */ +/* + * DSI port enable has to be done before pipe and plane enable, so we do it in + * the pre_enable hook instead of the enable hook. + */ static void intel_dsi_pre_enable(struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) @@ -895,17 +876,6 @@ static void intel_dsi_pre_enable(struct intel_encoder *encoder, } /* - * DSI port enable has to be done before pipe and plane enable, so we do it in - * the pre_enable hook. - */ -static void intel_dsi_enable_nop(struct intel_encoder *encoder, - const struct intel_crtc_state *pipe_config, - const struct drm_connector_state *conn_state) -{ - DRM_DEBUG_KMS("\n"); -} - -/* * DSI port disable has to be done after pipe and plane disable, so we do it in * the post_disable hook. */ @@ -1020,13 +990,15 @@ static bool intel_dsi_get_hw_state(struct intel_encoder *encoder, { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + intel_wakeref_t wakeref; enum port port; bool active = false; DRM_DEBUG_KMS("\n"); - if (!intel_display_power_get_if_enabled(dev_priv, - encoder->power_domain)) + wakeref = intel_display_power_get_if_enabled(dev_priv, + encoder->power_domain); + if (!wakeref) return false; /* @@ -1082,7 +1054,7 @@ static bool intel_dsi_get_hw_state(struct intel_encoder *encoder, } out_put_power: - intel_display_power_put(dev_priv, encoder->power_domain); + intel_display_power_put(dev_priv, encoder->power_domain, wakeref); return active; } @@ -1118,10 +1090,10 @@ static void bxt_dsi_get_pipe_config(struct intel_encoder *encoder, } fmt = I915_READ(MIPI_DSI_FUNC_PRG(port)) & VID_MODE_FORMAT_MASK; - pipe_config->pipe_bpp = - mipi_dsi_pixel_format_to_bpp( - pixel_format_from_register_bits(fmt)); - bpp = pipe_config->pipe_bpp; + bpp = mipi_dsi_pixel_format_to_bpp( + pixel_format_from_register_bits(fmt)); + + pipe_config->pipe_bpp = bdw_get_pipemisc_bpp(crtc); /* Enable Frame time stamo based scanline reporting */ adjusted_mode->private_flags |= @@ -1259,11 +1231,9 @@ static void intel_dsi_get_config(struct intel_encoder *encoder, if (IS_GEN9_LP(dev_priv)) { bxt_dsi_get_pipe_config(encoder, pipe_config); - pclk = bxt_dsi_get_pclk(encoder, pipe_config->pipe_bpp, - pipe_config); + pclk = bxt_dsi_get_pclk(encoder, pipe_config); } else { - pclk = vlv_dsi_get_pclk(encoder, pipe_config->pipe_bpp, - pipe_config); + pclk = vlv_dsi_get_pclk(encoder, pipe_config); } if (pclk) { @@ -1272,31 +1242,6 @@ static void intel_dsi_get_config(struct intel_encoder *encoder, } } -static enum drm_mode_status -intel_dsi_mode_valid(struct drm_connector *connector, - struct drm_display_mode *mode) -{ - struct intel_connector *intel_connector = to_intel_connector(connector); - const struct drm_display_mode *fixed_mode = intel_connector->panel.fixed_mode; - int max_dotclk = to_i915(connector->dev)->max_dotclk_freq; - - DRM_DEBUG_KMS("\n"); - - if (mode->flags & DRM_MODE_FLAG_DBLSCAN) - return MODE_NO_DBLESCAN; - - if (fixed_mode) { - if (mode->hdisplay > fixed_mode->hdisplay) - return MODE_PANEL; - if (mode->vdisplay > fixed_mode->vdisplay) - return MODE_PANEL; - if (fixed_mode->clock > max_dotclk) - return MODE_CLOCK_HIGH; - } - - return MODE_OK; -} - /* return txclkesc cycles in terms of divider and duration in us */ static u16 txclkesc(u32 divider, unsigned int us) { @@ -1619,39 +1564,6 @@ static void intel_dsi_unprepare(struct intel_encoder *encoder) } } -static int intel_dsi_get_modes(struct drm_connector *connector) -{ - struct intel_connector *intel_connector = to_intel_connector(connector); - struct drm_display_mode *mode; - - DRM_DEBUG_KMS("\n"); - - if (!intel_connector->panel.fixed_mode) { - DRM_DEBUG_KMS("no fixed mode\n"); - return 0; - } - - mode = drm_mode_duplicate(connector->dev, - intel_connector->panel.fixed_mode); - if (!mode) { - DRM_DEBUG_KMS("drm_mode_duplicate failed\n"); - return 0; - } - - drm_mode_probed_add(connector, mode); - return 1; -} - -static void intel_dsi_connector_destroy(struct drm_connector *connector) -{ - struct intel_connector *intel_connector = to_intel_connector(connector); - - DRM_DEBUG_KMS("\n"); - intel_panel_fini(&intel_connector->panel); - drm_connector_cleanup(connector); - kfree(connector); -} - static void intel_dsi_encoder_destroy(struct drm_encoder *encoder) { struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder); @@ -1676,7 +1588,7 @@ static const struct drm_connector_helper_funcs intel_dsi_connector_helper_funcs static const struct drm_connector_funcs intel_dsi_connector_funcs = { .late_register = intel_connector_register, .early_unregister = intel_connector_unregister, - .destroy = intel_dsi_connector_destroy, + .destroy = intel_connector_destroy, .fill_modes = drm_helper_probe_single_connector_modes, .atomic_get_property = intel_digital_connector_atomic_get_property, .atomic_set_property = intel_digital_connector_atomic_set_property, @@ -1684,27 +1596,59 @@ static const struct drm_connector_funcs intel_dsi_connector_funcs = { .atomic_duplicate_state = intel_digital_connector_duplicate_state, }; -static int intel_dsi_get_panel_orientation(struct intel_connector *connector) +static enum drm_panel_orientation +vlv_dsi_get_hw_panel_orientation(struct intel_connector *connector) { struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - int orientation = DRM_MODE_PANEL_ORIENTATION_NORMAL; - enum i9xx_plane_id i9xx_plane; + struct intel_encoder *encoder = connector->encoder; + enum intel_display_power_domain power_domain; + enum drm_panel_orientation orientation; + struct intel_plane *plane; + struct intel_crtc *crtc; + intel_wakeref_t wakeref; + enum pipe pipe; u32 val; - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { - if (connector->encoder->crtc_mask == BIT(PIPE_B)) - i9xx_plane = PLANE_B; - else - i9xx_plane = PLANE_A; + if (!encoder->get_hw_state(encoder, &pipe)) + return DRM_MODE_PANEL_ORIENTATION_UNKNOWN; - val = I915_READ(DSPCNTR(i9xx_plane)); - if (val & DISPPLANE_ROTATE_180) - orientation = DRM_MODE_PANEL_ORIENTATION_BOTTOM_UP; - } + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); + plane = to_intel_plane(crtc->base.primary); + + power_domain = POWER_DOMAIN_PIPE(pipe); + wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); + if (!wakeref) + return DRM_MODE_PANEL_ORIENTATION_UNKNOWN; + + val = I915_READ(DSPCNTR(plane->i9xx_plane)); + + if (!(val & DISPLAY_PLANE_ENABLE)) + orientation = DRM_MODE_PANEL_ORIENTATION_UNKNOWN; + else if (val & DISPPLANE_ROTATE_180) + orientation = DRM_MODE_PANEL_ORIENTATION_BOTTOM_UP; + else + orientation = DRM_MODE_PANEL_ORIENTATION_NORMAL; + + intel_display_power_put(dev_priv, power_domain, wakeref); return orientation; } +static enum drm_panel_orientation +vlv_dsi_get_panel_orientation(struct intel_connector *connector) +{ + struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + enum drm_panel_orientation orientation; + + if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { + orientation = vlv_dsi_get_hw_panel_orientation(connector); + if (orientation != DRM_MODE_PANEL_ORIENTATION_UNKNOWN) + return orientation; + } + + return intel_dsi_get_panel_orientation(connector); +} + static void intel_dsi_add_properties(struct intel_connector *connector) { struct drm_i915_private *dev_priv = to_i915(connector->base.dev); @@ -1713,7 +1657,7 @@ static void intel_dsi_add_properties(struct intel_connector *connector) u32 allowed_scalers; allowed_scalers = BIT(DRM_MODE_SCALE_ASPECT) | BIT(DRM_MODE_SCALE_FULLSCREEN); - if (!HAS_GMCH_DISPLAY(dev_priv)) + if (!HAS_GMCH(dev_priv)) allowed_scalers |= BIT(DRM_MODE_SCALE_CENTER); drm_connector_attach_scaling_mode_property(&connector->base, @@ -1722,7 +1666,7 @@ static void intel_dsi_add_properties(struct intel_connector *connector) connector->base.state->scaling_mode = DRM_MODE_SCALE_ASPECT; connector->base.display_info.panel_orientation = - intel_dsi_get_panel_orientation(connector); + vlv_dsi_get_panel_orientation(connector); drm_connector_init_panel_orientation_property( &connector->base, connector->panel.fixed_mode->hdisplay, @@ -1773,11 +1717,11 @@ void vlv_dsi_init(struct drm_i915_private *dev_priv) intel_encoder->compute_config = intel_dsi_compute_config; intel_encoder->pre_enable = intel_dsi_pre_enable; - intel_encoder->enable = intel_dsi_enable_nop; intel_encoder->disable = intel_dsi_disable; intel_encoder->post_disable = intel_dsi_post_disable; intel_encoder->get_hw_state = intel_dsi_get_hw_state; intel_encoder->get_config = intel_dsi_get_config; + intel_encoder->update_pipe = intel_panel_update_backlight; intel_connector->get_hw_state = intel_connector_get_hw_state; @@ -1806,7 +1750,8 @@ void vlv_dsi_init(struct drm_i915_private *dev_priv) for_each_dsi_port(port, intel_dsi->ports) { struct intel_dsi_host *host; - host = intel_dsi_host_init(intel_dsi, port); + host = intel_dsi_host_init(intel_dsi, &intel_dsi_host_ops, + port); if (!host) goto err; diff --git a/drivers/gpu/drm/i915/vlv_dsi_pll.c b/drivers/gpu/drm/i915/vlv_dsi_pll.c index a132a8037ecc..954d5a8c4fa7 100644 --- a/drivers/gpu/drm/i915/vlv_dsi_pll.c +++ b/drivers/gpu/drm/i915/vlv_dsi_pll.c @@ -252,20 +252,12 @@ void bxt_dsi_pll_disable(struct intel_encoder *encoder) DRM_ERROR("Timeout waiting for PLL lock deassertion\n"); } -static void assert_bpp_mismatch(enum mipi_dsi_pixel_format fmt, int pipe_bpp) -{ - int bpp = mipi_dsi_pixel_format_to_bpp(fmt); - - WARN(bpp != pipe_bpp, - "bpp match assertion failure (expected %d, current %d)\n", - bpp, pipe_bpp); -} - -u32 vlv_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp, +u32 vlv_dsi_get_pclk(struct intel_encoder *encoder, struct intel_crtc_state *config) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + int bpp = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format); u32 dsi_clock, pclk; u32 pll_ctl, pll_div; u32 m = 0, p = 0, n; @@ -319,15 +311,12 @@ u32 vlv_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp, dsi_clock = (m * refclk) / (p * n); - /* pixel_format and pipe_bpp should agree */ - assert_bpp_mismatch(intel_dsi->pixel_format, pipe_bpp); - - pclk = DIV_ROUND_CLOSEST(dsi_clock * intel_dsi->lane_count, pipe_bpp); + pclk = DIV_ROUND_CLOSEST(dsi_clock * intel_dsi->lane_count, bpp); return pclk; } -u32 bxt_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp, +u32 bxt_dsi_get_pclk(struct intel_encoder *encoder, struct intel_crtc_state *config) { u32 pclk; @@ -335,12 +324,7 @@ u32 bxt_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp, u32 dsi_ratio; struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - - /* Divide by zero */ - if (!pipe_bpp) { - DRM_ERROR("Invalid BPP(0)\n"); - return 0; - } + int bpp = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format); config->dsi_pll.ctrl = I915_READ(BXT_DSI_PLL_CTL); @@ -348,10 +332,7 @@ u32 bxt_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp, dsi_clk = (dsi_ratio * BXT_REF_CLOCK_KHZ) / 2; - /* pixel_format and pipe_bpp should agree */ - assert_bpp_mismatch(intel_dsi->pixel_format, pipe_bpp); - - pclk = DIV_ROUND_CLOSEST(dsi_clk * intel_dsi->lane_count, pipe_bpp); + pclk = DIV_ROUND_CLOSEST(dsi_clk * intel_dsi->lane_count, bpp); DRM_DEBUG_DRIVER("Calculated pclk=%u\n", pclk); return pclk; |